deepline 0.1.180 → 0.1.182

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -73,6 +73,7 @@ import {
 import { submitChildPlayThroughCoordinator } from './child-play-submit';
 import type { AnyBatchOperationStrategy } from '../../../shared_libs/play-runtime/batching-types';
 import {
+  attachToolResultListDataset,
   parseToolExecuteResponse,
   createToolExecuteResult,
   deserializeToolExecuteResult,
@@ -97,12 +98,20 @@ import {
   deriveToolRequestIdentity,
   derivePlayRowIdentity,
   derivePlayRowIdentityFromKey,
+  sha256Hex,
 } from '../../../shared_libs/plays/row-identity';
+import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
 import {
   buildDurableCtxCallCacheKey,
   buildDurableToolCallAuthScopeDigest,
   buildDurableToolCallCacheKey,
 } from '../../../shared_libs/play-runtime/durable-call-cache';
+import {
+  QUERY_RESULT_DATASET_PAGE_SIZE,
+  isCustomerDbDatasetTool,
+  isQueryResultDatasetReadRequest,
+  isQueryResultDatasetTool,
+} from '../../../shared_libs/play-runtime/query-result-dataset';
 import { buildScopedWorkReceiptKey } from '../../../shared_libs/play-runtime/work-receipts';
 import { DEDUPE_DUPLICATE_KEY_SAMPLE_CAP } from '../../../shared_libs/play-runtime/map-row-identity';
 import {
@@ -341,7 +350,40 @@ type WorkerFileRef = {
 
 const EXECUTE_TOOL_METADATA_HEADER = 'x-deepline-include-tool-metadata';
 const EXECUTE_RESPONSE_CONTRACT_HEADER = 'x-deepline-execute-response-contract';
+const EXECUTE_RESPONSE_INTENT_HEADER = 'x-deepline-execute-response-intent';
 const V2_EXECUTE_RESPONSE_CONTRACT = 'v2-tool-execution-result';
+type CustomerDbDatasetRequest = {
+  limit?: number;
+  offset?: number;
+  pageSize?: number;
+  totalRows?: number;
+};
+
+type WorkerToolDirectOptions = {
+  customerDbDataset?: CustomerDbDatasetRequest;
+  attachCustomerDbDataset?: boolean;
+};
+
+function rowsFromUnknown(value: unknown): Record<string, unknown>[] {
+  if (!Array.isArray(value)) return [];
+  return value.map((row) =>
+    row && typeof row === 'object' && !Array.isArray(row)
+      ? (row as Record<string, unknown>)
+      : { value: row },
+  );
+}
+
+function finiteNonNegativeInteger(value: unknown): number | null {
+  if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+    return null;
+  }
+  return Math.floor(value);
+}
+
+function finitePositiveInteger(value: unknown): number | null {
+  const integer = finiteNonNegativeInteger(value);
+  return integer !== null && integer > 0 ? integer : null;
+}
 
 function getStringField(value: unknown, key: string): string | null {
   if (!isRecord(value)) return null;
@@ -706,6 +748,8 @@ async function callRuntimeApiRpcBinding(
   if (metadata) headers[EXECUTE_TOOL_METADATA_HEADER] = metadata;
   const contract = h.get(EXECUTE_RESPONSE_CONTRACT_HEADER);
   if (contract) headers[EXECUTE_RESPONSE_CONTRACT_HEADER] = contract;
+  const responseIntent = h.get(EXECUTE_RESPONSE_INTENT_HEADER);
+  if (responseIntent) headers[EXECUTE_RESPONSE_INTENT_HEADER] = responseIntent;
   const rawBody = typeof init.body === 'string' ? init.body : '';
   const result = await binding.runtimeApiCall({
     executorToken: authorization.replace(/^Bearer\s+/i, '').trim(),
@@ -1291,6 +1335,7 @@ async function executeTool(
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
   runtimeTimeoutMs?: number,
+  directOptions?: WorkerToolDirectOptions,
 ): Promise<ToolExecuteResult> {
   if (args.toolId === 'test_wait_for_event' && workflowStep) {
     const result = await waitForSyntheticIntegrationEvent(
@@ -1314,6 +1359,7 @@ async function executeTool(
     abortSignal,
     runtimeDeadlineMs,
     runtimeTimeoutMs,
+    directOptions,
   );
 }
 
@@ -1328,6 +1374,7 @@ async function executeToolWithLifecycle(
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
   runtimeTimeoutMs?: number,
+  directOptions?: WorkerToolDirectOptions,
 ): Promise<ToolExecuteResult> {
   callbacks?.onToolCalled?.(args.toolId, nowMs());
   try {
@@ -1341,6 +1388,7 @@ async function executeToolWithLifecycle(
       abortSignal,
       runtimeDeadlineMs,
       runtimeTimeoutMs,
+      directOptions,
     );
   } catch (error) {
     callbacks?.onToolFailed?.(args.toolId, nowMs());
@@ -1490,6 +1538,7 @@ async function callToolDirect(
   abortSignal?: AbortSignal,
   runtimeDeadlineMs?: number,
   runtimeTimeoutMs?: number,
+  directOptions?: WorkerToolDirectOptions,
 ): Promise<ToolExecuteResult> {
   const { id, toolId, input } = args;
   const path = `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`;
@@ -1517,11 +1566,46 @@ async function callToolDirect(
             authorization: `Bearer ${req.executorToken}`,
             'x-deepline-request-id': `${req.runId}:${toolId}:${id}:attempt:${requestAttempt}`,
             [EXECUTE_RESPONSE_CONTRACT_HEADER]: V2_EXECUTE_RESPONSE_CONTRACT,
+            [EXECUTE_RESPONSE_INTENT_HEADER]: 'dataset',
             [EXECUTE_TOOL_METADATA_HEADER]: 'true',
           },
           body: JSON.stringify({
             payload: input,
-            metadata: { parent_run_id: req.runId },
+            metadata: {
+              parent_run_id: req.runId,
+              ...(directOptions?.customerDbDataset
+                ? {
+                    query_result_dataset: {
+                      limit: directOptions.customerDbDataset.limit,
+                      offset: directOptions.customerDbDataset.offset,
+                      page_size: directOptions.customerDbDataset.pageSize,
+                      ...(directOptions.customerDbDataset.totalRows !==
+                      undefined
+                        ? {
+                            total_rows:
+                              directOptions.customerDbDataset.totalRows,
+                          }
+                        : {}),
+                    },
+                    ...(isCustomerDbDatasetTool(toolId)
+                      ? {
+                          customer_db_dataset: {
+                            limit: directOptions.customerDbDataset.limit,
+                            offset: directOptions.customerDbDataset.offset,
+                            page_size: directOptions.customerDbDataset.pageSize,
+                            ...(directOptions.customerDbDataset.totalRows !==
+                            undefined
+                              ? {
+                                  total_rows:
+                                    directOptions.customerDbDataset.totalRows,
+                                }
+                              : {}),
+                          },
+                        }
+                      : {}),
+                  }
+                : {}),
+            },
             ...(req.integrationMode
               ? { integration_mode: req.integrationMode }
               : {}),
@@ -1561,6 +1645,9 @@ async function callToolDirect(
         parsed.result,
         parsed.metadata,
         parsed.status,
+        directOptions?.attachCustomerDbDataset === false
+          ? undefined
+          : { req, input },
       );
     }
 
@@ -1645,13 +1732,141 @@ function wrapWorkerToolResult(
   result: unknown,
   metadata: ToolResultMetadataInput | null,
   status = result == null ? 'no_result' : 'completed',
+  customerDbDatasetOptions?: {
+    req: RunRequest;
+    input: Record<string, unknown>;
+  },
 ): ToolExecuteResult {
-  return createToolExecuteResult({
+  const wrapped = createToolExecuteResult({
     status,
     result,
     metadata: metadata ?? { toolId },
     execution: toolExecutionMetadataForOutcome({ kind: 'live' }),
   });
+  return attachWorkerCustomerDbDatasetResult(
+    toolId,
+    wrapped,
+    customerDbDatasetOptions,
+  );
+}
+
+function attachWorkerCustomerDbDatasetResult(
+  toolId: string,
+  result: ToolExecuteResult,
+  options?: {
+    req: RunRequest;
+    input: Record<string, unknown>;
+  },
+): ToolExecuteResult {
+  if (!options || !isQueryResultDatasetTool(toolId)) return result;
+  const raw = isRecord(result.toolOutput.raw) ? result.toolOutput.raw : null;
+  const dataset = isRecord(raw?.dataset) ? raw.dataset : null;
+  const rows = rowsFromUnknown(raw?.rows);
+  const totalRows = finiteNonNegativeInteger(dataset?.total_rows);
+  const sql =
+    typeof options.input.sql === 'string'
+      ? options.input.sql
+      : typeof options.input.query === 'string'
+      ? options.input.query
+      : typeof raw?.sql === 'string'
+        ? raw.sql
+        : null;
+  if (!dataset || totalRows === null || !sql) {
+    return result;
+  }
+
+  const datasetLimit =
+    finitePositiveInteger(dataset.returned_limit) ?? totalRows;
+  const previewRows = rows.slice(0, Math.min(rows.length, 25));
+  const jobId = typeof result.job_id === 'string' ? result.job_id.trim() : '';
+  const datasetId = `worker-tool-list:${sha256Hex(
+    `${options.req.runId}:${jobId || Math.random()}`,
+  )}`;
+  const fetchPage = async (
+    offset: number,
+    limit: number,
+  ): Promise<Record<string, unknown>[]> => {
+    if (limit <= 0 || offset >= totalRows) return [];
+    const page = await callToolDirect(
+      options.req,
+      {
+        id: 'customer-db-dataset-page',
+        toolId,
+        input: options.input,
+      },
+      undefined,
+      undefined,
+      false,
+      undefined,
+      undefined,
+      undefined,
+      {
+        attachCustomerDbDataset: false,
+        customerDbDataset: {
+          limit: datasetLimit,
+          offset,
+          pageSize: Math.min(limit, QUERY_RESULT_DATASET_PAGE_SIZE),
+          totalRows,
+        },
+      },
+    );
+    const pageRaw = isRecord(page.toolOutput.raw) ? page.toolOutput.raw : null;
+    return rowsFromUnknown(pageRaw?.rows);
+  };
+  const collectRows = async (
+    limit: number | undefined,
+  ): Promise<Record<string, unknown>[]> => {
+    const target = Math.min(limit ?? totalRows, totalRows, datasetLimit);
+    const collected: Record<string, unknown>[] = [];
+    for (
+      let offset = 0;
+      offset < target;
+      offset += QUERY_RESULT_DATASET_PAGE_SIZE
+    ) {
+      collected.push(
+        ...(await fetchPage(
+          offset,
+          Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, target - offset),
+        )),
+      );
+    }
+    return collected.slice(0, target);
+  };
+  const playDataset = createDeferredPlayDataset({
+    datasetKind: 'csv',
+    datasetId,
+    count: Math.min(totalRows, datasetLimit),
+    previewRows,
+    sourceLabel: 'toolResponse.raw.rows',
+    tableNamespace: null,
+    resolvers: {
+      count: async () => Math.min(totalRows, datasetLimit),
+      peek: async (limit) => collectRows(limit),
+      materialize: async (limit) => collectRows(limit),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            const count = Math.min(totalRows, datasetLimit);
+            for (
+              let offset = 0;
+              offset < count;
+              offset += QUERY_RESULT_DATASET_PAGE_SIZE
+            ) {
+              yield* await fetchPage(
+                offset,
+                Math.min(QUERY_RESULT_DATASET_PAGE_SIZE, count - offset),
+              );
+            }
+          },
+        }) as AsyncIterable<Record<string, unknown>>,
+    },
+  });
+  return attachToolResultListDataset(result, {
+    name: 'rows',
+    path: 'toolResponse.raw.rows',
+    dataset: playDataset,
+    count: Math.min(totalRows, datasetLimit),
+  });
 }
 
 type RecordedStepProgramOutput = {
@@ -1685,6 +1900,7 @@ type WorkerToolBatchRequest = {
   toolId: string;
   input: Record<string, unknown>;
   workflowStep?: WorkflowStep;
+  directOptions?: WorkerToolDirectOptions;
   resolve: (value: unknown) => void;
   reject: (error: unknown) => void;
 };
@@ -1975,10 +2191,15 @@ class WorkerToolBatchScheduler {
         providerActionVersion,
         staleAfterSeconds: options?.staleAfterSeconds,
       });
+      const queryResultDatasetRead = isQueryResultDatasetReadRequest(
+        toolId,
+        input,
+      );
+      const receiptEligible = !queryResultDatasetRead;
       this.queue.push({
         id,
-        cacheKey: receiptKey,
-        receiptKey,
+        cacheKey: receiptEligible ? receiptKey : '',
+        receiptKey: receiptEligible ? receiptKey : null,
         force: options?.force === true || !!this.req.force,
         receiptWaitMaxAttempts: resolveRuntimeToolReceiptWaitMaxAttempts(
           typeof runtimeOptions?.receiptWaitMs === 'number'
@@ -1991,6 +2212,16 @@ class WorkerToolBatchScheduler {
         toolId,
         input,
         workflowStep,
+        ...(queryResultDatasetRead
+          ? {
+              directOptions: {
+                customerDbDataset: {
+                  offset: 0,
+                  pageSize: QUERY_RESULT_DATASET_PAGE_SIZE,
+                },
+              },
+            }
+          : {}),
         resolve: (value) => {
           recordRunnerPerfTrace({
             req: this.req,
@@ -2794,6 +3025,7 @@ class WorkerToolBatchScheduler {
                 resolveClaimedWorkerToolRuntimeTimeoutMs([claimed], {
                   runtimeDeadlineMs: this.runtimeDeadlineMs,
                 }),
+                request.directOptions,
               );
             } catch (error) {
               this.rejectRequests(
@@ -3056,6 +3288,11 @@ async function executeBatchedWorkerToolGroup(input: {
                 claimed.request.toolId,
                 splitResults[index] ?? null,
                 toolMetadataFallback(claimed.request.toolId),
+                splitResults[index] == null ? 'no_result' : 'completed',
+                {
+                  req: input.req,
+                  input: claimed.request.input,
+                },
               ),
             })),
           );

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts

@@ -5,6 +5,8 @@ import {
   sha256Hex,
 } from '../../../../shared_libs/plays/row-identity';
 import {
+  createDeferredPlayDataset,
+  isPlayDataset,
   isSerializedPlayDataset,
   type SerializedPlayDataset,
 } from '../../../../shared_libs/plays/dataset';
@@ -60,6 +62,22 @@ function datasetMetadata(value: unknown): Record<string, unknown> | null {
   }
 }
 
+function datasetCountFromMetadata(
+  metadata: Record<string, unknown> | null,
+): number {
+  return typeof metadata?.count === 'number' && Number.isFinite(metadata.count)
+    ? Math.max(0, Math.floor(metadata.count))
+    : 0;
+}
+
+function datasetPreviewFromMetadata(
+  metadata: Record<string, unknown> | null,
+): OutputDatasetRow[] {
+  return Array.isArray(metadata?.preview)
+    ? metadata.preview.filter(isRecord).map((row) => ({ ...row }))
+    : [];
+}
+
 function isRowArray(
   value: unknown,
   path: readonly string[],
@@ -226,6 +244,47 @@ function disambiguateTableNamespace(
   return candidate;
 }
 
+function collectExistingDatasetTableNamespaces(value: unknown): Set<string> {
+  const namespaces = new Set<string>();
+  const seen = new WeakSet<object>();
+
+  const addNamespace = (namespace: unknown) => {
+    if (typeof namespace !== 'string' || !namespace) return;
+    try {
+      namespaces.add(normalizeTableNamespace(namespace));
+    } catch {
+      // Historical malformed namespaces should not block generated result
+      // names. The original handle/envelope remains unchanged.
+    }
+  };
+
+  const visit = (candidate: unknown, depth: number): void => {
+    if (depth > 20 || candidate == null || typeof candidate !== 'object') {
+      return;
+    }
+    if (seen.has(candidate)) return;
+    seen.add(candidate);
+    if (isSerializedPlayDataset(candidate)) {
+      addNamespace(candidate.tableNamespace);
+      return;
+    }
+    if (isDatasetLike(candidate)) {
+      addNamespace(datasetMetadata(candidate)?.tableNamespace);
+      return;
+    }
+    if (Array.isArray(candidate)) {
+      for (const entry of candidate) visit(entry, depth + 1);
+      return;
+    }
+    for (const child of Object.values(candidate as Record<string, unknown>)) {
+      visit(child, depth + 1);
+    }
+  };
+
+  visit(value, 0);
+  return namespaces;
+}
+
 export function projectTerminalResultDatasets(
   value: unknown,
   createDataset: DatasetFactory,
@@ -239,30 +298,84 @@ export function projectTerminalResultDatasets(
   const handles: ProjectedResultDatasetHandle[] = [];
   const seen = new WeakMap<object, unknown>();
   const seenHandles = new WeakSet<object>();
+  const projectedHandles = new WeakMap<object, unknown>();
   const usedNamespaces = seedUsedNamespaces(options.reservedTableNamespaces);
+  for (const namespace of collectExistingDatasetTableNamespaces(value)) {
+    usedNamespaces.add(namespace);
+  }
   const maxTableNamespaceLength = maxTableNamespaceLengthForPlay(
     options.playName,
   );
 
-  const rememberHandle = (handle: unknown, path: readonly string[]): void => {
-    if (!isRecord(handle) || seenHandles.has(handle)) return;
+  const projectHandle = (handle: unknown, path: readonly string[]): unknown => {
+    if (!isRecord(handle)) return handle;
+    const existingProjection = projectedHandles.get(handle);
+    if (existingProjection) return existingProjection;
     const metadata = datasetMetadata(handle);
-    const tableNamespace =
-      typeof metadata?.tableNamespace === 'string'
+    let projectedHandle: unknown = handle;
+    let tableNamespace =
+      typeof metadata?.tableNamespace === 'string' && metadata.tableNamespace
         ? metadata.tableNamespace
         : null;
-    if (!tableNamespace) return;
-    seenHandles.add(handle);
+    if (!tableNamespace && isPlayDataset<OutputDatasetRow>(handle)) {
+      const resolvedPath = path.length > 0 ? path : ['rows'];
+      tableNamespace = disambiguateTableNamespace(
+        resultTableNamespaceForPath(resolvedPath, maxTableNamespaceLength),
+        resolvedPath,
+        usedNamespaces,
+        maxTableNamespaceLength,
+      );
+      projectedHandle = createDeferredPlayDataset({
+        datasetKind: 'csv',
+        datasetId:
+          typeof metadata?.datasetId === 'string'
+            ? metadata.datasetId
+            : `csv:${tableNamespace}`,
+        count: datasetCountFromMetadata(metadata),
+        previewRows: datasetPreviewFromMetadata(metadata),
+        sourceLabel:
+          typeof metadata?.sourceLabel === 'string'
+            ? metadata.sourceLabel
+            : null,
+        tableNamespace,
+        resolvers: {
+          count: () => handle.count(),
+          peek: (limit) => handle.peek(limit),
+          materialize: (limit) => handle.materialize(limit),
+          iterate: () =>
+            ({
+              async *[Symbol.asyncIterator]() {
+                yield* handle;
+              },
+            }) as AsyncIterable<OutputDatasetRow>,
+        },
+      });
+    }
+    projectedHandles.set(handle, projectedHandle);
+    if (!tableNamespace || seenHandles.has(projectedHandle as object)) {
+      return projectedHandle;
+    }
+    try {
+      usedNamespaces.add(normalizeTableNamespace(tableNamespace));
+    } catch {
+      // Existing dataset handles can carry historical namespaces. Keep the
+      // handle, but do not let a malformed namespace poison new result names.
+    }
+    seenHandles.add(projectedHandle as object);
+    const projectedMetadata =
+      projectedHandle === handle ? metadata : datasetMetadata(projectedHandle);
     const datasetKind =
-      metadata?.datasetKind === 'csv' || metadata?.datasetKind === 'map'
-        ? metadata.datasetKind
+      projectedMetadata?.datasetKind === 'csv' ||
+      projectedMetadata?.datasetKind === 'map'
+        ? projectedMetadata.datasetKind
         : null;
     handles.push({
       path: path.length > 0 ? path.join('.') : 'rows',
       tableNamespace,
       datasetKind,
-      handle,
+      handle: projectedHandle,
     });
+    return projectedHandle;
   };
 
   const promoteRows = (
@@ -287,7 +400,7 @@ export function projectTerminalResultDatasets(
       rows,
       handle,
     });
-    rememberHandle(handle, path);
+    projectHandle(handle, path);
     return handle;
   };
 
@@ -298,8 +411,7 @@ export function projectTerminalResultDatasets(
   ): unknown => {
     if (depth > 20 || candidate == null) return candidate;
     if (isDatasetLike(candidate)) {
-      rememberHandle(candidate, path);
-      return candidate;
+      return projectHandle(candidate, path);
     }
     if (isSerializedPlayDataset(candidate)) {
       return candidate;

package/dist/bundling-sources/sdk/src/client.ts

@@ -202,7 +202,8 @@ function chunkRegisterPlayArtifacts<T>(artifacts: T[]): T[][] {
 
 type ExecuteToolRawOptions = {
   includeToolMetadata?: boolean;
-  responseIntent?: 'raw' | 'row_artifact';
+  responseIntent?: 'dataset' | 'raw' | 'row_artifact';
+  metadata?: Record<string, unknown>;
   timeout?: number;
   maxRetries?: number;
 };
@@ -1310,13 +1311,14 @@ export class DeeplineClient {
       ...(options?.includeToolMetadata
         ? { [INCLUDE_TOOL_METADATA_HEADER]: 'true' }
         : {}),
-      ...(options?.responseIntent
-        ? { [EXECUTE_RESPONSE_INTENT_HEADER]: options.responseIntent }
-        : {}),
+      [EXECUTE_RESPONSE_INTENT_HEADER]: options?.responseIntent ?? 'raw',
     };
     return this.http.post<ToolExecution<TData, TMeta>>(
       `/api/v2/integrations/${encodeURIComponent(toolId)}/execute`,
-      { payload: input },
+      {
+        payload: input,
+        ...(options?.metadata ? { metadata: options.metadata } : {}),
+      },
       headers,
       {
         forbiddenAsApiError: true,