deepline 0.1.145 → 0.1.147

package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts

@@ -120,6 +120,7 @@ import {
   createInlineDatasetHandle,
   createMaterializedDatasetHandle,
   createPersistedDatasetHandle,
+  datasetHandleCapabilities,
   datasetRowCountHint,
   isDatasetHandle,
   iterDatasetChunks,
@@ -147,6 +148,12 @@ import {
   setHarnessBinding,
 } from '../../../sdk/src/plays/harness-stub';
 import { createHarnessWorkerReceiptStore } from './runtime/harness-receipt-store';
+import {
+  hydrateSerializedResultDatasets,
+  projectTerminalResultDatasets,
+  type ProjectedResultDatasetHandle,
+} from './runtime/output-datasets';
+import { collectStableResultDatasetPersistenceContract } from './runtime/result-dataset-persistence';
 import {
   runtimeCsvExecutionRow,
   publicCsvInputRow,
@@ -701,6 +708,39 @@ function buildOrderedNodeList(
   }));
 }
 
+function collectReservedResultDatasetNamespaces(
+  contractSnapshot: unknown,
+): string[] {
+  const snapshot = isRecord(contractSnapshot) ? contractSnapshot : null;
+  const pipeline =
+    (snapshot?.staticPipeline as PlayStaticPipeline | null | undefined) ?? null;
+  if (!pipeline) return [];
+
+  const namespaces = new Set<string>();
+  const addNamespace = (value: unknown) => {
+    if (typeof value === 'string' && value.trim()) {
+      namespaces.add(value);
+    }
+  };
+  addNamespace(pipeline.tableNamespace);
+  addNamespace(pipeline.sheetContract?.tableNamespace);
+
+  for (const substep of flattenStaticPipeline(pipeline)) {
+    if (substep.type === 'dataset') {
+      addNamespace(substep.tableNamespace ?? substep.field);
+      addNamespace(substep.sheetContract?.tableNamespace);
+      continue;
+    }
+    const record = substep as Record<string, unknown>;
+    addNamespace(record.tableNamespace);
+    const sheetContract = record.sheetContract;
+    if (isRecord(sheetContract)) {
+      addNamespace(sheetContract.tableNamespace);
+    }
+  }
+  return [...namespaces];
+}
+
 function recordRunnerPerfTrace(input: {
   req: RunRequest;
   phase: string;
@@ -959,6 +999,59 @@ function extractChildPlayOutput(status: Record<string, unknown>): unknown {
   return result ?? null;
 }
 
+function hydrateChildPlayOutputDatasets(input: {
+  req: RunRequest;
+  childRunId: string;
+  childPlayName: string;
+  value: unknown;
+}): unknown {
+  return hydrateSerializedResultDatasets(input.value, (dataset) => {
+    const tableNamespace = dataset.tableNamespace?.trim();
+    if (!tableNamespace) {
+      return dataset;
+    }
+    const count = Math.max(0, Math.floor(dataset.count));
+    const previewRows = dataset.preview.filter(isRecord);
+    return createPersistedDatasetHandle({
+      playName: input.childPlayName,
+      name: tableNamespace,
+      count,
+      datasetKind: dataset.datasetKind,
+      previewRows,
+      cachedRows: count <= previewRows.length ? previewRows : null,
+      sourceLabel: dataset.sourceLabel ?? null,
+      readRows: async ({ limit, offset }) => {
+        const result = await harnessReadSheetDatasetRows({
+          baseUrl: input.req.baseUrl,
+          executorToken: input.req.executorToken,
+          orgId: input.req.orgId,
+          playName: input.childPlayName,
+          tableNamespace,
+          runId: input.childRunId,
+          limit,
+          offset,
+          userEmail: input.req.userEmail,
+          preloadedDbSessions: input.req.preloadedDbSessions ?? null,
+        });
+        return result.rows as Array<Record<string, unknown>>;
+      },
+      trace: (phase, ms, extra) =>
+        recordRunnerPerfTrace({
+          req: input.req,
+          phase,
+          ms,
+          extra: {
+            ...(extra ?? {}),
+            childRunId: input.childRunId,
+            childPlayName: input.childPlayName,
+          },
+        }),
+      nowMs,
+      workProgress: dataset._metadata?.workProgress,
+    });
+  });
+}
+
 async function hashChildPlayEventKey(input: unknown): Promise<string> {
   return (await hashJson(input)).slice(0, 32);
 }
@@ -983,7 +1076,7 @@ function workflowTimeoutFromMs(timeoutMs: number): string {
 async function signalParentPlayTerminal(input: {
   req: RunRequest;
   status: 'completed' | 'failed' | 'cancelled';
-  result?: Record<string, unknown> | null;
+  result?: unknown;
   error?: string | null;
 }): Promise<void> {
   const governance = input.req.playCallGovernance;
@@ -5402,7 +5495,12 @@ function createMinimalWorkerCtx(
               message: `Completed child play ${resolvedName} (${normalizedKey})`,
               ts: nowMs(),
             });
-            return started.output ?? extractChildPlayOutput(started);
+            return hydrateChildPlayOutputDatasets({
+              req,
+              childRunId: workflowId,
+              childPlayName: resolvedName,
+              value: started.output ?? extractChildPlayOutput(started),
+            });
           }
           if (startedStatus === 'failed') {
             const startedError = isRecord(started.error)
@@ -5523,7 +5621,12 @@ function createMinimalWorkerCtx(
             message: `Completed child play ${resolvedName} (${normalizedKey})`,
             ts: nowMs(),
           });
-          return waitResult.output;
+          return hydrateChildPlayOutputDatasets({
+            req,
+            childRunId: workflowId,
+            childPlayName: resolvedName,
+            value: waitResult.output,
+          });
         } finally {
           childPlaySlot?.release();
         }
@@ -6362,14 +6465,51 @@ async function executeRunRequest(
       ms: nowMs() - playStartedAt,
     });
     stepLifecycle?.markAllTerminal(nowMs());
+    const promoted = projectTerminalResultDatasets(
+      result,
+      (dataset) =>
+        createInlineDatasetHandle(dataset.rows, {
+          name: dataset.tableNamespace,
+          kind: 'csv',
+          sourceLabel: dataset.path,
+        }),
+      {
+        playName: req.playName,
+        reservedTableNamespaces: collectReservedResultDatasetNamespaces(
+          req.contractSnapshot,
+        ),
+      },
+    );
     const serializeStartedAt = nowMs();
-    const serializedResult = serializePlayReturnValue(result);
+    const serializedResult = serializePlayReturnValue(promoted.result);
     recordRunnerPerfTrace({
       req,
       phase: 'runner.serialize_result',
       ms: nowMs() - serializeStartedAt,
+      extra: {
+        promotedOutputDatasets: promoted.datasets.length,
+      },
     });
     const terminalResult = trimResultForStatus(serializedResult);
+    let resultDatasetsPersisted = false;
+    const persistProjectedResultDatasets = async (): Promise<void> => {
+      if (resultDatasetsPersisted) return;
+      const ledgerFlushWaitStartedAt = nowMs();
+      await ledgerFlushInFlight;
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.run_ledger_flush_wait',
+        ms: nowMs() - ledgerFlushWaitStartedAt,
+      });
+      const resultDatasetStartedAt = nowMs();
+      await persistResultDatasets(req, promoted.handles, serializedResult);
+      resultDatasetsPersisted = true;
+      recordRunnerPerfTrace({
+        req,
+        phase: 'runner.persist_result_datasets',
+        ms: nowMs() - resultDatasetStartedAt,
+      });
+    };
     let parentSignalPromise: Promise<void> | null = null;
     const startParentTerminalSignal = (): Promise<void> => {
       if (!parentSignalPromise) {
@@ -6377,7 +6517,7 @@ async function executeRunRequest(
         parentSignalPromise = signalParentPlayTerminal({
           req,
           status: 'completed',
-          result: terminalResult as Record<string, unknown>,
+          result: req.playCallGovernance ? serializedResult : terminalResult,
         })
           .catch((error) => {
             console.error(
@@ -6396,21 +6536,11 @@ async function executeRunRequest(
       }
       return parentSignalPromise;
     };
+    if (req.playCallGovernance && !options?.persistResultDatasets) {
+      await persistProjectedResultDatasets();
+    }
     if (options?.persistResultDatasets) {
-      const ledgerFlushWaitStartedAt = nowMs();
-      await ledgerFlushInFlight;
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.run_ledger_flush_wait',
-        ms: nowMs() - ledgerFlushWaitStartedAt,
-      });
-      const resultDatasetStartedAt = nowMs();
-      await persistResultDatasets(req, result, serializedResult);
-      recordRunnerPerfTrace({
-        req,
-        phase: 'runner.persist_result_datasets',
-        ms: nowMs() - resultDatasetStartedAt,
-      });
+      await persistProjectedResultDatasets();
       const parentSignal = startParentTerminalSignal();
       // Capped runs settle compute billing BEFORE declaring run.completed: a
       // per-run cap denial (422 billing_cap_exceeded) must fail the run as
@@ -6767,17 +6897,30 @@ function isPlayCallGovernanceSnapshot(
 
 async function persistResultDatasets(
   req: RunRequest,
-  result: unknown,
+  resultDatasets: ProjectedResultDatasetHandle[],
   serializedResult: unknown,
 ): Promise<void> {
   const persistedNamespaces = new Set<string>();
-  for (const dataset of collectDatasetHandles(result)) {
+  for (const dataset of resultDatasets) {
     if (dataset.datasetKind === 'map') continue;
+    const handle = dataset.handle as WorkerDatasetHandle<
+      Record<string, unknown>
+    >;
+    const capabilities = datasetHandleCapabilities(handle);
+    const baseContract = baseSheetContractForResultDataset(
+      req,
+      dataset.tableNamespace,
+    );
+    const scan = await collectStableResultDatasetPersistenceContract({
+      contract: baseContract,
+      chunks: iterDatasetChunks(handle, RESULT_DATASET_PERSIST_CHUNK_ROWS),
+      cacheChunks: !capabilities.replayable,
+    });
     let inputOffset = 0;
-    for await (const chunk of iterDatasetChunks(
-      dataset.handle,
-      RESULT_DATASET_PERSIST_CHUNK_ROWS,
-    )) {
+    const chunks =
+      scan.cachedChunks ??
+      iterDatasetChunks(handle, RESULT_DATASET_PERSIST_CHUNK_ROWS);
+    for await (const chunk of chunks) {
       if (chunk.length === 0) continue;
       await harnessStartSheetDataset({
         baseUrl: req.baseUrl,
@@ -6785,7 +6928,7 @@ async function persistResultDatasets(
         orgId: req.orgId,
         playName: req.playName,
         tableNamespace: dataset.tableNamespace,
-        sheetContract: requireSheetContract(req, dataset.tableNamespace),
+        sheetContract: scan.sheetContract,
         rows: chunk.map((row) => ({ ...row })),
         runId: req.runId,
         inputOffset,
@@ -6800,8 +6943,20 @@ async function persistResultDatasets(
   const datasets = collectDatasetEnvelopes(serializedResult);
   for (const dataset of datasets) {
     if (dataset.datasetKind === 'map') continue;
-    if (dataset.rows.length === 0) continue;
     if (persistedNamespaces.has(dataset.tableNamespace)) continue;
+    if (dataset.rows.length < dataset.count) {
+      throw new Error(
+        `Returned serialized dataset ${JSON.stringify(
+          dataset.tableNamespace,
+        )} only includes ${dataset.rows.length}/${dataset.count} preview rows. ` +
+          'Return the live Dataset Handle instead of dataset.toJSON() so Deepline can persist and stream the full result.',
+      );
+    }
+    if (dataset.rows.length === 0) continue;
+    const scan = await collectStableResultDatasetPersistenceContract({
+      contract: baseSheetContractForResultDataset(req, dataset.tableNamespace),
+      chunks: [dataset.rows],
+    });
     await harnessStartSheetDataset({
       baseUrl: req.baseUrl,
       executorToken: req.executorToken,
@@ -6809,7 +6964,7 @@ async function persistResultDatasets(
       preloadedDbSessions: req.preloadedDbSessions ?? null,
       playName: req.playName,
       tableNamespace: dataset.tableNamespace,
-      sheetContract: requireSheetContract(req, dataset.tableNamespace),
+      sheetContract: scan.sheetContract,
       rows: dataset.rows,
       runId: req.runId,
       inputOffset: 0,
@@ -6820,52 +6975,16 @@ async function persistResultDatasets(
 
 const RESULT_DATASET_PERSIST_CHUNK_ROWS = 5_000;
 
-function collectDatasetHandles(value: unknown): Array<{
-  tableNamespace: string;
-  datasetKind: 'csv' | 'map' | null;
-  handle: WorkerDatasetHandle<Record<string, unknown>>;
-}> {
-  const datasets: Array<{
-    tableNamespace: string;
-    datasetKind: 'csv' | 'map' | null;
-    handle: WorkerDatasetHandle<Record<string, unknown>>;
-  }> = [];
-  const seen = new WeakSet<object>();
-  const walk = (candidate: unknown, depth: number) => {
-    if (depth > 12 || candidate == null) return;
-    if (isDatasetHandle(candidate)) {
-      const metadata = candidate.toJSON() as Record<string, unknown>;
-      const tableNamespace =
-        typeof metadata.tableNamespace === 'string'
-          ? metadata.tableNamespace
-          : null;
-      const datasetKind =
-        metadata.datasetKind === 'csv' || metadata.datasetKind === 'map'
-          ? metadata.datasetKind
-          : null;
-      if (tableNamespace) {
-        datasets.push({
-          tableNamespace,
-          datasetKind,
-          handle: candidate as WorkerDatasetHandle<Record<string, unknown>>,
-        });
-      }
-      return;
-    }
-    if (Array.isArray(candidate)) {
-      for (const item of candidate) walk(item, depth + 1);
-      return;
-    }
-    if (typeof candidate !== 'object') return;
-    const object = candidate as Record<string, unknown>;
-    if (seen.has(object)) return;
-    seen.add(object);
-    for (const child of Object.values(object)) {
-      walk(child, depth + 1);
+function baseSheetContractForResultDataset(
+  req: RunRequest,
+  tableNamespace: string,
+): PlaySheetContract {
+  return (
+    resolveSheetContractFromReq(req, tableNamespace) ?? {
+      tableNamespace,
+      columns: [],
     }
-  };
-  walk(value, 0);
-  return datasets;
+  );
 }
 
 function serializePlayReturnValue(value: unknown): unknown {
@@ -6943,11 +7062,13 @@ function serializeValue(value: unknown, depth: number): unknown {
 function collectDatasetEnvelopes(value: unknown): Array<{
   tableNamespace: string;
   datasetKind: 'csv' | 'map' | null;
+  count: number;
   rows: Record<string, unknown>[];
 }> {
   const datasets: Array<{
     tableNamespace: string;
     datasetKind: 'csv' | 'map' | null;
+    count: number;
     rows: Record<string, unknown>[];
   }> = [];
   const seen = new Set<string>();
@@ -6964,13 +7085,22 @@ function collectDatasetEnvelopes(value: unknown): Array<{
     const preview = Array.isArray(record.preview)
       ? (record.preview as Record<string, unknown>[])
       : null;
+    const count =
+      typeof record.count === 'number' && Number.isFinite(record.count)
+        ? Math.max(0, Math.floor(record.count))
+        : null;
     const datasetKind =
       record.datasetKind === 'csv' || record.datasetKind === 'map'
         ? record.datasetKind
         : null;
-    if (tableNamespace && preview && !seen.has(tableNamespace)) {
+    if (
+      tableNamespace &&
+      preview &&
+      count !== null &&
+      !seen.has(tableNamespace)
+    ) {
       seen.add(tableNamespace);
-      datasets.push({ tableNamespace, datasetKind, rows: preview });
+      datasets.push({ tableNamespace, datasetKind, count, rows: preview });
     }
     for (const [key, child] of Object.entries(record)) {
       if (key === 'preview') continue;

package/dist/bundling-sources/apps/play-runner-workers/src/runtime/dataset-handles.ts

@@ -454,6 +454,14 @@ export function datasetHandleCapabilities<T extends DatasetRow>(
     if (capabilities) {
       return capabilities;
     }
+    return {
+      storage: 'iterable',
+      chunkReadable: false,
+      pageBacked: Boolean(input.backing),
+      replayable: true,
+      countHint: null,
+      materialization: 'bounded',
+    };
   }
   return {
     storage: 'iterable',