deepline 0.1.304 → 0.1.305

package/dist/bundling-sources/sdk/src/release.ts

@@ -157,7 +157,7 @@ export const SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: '0.1.304',
+  version: '0.1.305',
   contracts: {
     api: {
       name: 'sdk-http-api',

package/dist/bundling-sources/shared_libs/play-runtime/context.ts

@@ -15,10 +15,15 @@
 import { AsyncLocalStorage } from 'async_hooks';
 import {
   createDeferredPlayDataset,
+  deserializeLegacyPlayDataset,
+  deserializePlayDatasetCell,
   isPlayDataset,
+  isSerializedPlayDataset,
+  isSerializedPlayDatasetCell,
   iteratePlayDatasetInputPages,
   materializePlayDatasetInput,
   resolveMaterializeLimitCap,
+  serializePlayDatasetCell,
 } from '@shared_libs/plays/dataset';
 import type { PlayDataset, PlayDatasetInput } from '@shared_libs/plays/dataset';
 import {
@@ -733,6 +738,14 @@ function runtimeSheetPatchFieldName(fieldName: string): string {
   return fieldName.includes('.') ? sqlSafePlayColumnName(fieldName) : fieldName;
 }
 
+function isPlainRecord(value: unknown): value is Record<string, unknown> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return false;
+  }
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
 type PersistableMapRow = MapRowOutcome;
 
 function comparePersistableMapRowsByInputIndex(
@@ -3088,18 +3101,15 @@ export class PlayContextImpl {
     );
   }
 
-  // --- Tool-result cells survive the dataset persist/resume boundary ---
-  // A ToolExecuteResult carries live getter methods that cannot be serialized
-  // to durable storage. We store its serialized form on write and rehydrate it
-  // when a cell value is handed back to play code (later column resolvers,
-  // staleAfterSeconds, previousCell), so `row.<col>.extractedValues.x.get()`
-  // works the same on a fresh pass and on a cached/resumed re-run. Both guards
-  // are no-ops for any value that is not a tool result.
+  // --- Runtime values survive the dataset persist/resume boundary ---
+  // Tool results and PlayDatasets carry live methods. Cells store JSON, so
+  // encode them before persistence and revive them before authored play code
+  // receives a row again.
 
-  private serializeCellValue(value: unknown): unknown {
+  private async serializeCellValue(value: unknown): Promise<unknown> {
     const serialized = isToolExecuteResult(value)
       ? serializeToolExecuteResult(value)
-      : value;
+      : await this.serializeDatasetCells(value);
     try {
       stringifyPostgresJson(serialized);
     } catch (error) {
@@ -3112,10 +3122,86 @@ export class PlayContextImpl {
     return serialized;
   }
 
+  private async serializeDatasetCells(value: unknown): Promise<unknown> {
+    // Keep cycles intact for stringifyPostgresJson to reject. This pass only
+    // encodes runtime values; it must not turn an invalid cyclic cell into a
+    // different graph or recurse forever before the JSON boundary can report it.
+    const visiting = new WeakSet<object>();
+    const visit = async (current: unknown): Promise<unknown> => {
+      if (isToolExecuteResult(current)) {
+        return serializeToolExecuteResult(current);
+      }
+      if (isPlayDataset(current)) {
+        return await serializePlayDatasetCell(current, visit);
+      }
+      if (!current || typeof current !== 'object') return current;
+      if (visiting.has(current)) return current;
+      visiting.add(current);
+      try {
+        if (Array.isArray(current)) {
+          let copy: unknown[] | null = null;
+          for (const [index, entry] of current.entries()) {
+            const serialized = await visit(entry);
+            if (serialized !== entry) {
+              copy ??= [...current];
+              copy[index] = serialized;
+            }
+          }
+          return copy ?? current;
+        }
+        if (!isPlainRecord(current)) return current;
+        let copy: Record<string, unknown> | null = null;
+        for (const [key, entry] of Object.entries(current)) {
+          const serialized = await visit(entry);
+          if (serialized !== entry) {
+            copy ??= { ...current };
+            copy[key] = serialized;
+          }
+        }
+        return copy ?? current;
+      } finally {
+        visiting.delete(current);
+      }
+    };
+    return await visit(value);
+  }
+
   private rehydrateCellValue(value: unknown): unknown {
-    return isSerializedToolExecuteResult(value)
-      ? deserializeToolExecuteResult(value)
-      : value;
+    const seen = new WeakMap<object, unknown>();
+    const visit = (current: unknown): unknown => {
+      if (isToolExecuteResult(current)) {
+        return current;
+      }
+      if (isSerializedToolExecuteResult(current)) {
+        return deserializeToolExecuteResult(current);
+      }
+      if (isSerializedPlayDatasetCell(current)) {
+        return deserializePlayDatasetCell(current, visit);
+      }
+      if (
+        isSerializedPlayDataset(current) &&
+        current.datasetId.startsWith('tool-list:')
+      ) {
+        return deserializeLegacyPlayDataset(current);
+      }
+      if (!current || typeof current !== 'object') return current;
+      const existing = seen.get(current);
+      if (existing !== undefined) return existing;
+      if (Array.isArray(current)) {
+        const copy: unknown[] = [];
+        seen.set(current, copy);
+        for (const entry of current) copy.push(visit(entry));
+        return copy;
+      }
+      if (!isPlainRecord(current)) return current;
+      const copy: Record<string, unknown> = {};
+      seen.set(current, copy);
+      for (const [key, entry] of Object.entries(current)) {
+        copy[key] = visit(entry);
+      }
+      return copy;
+    };
+    return visit(value);
   }
 
   /**
@@ -3128,10 +3214,7 @@ export class PlayContextImpl {
     row: Record<string, unknown>,
   ): Record<string, unknown> {
     for (const key of Object.keys(row)) {
-      const value = row[key];
-      if (isSerializedToolExecuteResult(value)) {
-        row[key] = deserializeToolExecuteResult(value);
-      }
+      row[key] = this.rehydrateCellValue(row[key]);
     }
     return row;
   }
@@ -5536,7 +5619,7 @@ export class PlayContextImpl {
                   this.previousCellForField(baseRow, fieldName),
                 ),
             );
-            cellValue = this.serializeCellValue(value);
+            cellValue = await this.serializeCellValue(value);
           } catch (error) {
             if (
               isPlayRowExecutionSuspendedError(error) ||
@@ -6010,7 +6093,7 @@ export class PlayContextImpl {
       const fieldName = stepPath.join('.');
       const patchFieldName = runtimeSheetPatchFieldName(fieldName);
       if (rowStore && shouldPersistMapCellField(patchFieldName)) {
-        const cellValue = this.serializeCellValue(value);
+        const cellValue = await this.serializeCellValue(value);
         this.emitScopedFieldMetaUpdate({
           rowId: rowStore.rowId,
           key: rowStore.rowKey ?? null,
@@ -6158,7 +6241,7 @@ export class PlayContextImpl {
               index,
               this.previousCellForField(baseRow, fieldName),
             );
-            computedFields[fieldName] = this.serializeCellValue(value);
+            computedFields[fieldName] = await this.serializeCellValue(value);
             if (shouldPersistMapCellField(fieldName)) {
               rowDataPatch[fieldName] = computedFields[fieldName];
             }
@@ -6536,11 +6619,16 @@ export class PlayContextImpl {
   ): Record<string, unknown> {
     const stripped = stripCsvProjectedFields(row);
     return Object.fromEntries(
-      Object.entries(stripped).filter(
-        ([fieldName]) =>
-          shouldPersistMapCellField(fieldName) &&
-          !fieldName.startsWith('__deepline'),
-      ),
+      Object.entries(stripped)
+        .filter(
+          ([fieldName]) =>
+            shouldPersistMapCellField(fieldName) &&
+            !fieldName.startsWith('__deepline'),
+        )
+        .map(([fieldName, value]) => [
+          fieldName,
+          this.rehydrateCellValue(value),
+        ]),
     );
   }
 
@@ -6549,11 +6637,16 @@ export class PlayContextImpl {
   ): Record<string, unknown> {
     const stripped = stripCsvProjectedFields(row);
     return Object.fromEntries(
-      Object.entries(stripped).filter(
-        ([fieldName]) =>
-          shouldPersistMapCellField(fieldName) &&
-          !fieldName.startsWith('__deepline'),
-      ),
+      Object.entries(stripped)
+        .filter(
+          ([fieldName]) =>
+            shouldPersistMapCellField(fieldName) &&
+            !fieldName.startsWith('__deepline'),
+        )
+        .map(([fieldName, value]) => [
+          fieldName,
+          this.rehydrateCellValue(value),
+        ]),
     );
   }
 

package/dist/bundling-sources/shared_libs/plays/dataset.ts

@@ -51,6 +51,25 @@ export interface SerializedPlayDataset<T> {
   preview: T[];
 }
 
+/**
+ * Durable cell representation for a PlayDataset returned from a map column.
+ *
+ * `PlayDataset.toJSON()` intentionally exposes only a bounded preview for
+ * normal output rendering. A runtime-sheet cell needs the complete rows so it
+ * can revive a live handle after a persistence or resume boundary.
+ */
+export interface SerializedPlayDatasetCell<T> {
+  __kind: 'deepline.play_dataset_cell.v1';
+  dataset: SerializedPlayDataset<T>;
+  rows: T[];
+}
+
+const PLAY_DATASET_CELL_KIND = 'deepline.play_dataset_cell.v1';
+const PLAY_DATASET_CELL_MAX_BYTES = 5 * 1024 * 1024;
+const DATASET_CELL_REHYDRATION_INCOMPLETE =
+  'DATASET_CELL_REHYDRATION_INCOMPLETE';
+const DATASET_CELL_CORRUPT = 'DATASET_CELL_CORRUPT';
+
 export type PlayDatasetInput<T> =
   | ReadonlyArray<T>
   | Iterable<T>
@@ -219,6 +238,151 @@ export function isSerializedPlayDataset<T>(
   );
 }
 
+export function isSerializedPlayDatasetCell<T>(
+  value: unknown,
+): value is SerializedPlayDatasetCell<T> {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return false;
+  }
+  const record = value as Record<string, unknown>;
+  return (
+    record.__kind === PLAY_DATASET_CELL_KIND &&
+    isSerializedPlayDataset<T>(record.dataset) &&
+    Array.isArray(record.rows)
+  );
+}
+
+/** Serialize every row because a Dataset cell must survive a new worker. */
+export async function serializePlayDatasetCell<T>(
+  dataset: PlayDataset<T>,
+  serializeRow: (row: T) => Promise<T> = async (row) => row,
+): Promise<SerializedPlayDatasetCell<T>> {
+  const source = dataset.toJSON();
+  const datasetMetadata = {
+    ...source,
+    count: 0,
+    preview: [] as T[],
+  };
+  // The persisted envelope does not repeat the preview rows. That keeps the
+  // byte budget honest; deserialize derives its preview from `rows`.
+  let encodedBytes = new TextEncoder().encode(
+    JSON.stringify({
+      __kind: PLAY_DATASET_CELL_KIND,
+      dataset: datasetMetadata,
+      rows: [],
+    }),
+  ).length;
+  const rows: T[] = [];
+  for await (const row of dataset) {
+    const serializedRow = await serializeRow(row);
+    // Array serialization gives the exact JSON representation even for an
+    // undefined row, which JSON encodes as null inside `rows`.
+    const rowJson = JSON.stringify([serializedRow]).slice(1, -1);
+    encodedBytes +=
+      new TextEncoder().encode(rowJson).length + (rows.length > 0 ? 1 : 0);
+    if (encodedBytes > PLAY_DATASET_CELL_MAX_BYTES) {
+      throw new Error(
+        `OUTPUT_TOO_LARGE: Dataset cell ${dataset.datasetId} exceeds the 5 MiB customer-output limit. ` +
+          'Keep large datasets as a top-level pipeline or return a bounded dataset from the column.',
+      );
+    }
+    rows.push(serializedRow);
+  }
+  return {
+    __kind: PLAY_DATASET_CELL_KIND,
+    dataset: {
+      ...source,
+      count: rows.length,
+      preview: [],
+    },
+    rows,
+  };
+}
+
+/** Restore the public lazy-handle contract from a persisted Dataset cell. */
+export function deserializePlayDatasetCell<T>(
+  value: SerializedPlayDatasetCell<T>,
+  deserializeRow: (row: T) => T = (row) => row,
+): PlayDataset<T> {
+  const { dataset } = value;
+  const rows = value.rows.map(deserializeRow);
+  if (dataset.count !== rows.length) {
+    throw new Error(
+      `${DATASET_CELL_CORRUPT}: Dataset cell ${dataset.datasetId} declares ${dataset.count} rows but stores ${rows.length}.`,
+    );
+  }
+  return createDeferredPlayDataset({
+    datasetKind: dataset.datasetKind,
+    datasetId: dataset.datasetId,
+    count: rows.length,
+    backing: dataset.backing,
+    previewRows: rows.slice(0, 10),
+    sourceLabel: dataset.sourceLabel ?? null,
+    tableNamespace: dataset.tableNamespace ?? null,
+    workProgress: dataset._metadata?.workProgress,
+    resolvers: {
+      count: async () => rows.length,
+      peek: async (limit) => rows.slice(0, Math.max(0, limit)),
+      materialize: async (limit) =>
+        limit === undefined ? [...rows] : rows.slice(0, Math.max(0, limit)),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            yield* rows;
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+}
+
+/**
+ * Compatibility reader for cells persisted before Dataset cells carried their
+ * full rows. Small legacy lists remain usable; incomplete previews fail with a
+ * clear migration error instead of masquerading as a live handle.
+ */
+export function deserializeLegacyPlayDataset<T>(
+  dataset: SerializedPlayDataset<T>,
+): PlayDataset<T> {
+  const complete = dataset.count === dataset.preview.length;
+  const incomplete = (): Error =>
+    new Error(
+      `${DATASET_CELL_REHYDRATION_INCOMPLETE}: Dataset cell ${dataset.datasetId} only stored ${dataset.preview.length} preview row(s) for ${dataset.count} total row(s). Re-run the producing column to persist the complete Dataset Handle.`,
+    );
+  const rows = dataset.preview;
+  return createDeferredPlayDataset({
+    datasetKind: dataset.datasetKind,
+    datasetId: dataset.datasetId,
+    count: dataset.count,
+    backing: dataset.backing,
+    previewRows: rows,
+    sourceLabel: dataset.sourceLabel ?? null,
+    tableNamespace: dataset.tableNamespace ?? null,
+    workProgress: dataset._metadata?.workProgress,
+    resolvers: {
+      count: async () => dataset.count,
+      peek: async (limit) => {
+        if (limit > rows.length && !complete) throw incomplete();
+        return rows.slice(0, Math.max(0, limit));
+      },
+      materialize: async (limit) => {
+        if (!complete && (limit === undefined || limit > rows.length)) {
+          throw incomplete();
+        }
+        return limit === undefined
+          ? [...rows]
+          : rows.slice(0, Math.max(0, limit));
+      },
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            if (!complete) throw incomplete();
+            yield* rows;
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+}
+
 export function trimSerializedPlayDatasetPreview<T>(
   dataset: SerializedPlayDataset<T>,
   limit: number,

package/dist/cli/index.js

@@ -1037,7 +1037,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.304",
+  version: "0.1.305",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/cli/index.mjs

@@ -1022,7 +1022,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.304",
+  version: "0.1.305",
   contracts: {
     api: {
       name: "sdk-http-api",

package/dist/index.js

@@ -760,7 +760,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.304",
+  version: "0.1.305",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -5780,6 +5780,7 @@ var PLAY_DATASET_BRAND = /* @__PURE__ */ Symbol.for("deepline.play.dataset");
 var NODE_INSPECT_CUSTOM = /* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom");
 var DEFAULT_MATERIALIZE_LIMIT = 1e4;
 var PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
+var PLAY_DATASET_CELL_MAX_BYTES = 5 * 1024 * 1024;
 function resolveMaterializeLimitCap() {
   const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
   const parsed = raw ? Number(raw) : NaN;

package/dist/index.mjs

@@ -686,7 +686,7 @@ var SDK_RELEASE = {
   // 0.1.253 makes play-page browser opening opt-in and retires --no-open.
   // 0.1.254 removes the internal operations tree from the published SDK CLI.
   // Operators use the checkout-local deepline-admin binary instead.
-  version: "0.1.304",
+  version: "0.1.305",
   contracts: {
     api: {
       name: "sdk-http-api",
@@ -5706,6 +5706,7 @@ var PLAY_DATASET_BRAND = /* @__PURE__ */ Symbol.for("deepline.play.dataset");
 var NODE_INSPECT_CUSTOM = /* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom");
 var DEFAULT_MATERIALIZE_LIMIT = 1e4;
 var PLAY_DATASET_EXECUTION_PAGE_BYTES = 64 * 1024 * 1024;
+var PLAY_DATASET_CELL_MAX_BYTES = 5 * 1024 * 1024;
 function resolveMaterializeLimitCap() {
   const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
   const parsed = raw ? Number(raw) : NaN;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.304",
+  "version": "0.1.305",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {