deepline 0.1.24 → 0.1.26

package/dist/repo/apps/play-runner-workers/src/runtime/dataset-handles.ts

@@ -0,0 +1,418 @@
+import {
+  applyCsvRenameProjection,
+  type CsvRenameOptions,
+} from '../../../../shared_libs/play-runtime/csv-rename';
+import {
+  createDeferredPlayDataset,
+  isPlayDataset,
+  type PlayDataset,
+  type PlayDatasetInput,
+  type PlayDatasetKind,
+  type PlayDatasetWorkProgressSummary,
+} from '../../../../shared_libs/plays/dataset';
+
+export const WORKER_DATASET_PREVIEW_ROWS = 5;
+export const WORKER_DATASET_IN_MEMORY_ROWS = 100;
+const STREAM_MATERIALIZE_CHUNK_ROWS = 5_000;
+const STREAM_ITERATE_CHUNK_ROWS = 1_000;
+
+export type DatasetRow = Record<string, unknown>;
+export type WorkerDatasetHandle<T extends DatasetRow> = PlayDataset<T>;
+export type WorkerDatasetInput<T extends DatasetRow> = PlayDatasetInput<T>;
+
+type DatasetChunkReader<T extends DatasetRow> = (
+  chunkSize: number,
+) => AsyncIterable<T[]>;
+
+const datasetChunkReaders = new WeakMap<object, DatasetChunkReader<DatasetRow>>();
+const datasetCountHints = new WeakMap<object, number | null>();
+
+function cloneRow<T extends DatasetRow>(row: T): T {
+  return { ...row };
+}
+
+function registerChunkReader<T extends DatasetRow>(
+  dataset: PlayDataset<T>,
+  reader: DatasetChunkReader<T>,
+  countHint: number | null,
+): PlayDataset<T> {
+  datasetChunkReaders.set(
+    dataset as object,
+    reader as DatasetChunkReader<DatasetRow>,
+  );
+  datasetCountHints.set(dataset as object, countHint);
+  return dataset;
+}
+
+function normalizedChunkSize(chunkSize: number): number {
+  return Math.max(1, Math.floor(chunkSize));
+}
+
+export function createMaterializedDatasetHandle<T extends DatasetRow>(input: {
+  name: string;
+  rows: readonly T[];
+  datasetKind?: PlayDatasetKind;
+  count?: number;
+  sourceLabel?: string | null;
+  workProgress?: PlayDatasetWorkProgressSummary;
+}): WorkerDatasetHandle<T> {
+  const datasetKind = input.datasetKind ?? 'map';
+  const materializedRows = input.rows.map(cloneRow);
+  const dataset = createDeferredPlayDataset({
+    datasetKind,
+    datasetId: `${datasetKind}:${input.name}`,
+    count: Math.max(0, Math.floor(input.count ?? materializedRows.length)),
+    previewRows: materializedRows
+      .slice(0, WORKER_DATASET_PREVIEW_ROWS)
+      .map(cloneRow),
+    sourceLabel: input.sourceLabel ?? null,
+    tableNamespace: input.name,
+    workProgress: input.workProgress,
+    resolvers: {
+      count: async () =>
+        Math.max(0, Math.floor(input.count ?? materializedRows.length)),
+      peek: async (limit) => materializedRows.slice(0, Math.max(0, limit)),
+      materialize: async (limit) =>
+        limit === undefined
+          ? materializedRows.map(cloneRow)
+          : materializedRows.slice(0, Math.max(0, limit)).map(cloneRow),
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            for (const row of materializedRows) {
+              yield cloneRow(row);
+            }
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+  return registerChunkReader(
+    dataset,
+    async function* (chunkSize: number) {
+      const size = normalizedChunkSize(chunkSize);
+      for (let offset = 0; offset < materializedRows.length; offset += size) {
+        yield materializedRows.slice(offset, offset + size).map(cloneRow);
+      }
+    },
+    Math.max(0, Math.floor(input.count ?? materializedRows.length)),
+  );
+}
+
+export function createPersistedDatasetHandle<T extends DatasetRow>(input: {
+  playName: string;
+  name: string;
+  count: number;
+  datasetKind?: PlayDatasetKind;
+  previewRows?: readonly T[];
+  cachedRows?: readonly T[] | null;
+  sourceLabel?: string | null;
+  workProgress?: PlayDatasetWorkProgressSummary;
+  readRows: (input: { limit: number; offset: number }) => Promise<readonly T[]>;
+  trace?: (
+    phase: string,
+    ms: number,
+    extra?: Record<string, unknown>,
+  ) => void;
+  nowMs: () => number;
+}): WorkerDatasetHandle<T> {
+  const datasetKind = input.datasetKind ?? 'map';
+  const count = Math.max(0, Math.floor(input.count));
+  const previewRows = (input.previewRows ?? [])
+    .slice(0, WORKER_DATASET_PREVIEW_ROWS)
+    .map(cloneRow);
+  const cachedRows =
+    input.cachedRows && input.cachedRows.length <= WORKER_DATASET_IN_MEMORY_ROWS
+      ? input.cachedRows.map(cloneRow)
+      : null;
+
+  async function loadRows(limit: number, offset: number): Promise<T[]> {
+    if (limit <= 0) {
+      return [];
+    }
+    const normalizedLimit = normalizedChunkSize(limit);
+    const normalizedOffset = Math.max(0, Math.floor(offset));
+    if (
+      cachedRows &&
+      normalizedOffset < cachedRows.length &&
+      normalizedOffset + normalizedLimit <= cachedRows.length
+    ) {
+      return cachedRows
+        .slice(normalizedOffset, normalizedOffset + normalizedLimit)
+        .map(cloneRow);
+    }
+    const startedAt = input.nowMs();
+    const rows = await input.readRows({
+      limit: normalizedLimit,
+      offset: normalizedOffset,
+    });
+    input.trace?.('dataset.read', input.nowMs() - startedAt, {
+      datasetKind,
+      playName: input.playName,
+      tableNamespace: input.name,
+      limit: normalizedLimit,
+      offset: normalizedOffset,
+      rows: rows.length,
+    });
+    return rows.map(cloneRow);
+  }
+
+  async function* readChunks(chunkSize: number): AsyncGenerator<T[], void, void> {
+    const size = normalizedChunkSize(chunkSize);
+    let offset = 0;
+    while (offset < count) {
+      const rows = await loadRows(Math.min(size, count - offset), offset);
+      if (rows.length === 0) {
+        return;
+      }
+      yield rows;
+      offset += rows.length;
+    }
+  }
+
+  const dataset = createDeferredPlayDataset({
+    datasetKind,
+    datasetId: `${datasetKind}:${input.name}`,
+    count,
+    backing: {
+      storage: 'neon_sheet',
+      sheet: {
+        playName: input.playName,
+        tableNamespace: input.name,
+      },
+    },
+    previewRows,
+    sourceLabel: input.sourceLabel ?? null,
+    tableNamespace: input.name,
+    workProgress: input.workProgress,
+    resolvers: {
+      count: async () => count,
+      peek: async (limit) => await loadRows(Math.max(0, limit), 0),
+      materialize: async (limit) => {
+        const rows: T[] = [];
+        const maxRows = limit ?? count;
+        for await (const chunk of readChunks(STREAM_MATERIALIZE_CHUNK_ROWS)) {
+          for (const row of chunk) {
+            if (rows.length >= maxRows) return rows;
+            rows.push(row);
+          }
+        }
+        return rows;
+      },
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            for await (const chunk of readChunks(STREAM_ITERATE_CHUNK_ROWS)) {
+              for (const row of chunk) yield row;
+            }
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+  return registerChunkReader(dataset, (chunkSize) => readChunks(chunkSize), count);
+}
+
+export function createCsvDatasetHandle<T extends DatasetRow>(input: {
+  name: string;
+  logicalPath: string;
+  expectedBytes?: number | null;
+  renameOptions?: CsvRenameOptions;
+  open: () => Promise<AsyncIterable<Uint8Array> | null>;
+  streamRows: (
+    byteChunks: AsyncIterable<Uint8Array>,
+    chunkSize: number,
+  ) => AsyncIterable<T[]>;
+  trace?: (
+    phase: string,
+    ms: number,
+    extra?: Record<string, unknown>,
+  ) => void;
+  nowMs: () => number;
+}): WorkerDatasetHandle<T> {
+  const datasetId = `csv:${input.name}`;
+  let cachedCount: number | null = null;
+
+  async function* readChunks(chunkSize: number): AsyncGenerator<T[], void, void> {
+    const startedAt = input.nowMs();
+    let yieldedRows = 0;
+    let yieldedChunks = 0;
+    let observedBytes = 0;
+    let observedByteChunks = 0;
+    const body = await input.open();
+    if (!body) {
+      throw new Error(
+        `ctx.csv("${input.logicalPath}"): dataset source is not reachable from the play runner.`,
+      );
+    }
+    const size = normalizedChunkSize(chunkSize);
+    const countedBody = {
+      async *[Symbol.asyncIterator]() {
+        for await (const chunk of body) {
+          observedBytes += chunk.byteLength;
+          observedByteChunks += 1;
+          yield chunk;
+        }
+      },
+    } satisfies AsyncIterable<Uint8Array>;
+    try {
+      for await (const chunk of input.streamRows(countedBody, size)) {
+        yieldedRows += chunk.length;
+        yieldedChunks += 1;
+        yield applyCsvRenameProjection(chunk, input.renameOptions) as T[];
+      }
+    } finally {
+      input.trace?.('csv.stream', input.nowMs() - startedAt, {
+        name: input.name,
+        logicalPath: input.logicalPath,
+        expectedBytes: input.expectedBytes ?? null,
+        observedBytes,
+        observedByteChunks,
+        yieldedRows,
+        yieldedChunks,
+        chunkSize: size,
+      });
+    }
+    if (
+      typeof input.expectedBytes === 'number' &&
+      input.expectedBytes > 0 &&
+      yieldedRows === 0
+    ) {
+      throw new Error(
+        observedBytes === 0
+          ? `ctx.csv("${input.logicalPath}"): dataset source yielded 0 bytes for ` +
+              `a non-empty ${input.expectedBytes} byte staged file.`
+          : `ctx.csv("${input.logicalPath}"): parsed 0 rows after reading ` +
+              `${observedBytes} bytes from a non-empty ${input.expectedBytes} byte dataset source.`,
+      );
+    }
+  }
+
+  const dataset = createDeferredPlayDataset({
+    datasetKind: 'csv',
+    datasetId,
+    count: 0,
+    previewRows: [],
+    tableNamespace: input.name,
+    resolvers: {
+      count: async () => {
+        if (cachedCount !== null) return cachedCount;
+        let total = 0;
+        for await (const chunk of readChunks(STREAM_MATERIALIZE_CHUNK_ROWS)) {
+          total += chunk.length;
+        }
+        cachedCount = total;
+        return total;
+      },
+      peek: async (limit) => {
+        const rows: T[] = [];
+        for await (const chunk of readChunks(Math.max(1, limit))) {
+          for (const row of chunk) {
+            rows.push(row);
+            if (rows.length >= limit) return rows;
+          }
+        }
+        return rows;
+      },
+      materialize: async (limit) => {
+        const rows: T[] = [];
+        const maxRows = limit ?? Number.POSITIVE_INFINITY;
+        for await (const chunk of readChunks(STREAM_MATERIALIZE_CHUNK_ROWS)) {
+          for (const row of chunk) {
+            if (rows.length >= maxRows) return rows;
+            rows.push(row);
+          }
+        }
+        return rows;
+      },
+      iterate: () =>
+        ({
+          async *[Symbol.asyncIterator]() {
+            for await (const chunk of readChunks(STREAM_ITERATE_CHUNK_ROWS)) {
+              for (const row of chunk) yield row;
+            }
+          },
+        }) as AsyncIterable<T>,
+    },
+  });
+  return registerChunkReader(dataset, (chunkSize) => readChunks(chunkSize), null);
+}
+
+export function createInlineDatasetHandle<T extends DatasetRow>(
+  rows: readonly T[],
+  metadata?: {
+    name?: string;
+    kind?: PlayDatasetKind;
+    sourceLabel?: string | null;
+  },
+): WorkerDatasetHandle<T> {
+  return createMaterializedDatasetHandle({
+    name: metadata?.name ?? 'csv',
+    rows,
+    datasetKind: metadata?.kind ?? 'csv',
+    sourceLabel: metadata?.sourceLabel ?? null,
+  });
+}
+
+export function datasetRowCountHint<T extends DatasetRow>(
+  input: WorkerDatasetInput<T>,
+): number | null {
+  if (Array.isArray(input)) return input.length;
+  if (isPlayDataset<T>(input)) {
+    const hinted = datasetCountHints.get(input as object);
+    if (hinted !== undefined) return hinted;
+  }
+  return null;
+}
+
+export function isDatasetHandle<T extends DatasetRow>(
+  value: unknown,
+): value is WorkerDatasetHandle<T> {
+  return isPlayDataset<T>(value);
+}
+
+export function iterDatasetChunks<T extends DatasetRow>(
+  input: WorkerDatasetInput<T>,
+  chunkSize: number,
+): AsyncIterable<T[]> {
+  const size = normalizedChunkSize(chunkSize);
+  if (isPlayDataset<T>(input)) {
+    const chunkReader = datasetChunkReaders.get(input as object);
+    if (chunkReader) {
+      return chunkReader(size) as AsyncIterable<T[]>;
+    }
+  }
+  return {
+    async *[Symbol.asyncIterator]() {
+      let chunk: T[] = [];
+      const flush = function* () {
+        if (chunk.length > 0) {
+          yield chunk;
+          chunk = [];
+        }
+      };
+      const push = function* (row: T) {
+        chunk.push(row);
+        if (chunk.length >= size) {
+          yield chunk;
+          chunk = [];
+        }
+      };
+
+      if (isPlayDataset<T>(input)) {
+        for await (const row of input) yield* push(row);
+        yield* flush();
+        return;
+      }
+      if (
+        input != null &&
+        typeof input === 'object' &&
+        Symbol.asyncIterator in input
+      ) {
+        for await (const row of input as AsyncIterable<T>) yield* push(row);
+        yield* flush();
+        return;
+      }
+      for (const row of input as Iterable<T>) yield* push(row);
+      yield* flush();
+    },
+  };
+}

package/dist/repo/sdk/src/client.ts

@@ -225,7 +225,11 @@ function updatePlayLiveStatusState(
       : state.runId;
   const status = normalizeLiveStatus(payload.status) ?? state.status;
   const logs = readStringArray(payload.logs);
-  if (logs.length > 0 || event.type === 'play.run.snapshot') {
+  if (
+    logs.length > 0 ||
+    event.type === 'play.run.snapshot' ||
+    event.type === 'play.run.final_status'
+  ) {
     state.logs = logs;
   }
   if ('result' in payload) {

package/dist/repo/sdk/src/plays/bundle-play-file.ts

@@ -43,7 +43,7 @@ export {
   extractDefinedPlayName,
 } from '../../../shared_libs/plays/bundling/index.js';
 
-const PLAY_BUNDLE_CACHE_VERSION = 26;
+const PLAY_BUNDLE_CACHE_VERSION = 30;
 const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
 const SDK_PACKAGE_ROOT = resolve(MODULE_DIR, '..', '..');
 const SOURCE_REPO_ROOT = resolve(SDK_PACKAGE_ROOT, '..');

package/dist/repo/sdk/src/plays/harness-stub.ts

@@ -37,6 +37,10 @@ import type {
   RuntimeApiCallInput,
   RuntimeApiCallResult,
   RuntimePayloadSchemaId,
+  SheetDatasetRowsInput,
+  SheetDatasetRowsResult,
+  StagedFileChunkInput,
+  StagedFileChunkResult,
   ValidatePayloadResult,
 } from '../../../apps/play-harness-worker/src/rpc-types';
 
@@ -46,9 +50,6 @@ import type {
  * promise-returning RPC stubs; the type matches `PlayHarnessRpc`.
  */
 export type HarnessBinding = PlayHarnessRpc;
-type HarnessFetchBinding = HarnessBinding & {
-  fetch(input: RequestInfo, init?: RequestInit): Promise<Response>;
-};
 
 /**
  * Module-level holder for the binding. Set by the per-play harness on
@@ -122,6 +123,24 @@ export async function harnessRuntimeApiCall(
   return requireBinding().runtimeApiCall(input);
 }
 
+/**
+ * Read a bounded staged-file byte range through typed harness RPC. This is the
+ * only staged-file data path for per-play Workers; there is intentionally no
+ * HTTP fetch fallback because stale parallel paths caused empty CSV streams in
+ * preview.
+ */
+export async function harnessReadStagedFileChunk(
+  input: StagedFileChunkInput,
+): Promise<StagedFileChunkResult> {
+  return requireBinding().readStagedFileChunk(input);
+}
+
+export async function harnessReadSheetDatasetRows(
+  input: SheetDatasetRowsInput,
+): Promise<SheetDatasetRowsResult> {
+  return requireBinding().readSheetDatasetRows(input);
+}
+
 /**
  * Warm Postgres sessions in the long-lived harness Worker. This preserves the
  * map fast path without bundling the Neon client into every dynamic play.
@@ -145,6 +164,7 @@ export async function harnessStartSheetDataset(input: {
   sheetContract: unknown;
   rows: Array<Record<string, unknown>>;
   runId: string;
+  inputOffset?: number;
   userEmail?: string | null;
   preloadedDbSessions?: PreloadedRuntimeDbSessionInput[] | null;
 }): Promise<{
@@ -176,38 +196,6 @@ export async function harnessPersistCompletedSheetRows(input: {
   return requireBinding().persistCompletedMapRows(input);
 }
 
-/**
- * Stream a staged R2 object through the harness Worker. Unlike the legacy
- * signed-URL helper this never touches the Vercel runtime route, and it keeps
- * large CSV bodies as streams instead of serializing them through RPC.
- */
-export async function harnessFetchStagedFile(input: {
-  executorToken: string;
-  storageKey: string;
-  method?: 'GET' | 'HEAD';
-  range?: { offset: number; length: number };
-}): Promise<Response> {
-  const binding = requireBinding() as HarnessFetchBinding;
-  if (typeof binding.fetch !== 'function') {
-    throw new Error(
-      '[harness-stub] env.HARNESS does not expose fetch(); cannot stream staged R2 files through the harness.',
-    );
-  }
-  const url = new URL('https://play-harness.internal/r2/staged-file');
-  url.searchParams.set('storageKey', input.storageKey);
-  const headers: Record<string, string> = {
-    authorization: `Bearer ${input.executorToken}`,
-  };
-  if (input.range) {
-    const end = input.range.offset + input.range.length - 1;
-    headers.range = `bytes=${input.range.offset}-${end}`;
-  }
-  return binding.fetch(url.toString(), {
-    method: input.method ?? 'GET',
-    headers,
-  });
-}
-
 /**
  * Validate a payload against a named schema. The schema definitions
  * (and zod itself) live in the harness Worker — see

package/dist/repo/sdk/src/version.ts

@@ -1,2 +1,2 @@
-export const SDK_VERSION = "0.1.24";
+export const SDK_VERSION = "0.1.26";
 export const SDK_API_CONTRACT = "2026-05-runs-v2";

package/dist/repo/shared_libs/play-runtime/execution-plan.ts

@@ -126,13 +126,29 @@ export function buildExecutionPlan(
 }
 
 export function chooseMapChunkSize(input: {
-  totalRows: number;
+  /**
+   * Known input row count. Pass null for streaming datasets where counting
+   * would require an extra full scan; unknown counts use the preferred chunk.
+   */
+  totalRows: number | null;
   mapCount: number;
   stepsPerChunk: number;
   preferredChunkSize?: number | null;
   softWorkflowStepBudget?: number | null;
 }): number {
-  const totalRows = Math.max(0, Math.floor(input.totalRows));
+  const totalRows =
+    typeof input.totalRows === 'number' && Number.isFinite(input.totalRows)
+      ? Math.max(0, Math.floor(input.totalRows))
+      : null;
+  const preferred = Math.max(
+    1,
+    Math.floor(
+      input.preferredChunkSize ?? EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
+    ),
+  );
+  if (totalRows === null) {
+    return preferred;
+  }
   if (totalRows <= EXECUTION_PLAN_DEFAULTS.inlineRowsLimit) {
     return Math.max(1, totalRows || 1);
   }
@@ -157,12 +173,6 @@ export function chooseMapChunkSize(input: {
     1,
     Math.ceil(totalRows / maxChunksPerMap),
   );
-  const preferred = Math.max(
-    1,
-    Math.floor(
-      input.preferredChunkSize ?? EXECUTION_PLAN_DEFAULTS.largeMapChunkSize,
-    ),
-  );
   return Math.max(preferred, minimumSurvivalChunkSize);
 }
 

package/dist/repo/shared_libs/play-runtime/scheduler-backend.ts

@@ -52,13 +52,14 @@ export type PlaySchedulerSubmitInput = {
     storageKey?: string;
     fileName?: string;
     logicalPath?: string;
+    contentType?: string;
+    bytes?: number;
   } | null;
   inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
   /**
-   * Tiny input files (<= WORKERS_EDGE_INLINE_INPUT_MAX_BYTES) carried as raw
-   * base64 bytes through the scheduler submit so the cf-workflows backend can
-   * build inlineCsv without an R2 PUT+GET round trip. Mirrors the
-   * Temporal-side `inlineInputFile` fast path.
+   * Temporal-only escape hatch for small legacy inputs. Workers-edge CSV
+   * inputs must be staged and streamed by handle instead of carried as raw
+   * scheduler payload bytes.
    */
   inlineInputFile?: {
     logicalPath: string;