deepline 0.1.54 → 0.1.56

package/dist/repo/shared_libs/plays/bundling/index.ts

@@ -2,7 +2,14 @@ import { createHash } from 'node:crypto';
 import { existsSync, readFileSync } from 'node:fs';
 import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path';
+import {
+  basename,
+  dirname,
+  extname,
+  isAbsolute,
+  join,
+  resolve,
+} from 'node:path';
 import { builtinModules } from 'node:module';
 import { build, type Message, type Plugin } from 'esbuild';
 import {
@@ -37,11 +44,23 @@ const PLAY_ARTIFACT_CACHE_DIR = join(
   `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,
 );
 const PLAY_PROXY_NAMESPACE = 'deepline-play-runtime-ref';
-const SOURCE_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.json'];
+const SOURCE_EXTENSIONS = [
+  '.ts',
+  '.tsx',
+  '.mts',
+  '.cts',
+  '.js',
+  '.jsx',
+  '.mjs',
+  '.cjs',
+  '.json',
+];
 const WORKERS_PLAY_ENTRY_VIRTUAL = 'deepline-play-entry';
 const PLAY_SOURCE_FILE_PATTERN = /\.play\.(?:[cm]?[jt]sx?)$/i;
 const NODE_BUILTIN_SET = new Set(
-  builtinModules.flatMap((name) => (name.startsWith('node:') ? [name, name.slice(5)] : [name, `node:${name}`])),
+  builtinModules.flatMap((name) =>
+    name.startsWith('node:') ? [name, name.slice(5)] : [name, `node:${name}`],
+  ),
 );
 
 export type {
@@ -87,7 +106,9 @@ export type PlayBundlingAdapter = {
   sdkWorkersEntryFile: string;
   workersHarnessEntryFile: string;
   workersHarnessFilesDir: string;
-  discoverPackagedLocalFiles(filePath: string): Promise<PlayLocalFileDiscoveryResult>;
+  discoverPackagedLocalFiles(
+    filePath: string,
+  ): Promise<PlayLocalFileDiscoveryResult>;
   typecheckPlaySource?(input: {
     sourceCode: string;
     sourcePath: string;
@@ -229,18 +250,25 @@ function isPlaySourceFile(filePath: string): boolean {
 function stripCommentsToSpaces(source: string): string {
   return source
     .replace(/\/\*[\s\S]*?\*\//g, (match) => match.replace(/[^\n]/g, ' '))
-    .replace(/(^|[^:])\/\/.*$/gm, (match, prefix: string) =>
-      prefix + ' '.repeat(Math.max(0, match.length - prefix.length)),
+    .replace(
+      /(^|[^:])\/\/.*$/gm,
+      (match, prefix: string) =>
+        prefix + ' '.repeat(Math.max(0, match.length - prefix.length)),
     );
 }
 
-function lineAndColumnAt(source: string, index: number): { line: number; column: number } {
+function lineAndColumnAt(
+  source: string,
+  index: number,
+): { line: number; column: number } {
   const prefix = source.slice(0, index);
   const lines = prefix.split('\n');
   return { line: lines.length, column: lines[lines.length - 1]!.length + 1 };
 }
 
-function findSourceImportReferences(sourceCode: string): SourceImportReference[] {
+function findSourceImportReferences(
+  sourceCode: string,
+): SourceImportReference[] {
   const source = stripCommentsToSpaces(sourceCode);
   const references: SourceImportReference[] = [];
   const addReference = (
@@ -261,17 +289,29 @@ function findSourceImportReferences(sourceCode: string): SourceImportReference[]
   const staticImportPattern =
     /\b(?:import|export)\s+(?!type\b)(?:[\s\S]*?\s+from\s*)?(['"])([^'"\n]+)\1/g;
   for (const match of source.matchAll(staticImportPattern)) {
-    addReference(match[2], match.index! + match[0].lastIndexOf(match[1]!), 'static');
+    addReference(
+      match[2],
+      match.index! + match[0].lastIndexOf(match[1]!),
+      'static',
+    );
   }
 
   const dynamicImportPattern = /\bimport\s*\(\s*(['"])([^'"\n]+)\1/g;
   for (const match of source.matchAll(dynamicImportPattern)) {
-    addReference(match[2], match.index! + match[0].lastIndexOf(match[1]!), 'dynamic-import');
+    addReference(
+      match[2],
+      match.index! + match[0].lastIndexOf(match[1]!),
+      'dynamic-import',
+    );
   }
 
   const requirePattern = /\brequire\s*\(\s*(['"])([^'"\n]+)\1/g;
   for (const match of source.matchAll(requirePattern)) {
-    addReference(match[2], match.index! + match[0].lastIndexOf(match[1]!), 'require');
+    addReference(
+      match[2],
+      match.index! + match[0].lastIndexOf(match[1]!),
+      'require',
+    );
   }
 
   const literalDynamicImportIndexes = new Set(
@@ -297,18 +337,27 @@ function findSourceImportReferences(sourceCode: string): SourceImportReference[]
   }
 
   return references.sort((left, right) =>
-    left.line === right.line ? left.column - right.column : left.line - right.line,
+    left.line === right.line
+      ? left.column - right.column
+      : left.line - right.line,
   );
 }
 
 function unquoteStringLiteral(literal: string): string | null {
   const trimmed = literal.trim();
   const quote = trimmed[0];
-  if ((quote !== '"' && quote !== "'") || trimmed[trimmed.length - 1] !== quote) {
+  if (
+    (quote !== '"' && quote !== "'") ||
+    trimmed[trimmed.length - 1] !== quote
+  ) {
     return null;
   }
   try {
-    return JSON.parse(quote === '"' ? trimmed : `"${trimmed.slice(1, -1).replace(/"/g, '\\"')}"`);
+    return JSON.parse(
+      quote === '"'
+        ? trimmed
+        : `"${trimmed.slice(1, -1).replace(/"/g, '\\"')}"`,
+    );
   } catch {
     return trimmed.slice(1, -1);
   }
@@ -345,7 +394,8 @@ function findMatchingBrace(source: string, openIndex: number): number {
 
 export function extractDefinedPlayName(sourceCode: string): string | null {
   const source = stripCommentsToSpaces(sourceCode);
-  const callPattern = /(?:\b[A-Za-z_$][\w$]*\s*\.\s*)?\b(?:definePlay|defineWorkflow)\s*\(/g;
+  const callPattern =
+    /(?:\b[A-Za-z_$][\w$]*\s*\.\s*)?\b(?:definePlay|defineWorkflow)\s*\(/g;
   for (const match of source.matchAll(callPattern)) {
     const openParen = match.index! + match[0].length - 1;
     const firstArgStart = openParen + 1;
@@ -354,7 +404,9 @@ export function extractDefinedPlayName(sourceCode: string): string | null {
     const argIndex = firstArgStart + firstNonSpace;
     const quote = source[argIndex];
     if (quote === '"' || quote === "'") {
-      const literalMatch = source.slice(argIndex).match(/^(['"])(?:\\.|(?!\1)[\s\S])*\1/);
+      const literalMatch = source
+        .slice(argIndex)
+        .match(/^(['"])(?:\\.|(?!\1)[\s\S])*\1/);
       const value = literalMatch ? unquoteStringLiteral(literalMatch[0]) : null;
       if (value?.trim()) return value.trim();
     }
@@ -362,7 +414,9 @@ export function extractDefinedPlayName(sourceCode: string): string | null {
       const closeBrace = findMatchingBrace(source, argIndex);
       if (closeBrace < 0) continue;
       const objectSource = source.slice(argIndex + 1, closeBrace);
-      const idMatch = objectSource.match(/(?:^|[,{\s])(?:id|['"]id['"])\s*:\s*(['"])([\s\S]*?)\1/);
+      const idMatch = objectSource.match(
+        /(?:^|[,{\s])(?:id|['"]id['"])\s*:\s*(['"])([\s\S]*?)\1/,
+      );
       if (idMatch?.[2]?.trim()) {
         return idMatch[2].trim();
       }
@@ -526,7 +580,12 @@ function workersNodeBuiltinStubPlugin(): Plugin {
   // These are the node builtins NOT exposed by Cloudflare's nodejs_compat.
   // Keep this list narrow — anything supported (path, crypto, etc.) should
   // pass through as `external: ['node:*']` in the build config.
-  const UNSUPPORTED = new Set(['node:fs', 'node:fs/promises', 'node:os', 'node:child_process']);
+  const UNSUPPORTED = new Set([
+    'node:fs',
+    'node:fs/promises',
+    'node:os',
+    'node:child_process',
+  ]);
   return {
     name: 'deepline-workers-node-builtin-stub',
     setup(buildContext) {
@@ -613,7 +672,8 @@ function zodNonEnglishLocaleStubPlugin(): Plugin {
   // Match any zod v4 locale module path EXCEPT en + en variants.
   // The filter uses a coarse regex; the namespace handler then
   // double-checks with a precise basename test before stubbing.
-  const LOCALE_PATH_FILTER = /[\\/]zod[\\/]v4[\\/]locales[\\/][^\\/]+\.(?:c?js|mjs)$/;
+  const LOCALE_PATH_FILTER =
+    /[\\/]zod[\\/]v4[\\/]locales[\\/][^\\/]+\.(?:c?js|mjs)$/;
   const NAMESPACE = 'deepline-zod-locale-stub';
   return {
     name: 'deepline-zod-non-english-locale-stub',
@@ -631,16 +691,13 @@ function zodNonEnglishLocaleStubPlugin(): Plugin {
         // Anything else: stub with an empty default export.
         return { path: args.path, namespace: NAMESPACE };
       });
-      buildContext.onLoad(
-        { filter: /.*/, namespace: NAMESPACE },
-        () => ({
-          // zod locales export a default object literal. Empty object is
-          // structurally compatible — accessing any locale key returns
-          // undefined and zod falls back to its built-in English.
-          contents: 'export default {};',
-          loader: 'js',
-        }),
-      );
+      buildContext.onLoad({ filter: /.*/, namespace: NAMESPACE }, () => ({
+        // zod locales export a default object literal. Empty object is
+        // structurally compatible — accessing any locale key returns
+        // undefined and zod falls back to its built-in English.
+        contents: 'export default {};',
+        loader: 'js',
+      }));
     },
   };
 }
@@ -735,7 +792,10 @@ function importedPlayProxyPlugin(
   }
 
   const dependenciesByPath = new Map(
-    importedPlayDependencies.map((dependency) => [dependency.filePath, dependency]),
+    importedPlayDependencies.map((dependency) => [
+      dependency.filePath,
+      dependency,
+    ]),
   );
 
   return {
@@ -759,19 +819,23 @@ function importedPlayProxyPlugin(
         };
       });
 
-      buildContext.onLoad({ filter: /.*/, namespace: PLAY_PROXY_NAMESPACE }, async (args) => {
-        const dependency = (args.pluginData as ImportedPlayDependency | undefined)
-          ?? dependenciesByPath.get(args.path);
-        if (!dependency) {
-          return null;
-        }
+      buildContext.onLoad(
+        { filter: /.*/, namespace: PLAY_PROXY_NAMESPACE },
+        async (args) => {
+          const dependency =
+            (args.pluginData as ImportedPlayDependency | undefined) ??
+            dependenciesByPath.get(args.path);
+          if (!dependency) {
+            return null;
+          }
 
-        return {
-          contents: buildImportedPlayProxyModule(dependency.playName),
-          loader: 'ts',
-          resolveDir: dirname(args.path),
-        };
-      });
+          return {
+            contents: buildImportedPlayProxyModule(dependency.playName),
+            loader: 'ts',
+            resolveDir: dirname(args.path),
+          };
+        },
+      );
     },
   };
 }
@@ -785,21 +849,32 @@ async function fileExists(filePath: string): Promise<boolean> {
   }
 }
 
-async function resolveLocalImport(fromFile: string, specifier: string): Promise<string> {
+async function resolveLocalImport(
+  fromFile: string,
+  specifier: string,
+): Promise<string> {
   if (specifier.startsWith('file:')) {
     return normalizeLocalPath(new URL(specifier).pathname);
   }
 
-  const base = isAbsolute(specifier) ? resolve(specifier) : resolve(dirname(fromFile), specifier);
+  const base = isAbsolute(specifier)
+    ? resolve(specifier)
+    : resolve(dirname(fromFile), specifier);
   const candidates: string[] = [base];
   const explicitExtension = extname(base).toLowerCase();
 
   if (!explicitExtension) {
-    candidates.push(...SOURCE_EXTENSIONS.map((extension) => `${base}${extension}`));
-    candidates.push(...SOURCE_EXTENSIONS.map((extension) => join(base, `index${extension}`)));
+    candidates.push(
+      ...SOURCE_EXTENSIONS.map((extension) => `${base}${extension}`),
+    );
+    candidates.push(
+      ...SOURCE_EXTENSIONS.map((extension) => join(base, `index${extension}`)),
+    );
   } else if (['.js', '.jsx', '.mjs', '.cjs'].includes(explicitExtension)) {
     const stem = base.slice(0, -explicitExtension.length);
-    candidates.push(...SOURCE_EXTENSIONS.map((extension) => `${stem}${extension}`));
+    candidates.push(
+      ...SOURCE_EXTENSIONS.map((extension) => `${stem}${extension}`),
+    );
   }
 
   for (const candidate of candidates) {
@@ -808,7 +883,9 @@ async function resolveLocalImport(fromFile: string, specifier: string): Promise<
     }
   }
 
-  throw new Error(`Could not resolve local import "${specifier}" from ${fromFile}`);
+  throw new Error(
+    `Could not resolve local import "${specifier}" from ${fromFile}`,
+  );
 }
 
 function resolvePackageImport(
@@ -876,7 +953,9 @@ async function analyzeSourceGraph(
       }
 
       if (NODE_BUILTIN_SET.has(specifier)) {
-        nodeBuiltins.add(specifier.startsWith('node:') ? specifier : `node:${specifier}`);
+        nodeBuiltins.add(
+          specifier.startsWith('node:') ? specifier : `node:${specifier}`,
+        );
         return;
       }
 
@@ -914,7 +993,11 @@ async function analyzeSourceGraph(
         );
       }
 
-      const packageImport = resolvePackageImport(specifier, absolutePath, adapter);
+      const packageImport = resolvePackageImport(
+        specifier,
+        absolutePath,
+        adapter,
+      );
       packages.set(packageImport.name, packageImport.version);
     };
 
@@ -976,8 +1059,9 @@ async function analyzeSourceGraph(
         .sort((left, right) => left.name.localeCompare(right.name)),
     },
     playName,
-    importedPlayDependencies: [...importedPlayDependencies.values()]
-      .sort((left, right) => left.filePath.localeCompare(right.filePath)),
+    importedPlayDependencies: [...importedPlayDependencies.values()].sort(
+      (left, right) => left.filePath.localeCompare(right.filePath),
+    ),
   };
 }
 
@@ -995,14 +1079,19 @@ async function computeWorkersHarnessFingerprintWithAdapter(
   adapter: PlayBundlingAdapter,
 ): Promise<string> {
   const { readdir } = await import('node:fs/promises');
-  const entries = await readdir(adapter.workersHarnessFilesDir, { withFileTypes: true });
+  const entries = await readdir(adapter.workersHarnessFilesDir, {
+    withFileTypes: true,
+  });
   const tsFiles = entries
     .filter((e) => e.isFile() && /\.[cm]?ts$/.test(e.name))
     .map((e) => e.name)
     .sort();
   const parts: Array<{ name: string; hash: string }> = [];
   for (const name of tsFiles) {
-    const contents = await readFile(join(adapter.workersHarnessFilesDir, name), 'utf-8');
+    const contents = await readFile(
+      join(adapter.workersHarnessFilesDir, name),
+      'utf-8',
+    );
     parts.push({ name, hash: sha256(contents) });
   }
   return sha256(JSON.stringify(parts));
@@ -1137,6 +1226,18 @@ type EsbuildBundleOutput = {
   outputExtension: 'cjs' | 'mjs';
 };
 
+type PlayArtifactTargetAdapter = {
+  artifactKind: PlayArtifactKind;
+  codeFormat: PlayBundleArtifact['codeFormat'];
+  includeWorkersHarnessInGraphHash: boolean;
+  runEsbuild(input: {
+    entryFile: string;
+    importedPlayDependencies: ImportedPlayDependency[];
+    adapter: PlayBundlingAdapter;
+    exportName: string;
+  }): Promise<EsbuildBundleOutput | string[]>;
+};
+
 async function runEsbuildForCjsNode(
   entryFile: string,
   importedPlayDependencies: ImportedPlayDependency[],
@@ -1268,12 +1369,60 @@ async function runEsbuildForEsmWorkers(
   };
 }
 
+const PLAY_ARTIFACT_TARGET_ADAPTERS: Record<
+  PlayArtifactKind,
+  PlayArtifactTargetAdapter
+> = {
+  [PLAY_ARTIFACT_KINDS.cjsNode20]: {
+    artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
+    codeFormat: 'cjs_module',
+    includeWorkersHarnessInGraphHash: false,
+    runEsbuild: ({
+      entryFile,
+      importedPlayDependencies,
+      adapter,
+      exportName,
+    }) =>
+      runEsbuildForCjsNode(
+        entryFile,
+        importedPlayDependencies,
+        adapter,
+        exportName,
+      ),
+  },
+  [PLAY_ARTIFACT_KINDS.esmWorkers]: {
+    artifactKind: PLAY_ARTIFACT_KINDS.esmWorkers,
+    codeFormat: 'esm_module',
+    includeWorkersHarnessInGraphHash: true,
+    runEsbuild: ({
+      entryFile,
+      importedPlayDependencies,
+      adapter,
+      exportName,
+    }) =>
+      runEsbuildForEsmWorkers(
+        entryFile,
+        importedPlayDependencies,
+        adapter,
+        exportName,
+      ),
+  },
+};
+
+function resolvePlayArtifactTargetAdapter(
+  artifactKind: PlayArtifactKind,
+): PlayArtifactTargetAdapter {
+  return PLAY_ARTIFACT_TARGET_ADAPTERS[artifactKind];
+}
+
 export async function bundlePlayFile(
   filePath: string,
   options: BundlePlayFileCoreOptions,
 ): Promise<BundledPlayFileResult> {
   const adapter = options.adapter;
-  const target: PlayArtifactKind = options.target ?? PLAY_ARTIFACT_KINDS.cjsNode20;
+  const target: PlayArtifactKind =
+    options.target ?? PLAY_ARTIFACT_KINDS.cjsNode20;
+  const targetAdapter = resolvePlayArtifactTargetAdapter(target);
   const exportName = options.exportName?.trim() || 'default';
   assertValidExportName(exportName);
   const absolutePath = await normalizeLocalPath(filePath);
@@ -1291,8 +1440,9 @@ export async function bundlePlayFile(
     // changes and gets a fresh deploy. This is the file-watcher-equivalent
     // for hot-reload of the harness: editing entry.ts → next play run
     // re-bundles + redeploys automatically.
-    if (target === PLAY_ARTIFACT_KINDS.esmWorkers) {
-      const harnessFingerprint = await computeWorkersHarnessFingerprintWithAdapter(adapter);
+    if (targetAdapter.includeWorkersHarnessInGraphHash) {
+      const harnessFingerprint =
+        await computeWorkersHarnessFingerprintWithAdapter(adapter);
       analysis.graphHash = sha256(
         `${analysis.graphHash}\nworkers-harness:${harnessFingerprint}`,
       );
@@ -1303,7 +1453,9 @@ export async function bundlePlayFile(
         sourcePath: absolutePath,
         importedFilePaths: [
           ...analysis.importPolicy.localFiles,
-          ...analysis.importedPlayDependencies.map((dependency) => dependency.filePath),
+          ...analysis.importedPlayDependencies.map(
+            (dependency) => dependency.filePath,
+          ),
         ],
       })) ?? []),
     ];
@@ -1318,8 +1470,13 @@ export async function bundlePlayFile(
     // Cache lookup happens after validation because a bundle cache hit is keyed
     // by source and target, while cloud descriptor typecheck results also depend
     // on generated tool metadata.
-    const cachedArtifact = await readArtifactCache(analysis.graphHash, target, adapter);
-    const discoveredFiles = await adapter.discoverPackagedLocalFiles(absolutePath);
+    const cachedArtifact = await readArtifactCache(
+      analysis.graphHash,
+      target,
+      adapter,
+    );
+    const discoveredFiles =
+      await adapter.discoverPackagedLocalFiles(absolutePath);
     if (cachedArtifact) {
       const cachedArtifactSizeError = getBundleSizeError(
         absolutePath,
@@ -1347,10 +1504,12 @@ export async function bundlePlayFile(
       };
     }
 
-    const buildOutcome =
-      target === PLAY_ARTIFACT_KINDS.esmWorkers
-        ? await runEsbuildForEsmWorkers(absolutePath, analysis.importedPlayDependencies, adapter, exportName)
-        : await runEsbuildForCjsNode(absolutePath, analysis.importedPlayDependencies, adapter, exportName);
+    const buildOutcome = await targetAdapter.runEsbuild({
+      entryFile: absolutePath,
+      importedPlayDependencies: analysis.importedPlayDependencies,
+      adapter,
+      exportName,
+    });
     if (Array.isArray(buildOutcome)) {
       return {
         success: false,
@@ -1380,11 +1539,8 @@ export async function bundlePlayFile(
       };
     }
 
-    const codeFormat: PlayBundleArtifact['codeFormat'] =
-      target === PLAY_ARTIFACT_KINDS.esmWorkers ? 'esm_module' : 'cjs_module';
-
     const artifact: PlayBundleArtifact = {
-      codeFormat,
+      codeFormat: targetAdapter.codeFormat,
       artifactKind: target,
       entryFile: absolutePath,
       virtualFilename,
@@ -1416,7 +1572,7 @@ export async function bundlePlayFile(
   } catch (error) {
     if (error && typeof error === 'object' && 'errors' in error) {
       const errors = Array.isArray((error as { errors?: Message[] }).errors)
-        ? ((error as { errors: Message[] }).errors).map(formatEsbuildMessage)
+        ? (error as { errors: Message[] }).errors.map(formatEsbuildMessage)
         : ['Play bundling failed.'];
       return {
         success: false,

package/dist/repo/shared_libs/plays/dataset.ts

@@ -55,14 +55,34 @@ export type PlayDatasetInput<T> =
   | AsyncIterable<T>
   | PlayDataset<T>;
 
+/**
+ * Durable handle for rows produced by `ctx.csv(...)` or `ctx.map(...).run()`.
+ *
+ * A `PlayDataset` is not a normal in-memory array. It points at runtime-managed
+ * rows, usually backed by persisted sheet storage, and carries metadata such as
+ * dataset kind, dataset id, table namespace, count, and preview rows.
+ *
+ * Pass dataset handles directly into later `ctx.map(...)` stages by default so
+ * Deepline keeps row progress, retries, memory use, and table output under
+ * runtime control. Use `count()` and `peek()` for bounded inspection. Use
+ * `materialize(limit)` or async iteration only when the dataset is intentionally
+ * small and bounded.
+ */
 export interface PlayDataset<T> extends AsyncIterable<T> {
   readonly [PLAY_DATASET_BRAND]: true;
+  /** Dataset kind. */
   readonly datasetKind: PlayDatasetKind;
+  /** Dataset id. */
   readonly datasetId: string;
+  /** Backing store info. */
   readonly backing?: PlayDatasetBacking;
+  /** Display label. */
   readonly sourceLabel?: string | null;
+  /** Runtime table name. */
   readonly tableNamespace?: string | null;
+  /** Row count. */
   count(): Promise<number>;
+  /** Preview rows. */
   peek(limit?: number): Promise<T[]>;
   /**
    * Explicit escape hatch for bounded result sets.
@@ -310,7 +330,11 @@ export async function materializePlayDatasetInput<T>(
   input: PlayDatasetInput<T>,
 ): Promise<T[]> {
   if (isPlayDataset(input)) {
-    return await input.materialize();
+    const rows: T[] = [];
+    for await (const row of input) {
+      rows.push(row);
+    }
+    return rows;
   }
 
   if (Array.isArray(input)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.54",
+  "version": "0.1.56",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {