deepline 0.1.78 → 0.1.80

package/dist/repo/shared_libs/play-runtime/submit-limits.ts

@@ -0,0 +1,35 @@
+/**
+ * Single source of truth for play submit-path size/wait limits.
+ *
+ * Dependency-free leaf shared by the Next.js run route
+ * (`POST /api/v2/plays/run`), the Cloudflare coordinator's workflow retry-state
+ * codec, and the limits documentation generator. These bound what a single
+ * submission may carry inline, the hard ceiling on submitted input, and how
+ * long the API will block for a synchronous result.
+ */
+
+/**
+ * Max size of a CSV/file input that may be inlined into the submit payload
+ * (and the workflow event history). Larger inputs must be staged via
+ * `POST /api/v2/plays/files/stage` and referenced by handle.
+ */
+export const MAX_TEMPORAL_INLINE_INPUT_FILE_BYTES = 64 * 1024;
+
+/**
+ * Max time the run route will block waiting for a play to finish before
+ * returning a pending handle the caller can poll/stream.
+ */
+export const MAX_WAIT_FOR_COMPLETION_MS = 15_000;
+
+/**
+ * Submitted input/retry-state at or below this size stays inline in the
+ * coordinator Durable Object. Above it, the params are externalized to a
+ * short-lived play artifact (up to {@link PLAY_SUBMIT_INPUT_MAX_BYTES}).
+ */
+export const PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES = 100_000;
+
+/**
+ * Absolute ceiling for submitted input/retry-state. A submission larger than
+ * this is rejected with guidance to use staged files or `ctx.csv` inputs.
+ */
+export const PLAY_SUBMIT_INPUT_MAX_BYTES = 1024 * 1024;

package/dist/repo/shared_libs/plays/bundling/index.ts

@@ -26,20 +26,12 @@ import type {
 } from '../artifact-types';
 import { buildPlayContractCompatibility } from '../contracts';
 import { validatePlaySourceFilesHaveNoInlineSecrets } from '../secret-guardrails';
+import {
+  MAX_ESM_WORKERS_BUNDLE_BYTES,
+  MAX_PLAY_BUNDLE_BYTES,
+} from './limits';
 
 const PLAY_BUNDLE_CACHE_VERSION = 24;
-const MAX_PLAY_BUNDLE_BYTES = 30 * 1024 * 1024;
-// workerd local-mode (`wrangler dev` Worker Loader) silently fails to
-// instantiate per-graphHash play Workers when the bundled code passes a
-// threshold somewhere between 1.04 MiB (44-package-imports — works) and
-// 1.18 MiB (the same play with date-fns added — hangs forever). The
-// workflow body never runs, no error is logged anywhere, and the run
-// hangs indefinitely. We surface this as a hard bundle failure so the
-// user gets an actionable message at submit time instead of a 5-minute
-// silent timeout. Real CF (workers.dev) accepts much larger bundles, but
-// `dev:v2 cloudflare` is the regression entrypoint so the local limit is
-// the binding one.
-const MAX_ESM_WORKERS_BUNDLE_BYTES = 1_150_000;
 const PLAY_ARTIFACT_CACHE_DIR = join(
   tmpdir(),
   `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,

package/dist/repo/shared_libs/plays/bundling/limits.ts

@@ -0,0 +1,29 @@
+/**
+ * Single source of truth for play bundle/compile size limits.
+ *
+ * Kept in a dependency-free leaf module (no esbuild, no Node-only imports) so
+ * both the bundler (`./index.ts`) and the limits documentation generator
+ * (`scripts/generate-limits-docs.ts`) can import the numbers without pulling in
+ * the compile toolchain. Do not restate these values in docs — the public page
+ * is generated from here.
+ */
+
+/**
+ * Absolute hard ceiling for a compiled play bundle, across every artifact kind.
+ * A bundle larger than this is rejected at submit time.
+ */
+export const MAX_PLAY_BUNDLE_BYTES = 30 * 1024 * 1024;
+
+/**
+ * Tighter ceiling for the `esm_workers` artifact kind. workerd local-mode
+ * (`wrangler dev` Worker Loader) silently fails to instantiate per-graphHash
+ * play Workers when the bundled code passes a threshold somewhere between
+ * 1.04 MiB (44-package-imports — works) and 1.18 MiB (the same play with
+ * date-fns added — hangs forever). The workflow body never runs, no error is
+ * logged anywhere, and the run hangs indefinitely. We surface this as a hard
+ * bundle failure so the user gets an actionable message at submit time instead
+ * of a 5-minute silent timeout. Real CF (workers.dev) accepts much larger
+ * bundles, but `dev:v2 cloudflare` is the regression entrypoint so the local
+ * limit is the binding one.
+ */
+export const MAX_ESM_WORKERS_BUNDLE_BYTES = 1_150_000;

package/dist/repo/shared_libs/plays/static-pipeline.ts

@@ -1,13 +1,31 @@
 import { normalizeTableNamespace } from './row-identity';
 
+/**
+ * A top-level key the play's function literally `return`s. Derived from the
+ * `return { ... }` object literal — NOT from dataset `.withColumn(...)` names —
+ * so the "Returns" graph node mirrors the function's real output shape.
+ * `isDataset` is true when the key's value is a `PlayDataset` handle (a table).
+ */
+export interface PlayStaticReturnField {
+  name: string;
+  isDataset: boolean;
+}
+
 export interface PlayStaticPipeline {
   tableNamespace?: string;
   inputFields?: string[];
+  rowKeyFields?: string[];
   csvArg?: string;
   hasInlineData?: boolean;
   csvDescription?: string;
   datasetDescription?: string;
   fields: string[];
+  /**
+   * Top-level keys of the play's `return { ... }` object literal, in source
+   * order. Undefined when the terminal return isn't a statically-known object
+   * literal (bare value, dataset handle, conditional returns, etc.).
+   */
+  returnFields?: PlayStaticReturnField[];
   stages?: PlayStaticSubstep[];
   substeps: PlayStaticSubstep[];
   sheetContract?: PlaySheetContract | null;
@@ -32,6 +50,7 @@ export interface PlaySheetColumnContract {
   outputSqlName?: string;
   stepId?: string;
   toolId?: string;
+  isRowKey?: boolean;
 }
 
 export interface PlaySheetContract {
@@ -39,6 +58,42 @@ export interface PlaySheetContract {
   columns: PlaySheetColumnContract[];
 }
 
+export type PlayStaticColumnProducerKind =
+  | 'tool'
+  | 'waterfall'
+  | 'stepProgram'
+  | 'playCall'
+  | 'transform';
+
+export interface PlayStaticColumnProducer {
+  id: string;
+  kind: PlayStaticColumnProducerKind;
+  field: string;
+  toolId?: string;
+  playId?: string;
+  staleAfterSeconds?: number;
+  conditional?: boolean;
+  sourceRange?: PlayStaticSourceRange;
+  steps?: PlayStaticColumnProducer[];
+  substep: PlayStaticSubstep;
+}
+
+export interface PlayStaticDatasetColumn {
+  id: string;
+  source: PlaySheetColumnSource;
+  sqlName?: string;
+  staleAfterSeconds?: number;
+  producers: PlayStaticColumnProducer[];
+}
+
+export interface PlayCompiledStaticGraph {
+  topLevel: PlayStaticSubstep[];
+  datasets: Array<{
+    tableNamespace: string;
+    columns: PlayStaticDatasetColumn[];
+  }>;
+}
+
 export function ensureCompiledSheetContract(
   pipeline: PlayStaticPipeline | null | undefined,
 ): PlayStaticPipeline | null | undefined {
@@ -92,8 +147,27 @@ function truncateStaticSubstepsForStorage(
       return {
         ...base,
         inputFields: base.inputFields ? [...base.inputFields] : undefined,
+        rowKeyFields: base.rowKeyFields ? [...base.rowKeyFields] : undefined,
         outputFields: base.outputFields ? [...base.outputFields] : undefined,
+        columns: base.columns
+          ? base.columns.map((column) => ({
+              ...column,
+              producers: column.producers.map((producer) => ({
+                ...producer,
+                sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
+                steps: producer.steps
+                  ? producer.steps.map((stepProducer) => ({
+                      ...stepProducer,
+                      sourceRange: cloneStorageSafeSourceRange(
+                        stepProducer.sourceRange,
+                      ),
+                    }))
+                  : undefined,
+              })),
+            }))
+          : undefined,
         waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
+        steps: truncateStaticSubstepsForStorage(base.steps, input),
         sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
       };
     }
@@ -153,7 +227,13 @@ export function truncateStaticPipelineForStorage(
   return {
     ...pipeline,
     inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+    rowKeyFields: pipeline.rowKeyFields
+      ? [...pipeline.rowKeyFields]
+      : undefined,
     fields: [...(pipeline.fields ?? [])],
+    returnFields: pipeline.returnFields
+      ? pipeline.returnFields.map((field) => ({ ...field }))
+      : undefined,
     stages: truncateStaticSubstepsForStorage(pipeline.stages, {
       embeddedPlayCallPipelineDepth,
       maxEmbeddedPlayCallPipelineDepth,
@@ -179,6 +259,7 @@ export interface PlayStaticSourceRange {
 
 type PlayStaticSubstepMetadata = {
   conditional?: boolean;
+  staleAfterSeconds?: number;
 };
 
 export type PlayStaticSubstep = PlayStaticSubstepMetadata &
@@ -198,8 +279,11 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         name?: string;
         tableNamespace?: string;
         inputFields?: string[];
+        rowKeyFields?: string[];
         outputFields?: string[];
+        columns?: PlayStaticDatasetColumn[];
         waterfallIds?: string[];
+        steps?: PlayStaticSubstep[];
         sheetContract?: PlaySheetContract | null;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
@@ -210,6 +294,8 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         type: 'tool';
         toolId: string;
         field: string;
+        paramsSource?: string;
+        sourceText?: string;
         description?: string;
         inLoop?: boolean;
         isEventWait?: boolean;
@@ -263,6 +349,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
     | {
         type: 'run_javascript';
         alias: string;
+        sourceText?: string;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
         callDepth?: number;
@@ -271,6 +358,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
     | {
         type: 'code';
         field: string;
+        sourceText?: string;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
         callDepth?: number;
@@ -289,6 +377,12 @@ export function getCompiledPipelineSubsteps(
 
 export function getTopLevelPipelineSubsteps(
   pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticSubstep[] {
+  return compileStaticGraph(pipeline).topLevel;
+}
+
+function getRawTopLevelPipelineSubsteps(
+  pipeline: PlayStaticPipeline | null | undefined,
 ): PlayStaticSubstep[] {
   if (!pipeline) {
     return [];
@@ -328,6 +422,14 @@ export function flattenStaticSubsteps(
 
   for (const substep of substeps) {
     flattened.push(substep);
+    if (substep.type === 'dataset' && substep.steps?.length) {
+      flattened.push(...flattenStaticSubsteps(substep.steps));
+      continue;
+    }
+    if (substep.type === 'step_suite') {
+      flattened.push(...flattenStaticSubsteps(substep.steps));
+      continue;
+    }
     if (substep.type === 'play_call' && substep.pipeline) {
       const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
       if (nestedSubsteps.length > 0) {
@@ -345,6 +447,167 @@ export function flattenStaticPipeline(
   return flattenStaticSubsteps(getCompiledPipelineSubsteps(pipeline));
 }
 
+export function compileStaticGraph(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayCompiledStaticGraph {
+  const rawTopLevel = getRawTopLevelPipelineSubsteps(pipeline);
+  const datasets: PlayCompiledStaticGraph['datasets'] = [];
+  const topLevel = rawTopLevel.map((substep) => {
+    if (substep.type !== 'dataset') {
+      return substep;
+    }
+    const columns = compileDatasetColumns(substep, pipeline?.substeps ?? []);
+    const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
+    if (tableNamespace) {
+      datasets.push({ tableNamespace, columns });
+    }
+    return {
+      ...substep,
+      columns,
+    } satisfies PlayStaticSubstep;
+  });
+  return { topLevel, datasets };
+}
+
+function compileDatasetColumns(
+  dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
+  pipelineSubsteps: PlayStaticSubstep[] = [],
+): PlayStaticDatasetColumn[] {
+  const columnsById = new Map<string, PlayStaticDatasetColumn>();
+  const ensureColumn = (
+    id: string,
+    source: PlaySheetColumnSource,
+    sqlName?: string,
+  ) => {
+    const trimmed = id.trim();
+    if (!trimmed) return null;
+    const existing = columnsById.get(trimmed);
+    if (existing) {
+      if (!existing.sqlName && sqlName) existing.sqlName = sqlName;
+      return existing;
+    }
+    const column: PlayStaticDatasetColumn = {
+      id: trimmed,
+      source,
+      ...(sqlName ? { sqlName } : {}),
+      producers: [],
+    };
+    columnsById.set(trimmed, column);
+    return column;
+  };
+
+  for (const column of dataset.sheetContract?.columns ?? []) {
+    ensureColumn(column.id, column.source, column.sqlName);
+  }
+  for (const field of dataset.inputFields ?? []) {
+    ensureColumn(field, 'input', sqlSafePlayColumnName(field));
+  }
+  for (const field of dataset.outputFields ?? []) {
+    ensureColumn(field, 'datasetColumn', sqlSafePlayColumnName(field));
+  }
+
+  const datasetProducerSteps =
+    dataset.steps && dataset.steps.length > 0
+      ? dataset.steps
+      : pipelineSubsteps.filter((substep) => {
+          const field = fieldForColumnProducer(substep);
+          return field ? (dataset.outputFields ?? []).includes(field) : false;
+        });
+
+  for (const substep of datasetProducerSteps) {
+    const field = fieldForColumnProducer(substep);
+    if (!field) continue;
+    const column = ensureColumn(
+      field,
+      'datasetColumn',
+      sqlSafePlayColumnName(field),
+    );
+    if (!column) continue;
+    const pipelineSubstep =
+      substep.staleAfterSeconds === undefined
+        ? pipelineSubsteps.find(
+            (candidate) => fieldForColumnProducer(candidate) === field,
+          )
+        : undefined;
+    const producer = columnProducerFromSubstep(
+      pipelineSubstep && pipelineSubstep.staleAfterSeconds !== undefined
+        ? pipelineSubstep
+        : substep,
+      field,
+    );
+    column.producers.push(producer);
+    if (
+      column.staleAfterSeconds === undefined &&
+      producer.staleAfterSeconds !== undefined
+    ) {
+      column.staleAfterSeconds = producer.staleAfterSeconds;
+    }
+  }
+
+  return [...columnsById.values()];
+}
+
+function fieldForColumnProducer(substep: PlayStaticSubstep): string | null {
+  if ('field' in substep && typeof substep.field === 'string') {
+    return substep.field.trim() || null;
+  }
+  if (substep.type === 'run_javascript') {
+    return substep.alias.trim() || null;
+  }
+  return null;
+}
+
+function columnProducerFromSubstep(
+  substep: PlayStaticSubstep,
+  field: string,
+): PlayStaticColumnProducer {
+  const steps =
+    substep.type === 'step_suite'
+      ? substep.steps
+          .map((step) => {
+            const stepField = fieldForColumnProducer(step) ?? field;
+            return columnProducerFromSubstep(step, stepField);
+          })
+          .filter((producer) => producer.field.trim())
+      : undefined;
+  const kind: PlayStaticColumnProducerKind =
+    substep.type === 'tool'
+      ? 'tool'
+      : substep.type === 'waterfall'
+        ? 'waterfall'
+        : substep.type === 'step_suite'
+          ? 'stepProgram'
+          : substep.type === 'play_call'
+            ? 'playCall'
+            : 'transform';
+  return {
+    id: producerId(substep, field),
+    kind,
+    field,
+    ...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
+    ...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
+    ...(substep.staleAfterSeconds !== undefined
+      ? { staleAfterSeconds: substep.staleAfterSeconds }
+      : {}),
+    ...(substep.conditional ? { conditional: true } : {}),
+    ...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
+    ...(steps && steps.length > 0 ? { steps } : {}),
+    substep,
+  };
+}
+
+function producerId(substep: PlayStaticSubstep, field: string): string {
+  if (substep.type === 'tool') return `${field}:tool:${substep.toolId}`;
+  if (substep.type === 'waterfall') {
+    return `${field}:waterfall:${substep.id ?? substep.tool ?? 'unknown'}`;
+  }
+  if (substep.type === 'play_call') return `${field}:play:${substep.playId}`;
+  if (substep.type === 'step_suite') return `${field}:steps`;
+  if (substep.type === 'run_javascript')
+    return `${field}:transform:${substep.alias}`;
+  return `${field}:transform`;
+}
+
 export function resolveSheetContractForTableNamespace(
   pipeline: PlayStaticPipeline | null | undefined,
   tableNamespace: string | null | undefined,
@@ -400,6 +663,50 @@ export function resolveSheetContractForTableNamespace(
   return resolveFromPipeline(pipeline);
 }
 
+export function resolveStaticDatasetColumnsForTableNamespace(
+  pipeline: PlayStaticPipeline | null | undefined,
+  tableNamespace: string | null | undefined,
+): PlayStaticDatasetColumn[] {
+  const requestedNamespace = tableNamespace?.trim();
+  if (!pipeline || !requestedNamespace) {
+    return [];
+  }
+  const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
+  const seen = new Set<PlayStaticPipeline>();
+
+  const resolveFromPipeline = (
+    currentPipeline: PlayStaticPipeline | null | undefined,
+  ): PlayStaticDatasetColumn[] | null => {
+    if (!currentPipeline || seen.has(currentPipeline)) {
+      return null;
+    }
+    seen.add(currentPipeline);
+
+    const compiled = compileStaticGraph(currentPipeline);
+    const matchingDataset = compiled.datasets.find(
+      (dataset) =>
+        normalizeTableNamespace(dataset.tableNamespace) ===
+        normalizedNamespace,
+    );
+    if (matchingDataset) {
+      return matchingDataset.columns;
+    }
+
+    for (const substep of getCompiledPipelineSubsteps(currentPipeline)) {
+      if (substep.type === 'play_call') {
+        const nestedColumns = resolveFromPipeline(substep.pipeline);
+        if (nestedColumns) {
+          return nestedColumns;
+        }
+      }
+    }
+
+    return null;
+  };
+
+  return resolveFromPipeline(pipeline) ?? [];
+}
+
 export function sqlSafePlayColumnName(id: string): string {
   const normalized = id
     .trim()
@@ -441,8 +748,14 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
   const inputFields = pipeline.inputFields?.length
     ? pipeline.inputFields
     : [tableNamespace];
+  const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
   for (const inputField of inputFields) {
-    addColumn({ id: inputField, source: 'input', field: inputField });
+    addColumn({
+      id: inputField,
+      source: 'input',
+      field: inputField,
+      ...(rowKeyFieldSet.has(inputField) ? { isRowKey: true } : {}),
+    });
   }
 
   for (const field of pipeline.fields) {

package/dist/repo/shared_libs/temporal/constants.ts

@@ -0,0 +1,38 @@
+/**
+ * Shared Temporal execution constants.
+ *
+ * Keep values that both the API/auth surface and the worker need here so the
+ * API never imports from worker-only modules.
+ */
+
+/**
+ * Local Temporal dev defaults.
+ *
+ * These match the host ports exposed by docker-compose.yml and the env files
+ * used by the local dev flows (`.env.local`, `.env.worktree`).
+ */
+export const LOCAL_TEMPORAL_FRONTEND_PORT = 17233;
+export const LOCAL_TEMPORAL_UI_PORT = 18233;
+export const LOCAL_TEMPORAL_NAMESPACE = 'default';
+export const LOCAL_TEMPORAL_ADDRESS = `127.0.0.1:${LOCAL_TEMPORAL_FRONTEND_PORT}`;
+export const LOCAL_TEMPORAL_UI_URL = `http://127.0.0.1:${LOCAL_TEMPORAL_UI_PORT}`;
+
+/** Maximum active user-code runtime for a standard play, in seconds. */
+export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 10 * 60; // 10 minutes
+
+/**
+ * Activity timeout includes cleanup/billing headroom after the 10 minute
+ * user-code runtime cap. Keep this higher than STANDARD_PLAY_RUNTIME_LIMIT_SECONDS.
+ */
+export const PLAY_ACTIVITY_TIMEOUT_SECONDS = 12 * 60; // 12 minutes
+
+/** Heartbeat cadence for the long-running play execution activity. */
+export const PLAY_EXECUTE_ACTIVITY_HEARTBEAT_INTERVAL_SECONDS = 15;
+
+/**
+ * TTL for workflow executor tokens, in seconds.
+ * Matches the activity timeout so tokens expire when the activity would
+ * time out anyway.
+ */
+export const WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS =
+  PLAY_ACTIVITY_TIMEOUT_SECONDS;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.78",
+  "version": "0.1.80",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {