deepline 0.1.79 → 0.1.81

package/dist/repo/shared_libs/play-runtime/submit-limits.ts

@@ -0,0 +1,35 @@
+/**
+ * Single source of truth for play submit-path size/wait limits.
+ *
+ * Dependency-free leaf shared by the Next.js run route
+ * (`POST /api/v2/plays/run`), the Cloudflare coordinator's workflow retry-state
+ * codec, and the limits documentation generator. These bound what a single
+ * submission may carry inline, the hard ceiling on submitted input, and how
+ * long the API will block for a synchronous result.
+ */
+
+/**
+ * Max size of a CSV/file input that may be inlined into the submit payload
+ * (and the workflow event history). Larger inputs must be staged via
+ * `POST /api/v2/plays/files/stage` and referenced by handle.
+ */
+export const MAX_TEMPORAL_INLINE_INPUT_FILE_BYTES = 64 * 1024;
+
+/**
+ * Max time the run route will block waiting for a play to finish before
+ * returning a pending handle the caller can poll/stream.
+ */
+export const MAX_WAIT_FOR_COMPLETION_MS = 15_000;
+
+/**
+ * Submitted input/retry-state at or below this size stays inline in the
+ * coordinator Durable Object. Above it, the params are externalized to a
+ * short-lived play artifact (up to {@link PLAY_SUBMIT_INPUT_MAX_BYTES}).
+ */
+export const PLAY_SUBMIT_INPUT_INLINE_MAX_BYTES = 100_000;
+
+/**
+ * Absolute ceiling for submitted input/retry-state. A submission larger than
+ * this is rejected with guidance to use staged files or `ctx.csv` inputs.
+ */
+export const PLAY_SUBMIT_INPUT_MAX_BYTES = 1024 * 1024;

package/dist/repo/shared_libs/plays/bundling/index.ts

@@ -26,20 +26,12 @@ import type {
 } from '../artifact-types';
 import { buildPlayContractCompatibility } from '../contracts';
 import { validatePlaySourceFilesHaveNoInlineSecrets } from '../secret-guardrails';
+import {
+  MAX_ESM_WORKERS_BUNDLE_BYTES,
+  MAX_PLAY_BUNDLE_BYTES,
+} from './limits';
 
 const PLAY_BUNDLE_CACHE_VERSION = 24;
-const MAX_PLAY_BUNDLE_BYTES = 30 * 1024 * 1024;
-// workerd local-mode (`wrangler dev` Worker Loader) silently fails to
-// instantiate per-graphHash play Workers when the bundled code passes a
-// threshold somewhere between 1.04 MiB (44-package-imports — works) and
-// 1.18 MiB (the same play with date-fns added — hangs forever). The
-// workflow body never runs, no error is logged anywhere, and the run
-// hangs indefinitely. We surface this as a hard bundle failure so the
-// user gets an actionable message at submit time instead of a 5-minute
-// silent timeout. Real CF (workers.dev) accepts much larger bundles, but
-// `dev:v2 cloudflare` is the regression entrypoint so the local limit is
-// the binding one.
-const MAX_ESM_WORKERS_BUNDLE_BYTES = 1_150_000;
 const PLAY_ARTIFACT_CACHE_DIR = join(
   tmpdir(),
   `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,

package/dist/repo/shared_libs/plays/bundling/limits.ts

@@ -0,0 +1,29 @@
+/**
+ * Single source of truth for play bundle/compile size limits.
+ *
+ * Kept in a dependency-free leaf module (no esbuild, no Node-only imports) so
+ * both the bundler (`./index.ts`) and the limits documentation generator
+ * (`scripts/generate-limits-docs.ts`) can import the numbers without pulling in
+ * the compile toolchain. Do not restate these values in docs — the public page
+ * is generated from here.
+ */
+
+/**
+ * Absolute hard ceiling for a compiled play bundle, across every artifact kind.
+ * A bundle larger than this is rejected at submit time.
+ */
+export const MAX_PLAY_BUNDLE_BYTES = 30 * 1024 * 1024;
+
+/**
+ * Tighter ceiling for the `esm_workers` artifact kind. workerd local-mode
+ * (`wrangler dev` Worker Loader) silently fails to instantiate per-graphHash
+ * play Workers when the bundled code passes a threshold somewhere between
+ * 1.04 MiB (44-package-imports — works) and 1.18 MiB (the same play with
+ * date-fns added — hangs forever). The workflow body never runs, no error is
+ * logged anywhere, and the run hangs indefinitely. We surface this as a hard
+ * bundle failure so the user gets an actionable message at submit time instead
+ * of a 5-minute silent timeout. Real CF (workers.dev) accepts much larger
+ * bundles, but `dev:v2 cloudflare` is the regression entrypoint so the local
+ * limit is the binding one.
+ */
+export const MAX_ESM_WORKERS_BUNDLE_BYTES = 1_150_000;

package/dist/repo/shared_libs/plays/static-pipeline.ts

@@ -1,5 +1,16 @@
 import { normalizeTableNamespace } from './row-identity';
 
+/**
+ * A top-level key the play's function literally `return`s. Derived from the
+ * `return { ... }` object literal — NOT from dataset `.withColumn(...)` names —
+ * so the "Returns" graph node mirrors the function's real output shape.
+ * `isDataset` is true when the key's value is a `PlayDataset` handle (a table).
+ */
+export interface PlayStaticReturnField {
+  name: string;
+  isDataset: boolean;
+}
+
 export interface PlayStaticPipeline {
   tableNamespace?: string;
   inputFields?: string[];
@@ -9,6 +20,12 @@ export interface PlayStaticPipeline {
   csvDescription?: string;
   datasetDescription?: string;
   fields: string[];
+  /**
+   * Top-level keys of the play's `return { ... }` object literal, in source
+   * order. Undefined when the terminal return isn't a statically-known object
+   * literal (bare value, dataset handle, conditional returns, etc.).
+   */
+  returnFields?: PlayStaticReturnField[];
   stages?: PlayStaticSubstep[];
   substeps: PlayStaticSubstep[];
   sheetContract?: PlaySheetContract | null;
@@ -54,6 +71,7 @@ export interface PlayStaticColumnProducer {
   field: string;
   toolId?: string;
   playId?: string;
+  staleAfterSeconds?: number;
   conditional?: boolean;
   sourceRange?: PlayStaticSourceRange;
   steps?: PlayStaticColumnProducer[];
@@ -64,6 +82,7 @@ export interface PlayStaticDatasetColumn {
   id: string;
   source: PlaySheetColumnSource;
   sqlName?: string;
+  staleAfterSeconds?: number;
   producers: PlayStaticColumnProducer[];
 }
 
@@ -212,6 +231,9 @@ export function truncateStaticPipelineForStorage(
       ? [...pipeline.rowKeyFields]
       : undefined,
     fields: [...(pipeline.fields ?? [])],
+    returnFields: pipeline.returnFields
+      ? pipeline.returnFields.map((field) => ({ ...field }))
+      : undefined,
     stages: truncateStaticSubstepsForStorage(pipeline.stages, {
       embeddedPlayCallPipelineDepth,
       maxEmbeddedPlayCallPipelineDepth,
@@ -237,6 +259,7 @@ export interface PlayStaticSourceRange {
 
 type PlayStaticSubstepMetadata = {
   conditional?: boolean;
+  staleAfterSeconds?: number;
 };
 
 export type PlayStaticSubstep = PlayStaticSubstepMetadata &
@@ -433,7 +456,7 @@ export function compileStaticGraph(
     if (substep.type !== 'dataset') {
       return substep;
     }
-    const columns = compileDatasetColumns(substep);
+    const columns = compileDatasetColumns(substep, pipeline?.substeps ?? []);
     const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
     if (tableNamespace) {
       datasets.push({ tableNamespace, columns });
@@ -448,6 +471,7 @@ export function compileStaticGraph(
 
 function compileDatasetColumns(
   dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
+  pipelineSubsteps: PlayStaticSubstep[] = [],
 ): PlayStaticDatasetColumn[] {
   const columnsById = new Map<string, PlayStaticDatasetColumn>();
   const ensureColumn = (
@@ -482,7 +506,15 @@ function compileDatasetColumns(
     ensureColumn(field, 'datasetColumn', sqlSafePlayColumnName(field));
   }
 
-  for (const substep of dataset.steps ?? []) {
+  const datasetProducerSteps =
+    dataset.steps && dataset.steps.length > 0
+      ? dataset.steps
+      : pipelineSubsteps.filter((substep) => {
+          const field = fieldForColumnProducer(substep);
+          return field ? (dataset.outputFields ?? []).includes(field) : false;
+        });
+
+  for (const substep of datasetProducerSteps) {
     const field = fieldForColumnProducer(substep);
     if (!field) continue;
     const column = ensureColumn(
@@ -491,7 +523,25 @@ function compileDatasetColumns(
       sqlSafePlayColumnName(field),
     );
     if (!column) continue;
-    column.producers.push(columnProducerFromSubstep(substep, field));
+    const pipelineSubstep =
+      substep.staleAfterSeconds === undefined
+        ? pipelineSubsteps.find(
+            (candidate) => fieldForColumnProducer(candidate) === field,
+          )
+        : undefined;
+    const producer = columnProducerFromSubstep(
+      pipelineSubstep && pipelineSubstep.staleAfterSeconds !== undefined
+        ? pipelineSubstep
+        : substep,
+      field,
+    );
+    column.producers.push(producer);
+    if (
+      column.staleAfterSeconds === undefined &&
+      producer.staleAfterSeconds !== undefined
+    ) {
+      column.staleAfterSeconds = producer.staleAfterSeconds;
+    }
   }
 
   return [...columnsById.values()];
@@ -536,6 +586,9 @@ function columnProducerFromSubstep(
     field,
     ...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
     ...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
+    ...(substep.staleAfterSeconds !== undefined
+      ? { staleAfterSeconds: substep.staleAfterSeconds }
+      : {}),
     ...(substep.conditional ? { conditional: true } : {}),
     ...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
     ...(steps && steps.length > 0 ? { steps } : {}),

package/dist/repo/shared_libs/temporal/constants.ts

@@ -0,0 +1,38 @@
+/**
+ * Shared Temporal execution constants.
+ *
+ * Keep values that both the API/auth surface and the worker need here so the
+ * API never imports from worker-only modules.
+ */
+
+/**
+ * Local Temporal dev defaults.
+ *
+ * These match the host ports exposed by docker-compose.yml and the env files
+ * used by the local dev flows (`.env.local`, `.env.worktree`).
+ */
+export const LOCAL_TEMPORAL_FRONTEND_PORT = 17233;
+export const LOCAL_TEMPORAL_UI_PORT = 18233;
+export const LOCAL_TEMPORAL_NAMESPACE = 'default';
+export const LOCAL_TEMPORAL_ADDRESS = `127.0.0.1:${LOCAL_TEMPORAL_FRONTEND_PORT}`;
+export const LOCAL_TEMPORAL_UI_URL = `http://127.0.0.1:${LOCAL_TEMPORAL_UI_PORT}`;
+
+/** Maximum active user-code runtime for a standard play, in seconds. */
+export const STANDARD_PLAY_RUNTIME_LIMIT_SECONDS = 10 * 60; // 10 minutes
+
+/**
+ * Activity timeout includes cleanup/billing headroom after the 10 minute
+ * user-code runtime cap. Keep this higher than STANDARD_PLAY_RUNTIME_LIMIT_SECONDS.
+ */
+export const PLAY_ACTIVITY_TIMEOUT_SECONDS = 12 * 60; // 12 minutes
+
+/** Heartbeat cadence for the long-running play execution activity. */
+export const PLAY_EXECUTE_ACTIVITY_HEARTBEAT_INTERVAL_SECONDS = 15;
+
+/**
+ * TTL for workflow executor tokens, in seconds.
+ * Matches the activity timeout so tokens expire when the activity would
+ * time out anyway.
+ */
+export const WORKFLOW_EXECUTOR_TOKEN_TTL_SECONDS =
+  PLAY_ACTIVITY_TIMEOUT_SECONDS;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.79",
+  "version": "0.1.81",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {

package/dist/repo/shared_libs/play-runtime/tool-batch-executor.ts

@@ -1,149 +0,0 @@
-export type ToolBatchItem = {
-  itemKey: string;
-  payload: Record<string, unknown>;
-  inputHash?: string | null;
-};
-
-export type ToolBatchRequest = {
-  runId: string;
-  orgId: string;
-  toolId: string;
-  operation: string;
-  provider: string;
-  items: ToolBatchItem[];
-  waterfallId?: string | null;
-  stageId?: string | null;
-  fieldName?: string | null;
-  mapName?: string | null;
-  chunkIndex?: number | null;
-  userProvidedRateLimitKey?: string | null;
-  providerBatchSize: number;
-};
-
-export type ToolBatchItemResult = {
-  itemKey: string;
-  result: unknown;
-  cached?: boolean;
-};
-
-export type ToolBatchResult = {
-  runId: string;
-  toolId: string;
-  operation: string;
-  provider: string;
-  batchCount: number;
-  itemCount: number;
-  results: ToolBatchItemResult[];
-};
-
-export type ToolBatchExecutorTransport = {
-  executeProviderBatch(input: {
-    request: ToolBatchRequest;
-    batchIndex: number;
-    idempotencyKeys: string[];
-    rateLimitKey: string;
-    items: ToolBatchItem[];
-  }): Promise<ToolBatchItemResult[]>;
-};
-
-export type ToolBatchExecutor = {
-  executeToolBatch(request: ToolBatchRequest): Promise<ToolBatchResult>;
-};
-
-export function createToolBatchExecutor(
-  transport: ToolBatchExecutorTransport,
-): ToolBatchExecutor {
-  return {
-    async executeToolBatch(request) {
-      const providerBatchSize = Math.max(
-        1,
-        Math.floor(request.providerBatchSize),
-      );
-      const batches = chunkToolBatchItems(request.items, providerBatchSize);
-      const results: ToolBatchItemResult[] = [];
-      for (let batchIndex = 0; batchIndex < batches.length; batchIndex += 1) {
-        const items = batches[batchIndex]!;
-        results.push(
-          ...(await transport.executeProviderBatch({
-            request,
-            batchIndex,
-            items,
-            rateLimitKey: buildToolBatchRateLimitKey(request),
-            idempotencyKeys: items.map((item) =>
-              buildToolBatchIdempotencyKey(request, item),
-            ),
-          })),
-        );
-      }
-      return {
-        runId: request.runId,
-        toolId: request.toolId,
-        operation: request.operation,
-        provider: request.provider,
-        batchCount: batches.length,
-        itemCount: request.items.length,
-        results,
-      };
-    },
-  };
-}
-
-export function buildToolBatchIdempotencyKey(
-  request: ToolBatchRequest,
-  item: ToolBatchItem,
-): string {
-  return [
-    request.runId,
-    request.mapName ?? '',
-    request.chunkIndex ?? '',
-    item.itemKey,
-    request.fieldName ?? '',
-    request.waterfallId ?? '',
-    request.stageId ?? '',
-    item.inputHash ?? stableToolBatchHash(item.payload),
-  ].join(':');
-}
-
-export function buildToolBatchRateLimitKey(request: ToolBatchRequest): string {
-  return [
-    request.orgId,
-    request.provider,
-    request.operation,
-    request.userProvidedRateLimitKey ?? '',
-  ].join(':');
-}
-
-function chunkToolBatchItems(
-  items: readonly ToolBatchItem[],
-  size: number,
-): ToolBatchItem[][] {
-  const chunks: ToolBatchItem[][] = [];
-  for (let index = 0; index < items.length; index += size) {
-    chunks.push(items.slice(index, index + size));
-  }
-  return chunks;
-}
-
-function stableToolBatchHash(value: unknown): string {
-  const text = stableStringify(value);
-  let hash = 2166136261;
-  for (let index = 0; index < text.length; index += 1) {
-    hash ^= text.charCodeAt(index);
-    hash = Math.imul(hash, 16777619);
-  }
-  return (hash >>> 0).toString(36);
-}
-
-function stableStringify(value: unknown): string {
-  if (value === null || typeof value !== 'object') {
-    return JSON.stringify(value);
-  }
-  if (Array.isArray(value)) {
-    return `[${value.map(stableStringify).join(',')}]`;
-  }
-  const record = value as Record<string, unknown>;
-  return `{${Object.keys(record)
-    .sort()
-    .map((key) => `${JSON.stringify(key)}:${stableStringify(record[key])}`)
-    .join(',')}}`;
-}