deepline 0.1.78 → 0.1.79

package/dist/cli/index.js

@@ -229,10 +229,10 @@ var import_node_path2 = require("path");
 
 // src/release.ts
 var SDK_RELEASE = {
-  version: "0.1.78",
+  version: "0.1.79",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.78",
+    latest: "0.1.79",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }
@@ -7734,7 +7734,6 @@ ${hint}`;
 }
 
 // src/cli/commands/play.ts
-var PLAY_START_STREAM_FAST_COMPLETION_WAIT_MS = 2500;
 var PLAY_RUN_RESERVED_BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
   "--json",
   "--wait",
@@ -8639,10 +8638,6 @@ async function startAndWaitForPlayCompletionByStreamOnce(input2) {
   let eventCount = 0;
   let firstRunIdMs = null;
   let lastPhase = null;
-  const startRequest = {
-    ...input2.request,
-    waitForCompletionMs: typeof input2.request.waitForCompletionMs === "number" ? input2.request.waitForCompletionMs : PLAY_START_STREAM_FAST_COMPLETION_WAIT_MS
-  };
   const timeout = input2.waitTimeoutMs === null ? null : setTimeout(
     () => {
       timedOut = true;
@@ -8651,7 +8646,7 @@ async function startAndWaitForPlayCompletionByStreamOnce(input2) {
     Math.max(1, input2.waitTimeoutMs)
   );
   try {
-    for await (const event of input2.client.startPlayRunStream(startRequest, {
+    for await (const event of input2.client.startPlayRunStream(input2.request, {
       signal: controller.signal
     })) {
       eventCount += 1;

package/dist/cli/index.mjs

@@ -206,10 +206,10 @@ import { join as join2 } from "path";
 
 // src/release.ts
 var SDK_RELEASE = {
-  version: "0.1.78",
+  version: "0.1.79",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.78",
+    latest: "0.1.79",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }
@@ -7737,7 +7737,6 @@ ${hint}`;
 }
 
 // src/cli/commands/play.ts
-var PLAY_START_STREAM_FAST_COMPLETION_WAIT_MS = 2500;
 var PLAY_RUN_RESERVED_BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
   "--json",
   "--wait",
@@ -8642,10 +8641,6 @@ async function startAndWaitForPlayCompletionByStreamOnce(input2) {
   let eventCount = 0;
   let firstRunIdMs = null;
   let lastPhase = null;
-  const startRequest = {
-    ...input2.request,
-    waitForCompletionMs: typeof input2.request.waitForCompletionMs === "number" ? input2.request.waitForCompletionMs : PLAY_START_STREAM_FAST_COMPLETION_WAIT_MS
-  };
   const timeout = input2.waitTimeoutMs === null ? null : setTimeout(
     () => {
       timedOut = true;
@@ -8654,7 +8649,7 @@ async function startAndWaitForPlayCompletionByStreamOnce(input2) {
     Math.max(1, input2.waitTimeoutMs)
   );
   try {
-    for await (const event of input2.client.startPlayRunStream(startRequest, {
+    for await (const event of input2.client.startPlayRunStream(input2.request, {
       signal: controller.signal
     })) {
       eventCount += 1;

package/dist/index.d.mts

@@ -16,6 +16,7 @@ type PlayArtifactKind = (typeof PLAY_ARTIFACT_KINDS)[keyof typeof PLAY_ARTIFACT_
 interface PlayStaticPipelineSnapshot {
     tableNamespace?: string;
     inputFields?: string[];
+    rowKeyFields?: string[];
     csvArg?: string;
     hasInlineData?: boolean;
     csvDescription?: string;
@@ -39,11 +40,30 @@ interface PlaySheetColumnContractSnapshot {
     outputSqlName?: string;
     stepId?: string;
     toolId?: string;
+    isRowKey?: boolean;
 }
 interface PlaySheetContractSnapshot {
     tableNamespace: string;
     columns: PlaySheetColumnContractSnapshot[];
 }
+type PlayStaticColumnProducerKindSnapshot = 'tool' | 'waterfall' | 'stepProgram' | 'playCall' | 'transform';
+interface PlayStaticColumnProducerSnapshot {
+    id: string;
+    kind: PlayStaticColumnProducerKindSnapshot;
+    field: string;
+    toolId?: string;
+    playId?: string;
+    conditional?: boolean;
+    sourceRange?: PlayStaticSourceRangeSnapshot;
+    steps?: PlayStaticColumnProducerSnapshot[];
+    substep: PlayStaticSubstepSnapshot;
+}
+interface PlayStaticDatasetColumnSnapshot {
+    id: string;
+    source: PlaySheetColumnSourceSnapshot;
+    sqlName?: string;
+    producers: PlayStaticColumnProducerSnapshot[];
+}
 interface PlayStaticSourceRangeSnapshot {
     sourcePath?: string;
     startLine: number;
@@ -68,8 +88,11 @@ type PlayStaticSubstepSnapshot = PlayStaticSubstepMetadataSnapshot & ({
     name?: string;
     tableNamespace?: string;
     inputFields?: string[];
+    rowKeyFields?: string[];
     outputFields?: string[];
+    columns?: PlayStaticDatasetColumnSnapshot[];
     waterfallIds?: string[];
+    steps?: PlayStaticSubstepSnapshot[];
     sheetContract?: PlaySheetContractSnapshot | null;
     description?: string;
     sourceRange?: PlayStaticSourceRangeSnapshot;

package/dist/index.d.ts

@@ -16,6 +16,7 @@ type PlayArtifactKind = (typeof PLAY_ARTIFACT_KINDS)[keyof typeof PLAY_ARTIFACT_
 interface PlayStaticPipelineSnapshot {
     tableNamespace?: string;
     inputFields?: string[];
+    rowKeyFields?: string[];
     csvArg?: string;
     hasInlineData?: boolean;
     csvDescription?: string;
@@ -39,11 +40,30 @@ interface PlaySheetColumnContractSnapshot {
     outputSqlName?: string;
     stepId?: string;
     toolId?: string;
+    isRowKey?: boolean;
 }
 interface PlaySheetContractSnapshot {
     tableNamespace: string;
     columns: PlaySheetColumnContractSnapshot[];
 }
+type PlayStaticColumnProducerKindSnapshot = 'tool' | 'waterfall' | 'stepProgram' | 'playCall' | 'transform';
+interface PlayStaticColumnProducerSnapshot {
+    id: string;
+    kind: PlayStaticColumnProducerKindSnapshot;
+    field: string;
+    toolId?: string;
+    playId?: string;
+    conditional?: boolean;
+    sourceRange?: PlayStaticSourceRangeSnapshot;
+    steps?: PlayStaticColumnProducerSnapshot[];
+    substep: PlayStaticSubstepSnapshot;
+}
+interface PlayStaticDatasetColumnSnapshot {
+    id: string;
+    source: PlaySheetColumnSourceSnapshot;
+    sqlName?: string;
+    producers: PlayStaticColumnProducerSnapshot[];
+}
 interface PlayStaticSourceRangeSnapshot {
     sourcePath?: string;
     startLine: number;
@@ -68,8 +88,11 @@ type PlayStaticSubstepSnapshot = PlayStaticSubstepMetadataSnapshot & ({
     name?: string;
     tableNamespace?: string;
     inputFields?: string[];
+    rowKeyFields?: string[];
     outputFields?: string[];
+    columns?: PlayStaticDatasetColumnSnapshot[];
     waterfallIds?: string[];
+    steps?: PlayStaticSubstepSnapshot[];
     sheetContract?: PlaySheetContractSnapshot | null;
     description?: string;
     sourceRange?: PlayStaticSourceRangeSnapshot;

package/dist/index.js

@@ -241,10 +241,10 @@ var import_node_path2 = require("path");
 
 // src/release.ts
 var SDK_RELEASE = {
-  version: "0.1.78",
+  version: "0.1.79",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.78",
+    latest: "0.1.79",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }

package/dist/index.mjs

@@ -179,10 +179,10 @@ import { join as join2 } from "path";
 
 // src/release.ts
 var SDK_RELEASE = {
-  version: "0.1.78",
+  version: "0.1.79",
   apiContract: "2026-06-dataset-column-cell-stale-hard-cutover",
   supportPolicy: {
-    latest: "0.1.78",
+    latest: "0.1.79",
     minimumSupported: "0.1.53",
     deprecatedBelow: "0.1.53"
   }

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -53,6 +53,14 @@ import {
   decideWorkflowPlatformRetry,
   PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
 } from './workflow-retry';
+import {
+  WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES,
+  WORKFLOW_RETRY_PARAMS_MAX_BYTES,
+  buildWorkflowRetryParams,
+  jsonByteLength,
+  workflowRetryParamsStorageKey,
+  type WorkflowRetryParamsRef,
+} from './workflow-retry-state';
 import { sanitizeLiveLogLines } from './runtime/live-progress';
 
 export { DynamicWorkflowBinding };
@@ -563,7 +571,7 @@ const WORKFLOW_POOL_PROTOCOL_VERSION =
 const WORKFLOW_POOL_DO_NAME = 'workflow-pool:v2';
 const WORKFLOW_POOL_START_EVENT_TYPE = 'play_start';
 const WORKFLOW_POOL_TTL_MS = 8 * 60 * 1000;
-const WORKFLOW_POOL_TARGET_SIZE = 0;
+const WORKFLOW_POOL_TARGET_SIZE = 4;
 const WORKFLOW_POOL_READY_TIMEOUT_MS = 1_500;
 const WORKFLOW_POOL_READY_POLL_MS = 250;
 const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
@@ -571,6 +579,8 @@ const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 4;
 const WORKFLOW_POOL_CONTROL_TIMEOUT_MS = 750;
 const WORKFLOW_POOL_START_ACK_TIMEOUT_MS = 750;
 const WORKFLOW_POOL_START_ACK_POLL_MS = 25;
+const WORKFLOW_POOL_MISS_CLAIM_RETRY_TIMEOUT_MS = 3_000;
+const WORKFLOW_POOL_MISS_CLAIM_RETRY_POLL_MS = 50;
 const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
 const SUBMIT_INITIAL_STATE_POLL_MS = 50;
 const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
@@ -1117,6 +1127,115 @@ async function leaseWorkflowPoolId(
   return typeof body.id === 'string' && body.id ? body.id : null;
 }
 
+async function leaseWorkflowPoolIdWithMissRecovery(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  recordSubmitTiming: (timing: CoordinatorTiming) => void;
+  graphHash?: string | null;
+}): Promise<{
+  pooledInstanceId: string | null;
+  missCounts: WorkflowPoolCounts | null;
+  leaseError: string | null;
+}> {
+  let leaseError: string | null = null;
+  let pooledInstanceId = await leaseWorkflowPoolId(
+    input.env,
+    input.runId,
+  ).catch((error) => {
+    leaseError = error instanceof Error ? error.message : String(error);
+    return null;
+  });
+  let missCounts = pooledInstanceId
+    ? null
+    : await workflowPoolCount(input.env).catch(() => null);
+  if (
+    pooledInstanceId ||
+    leaseError ||
+    !missCounts ||
+    missCounts.available + missCounts.warming <= 0
+  ) {
+    return { pooledInstanceId, missCounts, leaseError };
+  }
+
+  const recoveryStartedAt = Date.now();
+  const refill = await refillWorkflowPool(input.env, {
+    minAvailable: 1,
+    waitReady: true,
+    waitTimeoutMs: WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS,
+  }).catch((error) => {
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_refill_on_miss',
+      ms: Date.now() - recoveryStartedAt,
+      graphHash: input.graphHash ?? null,
+      extra: {
+        status: 'failed',
+        error: error instanceof Error ? error.message : String(error),
+        available: missCounts?.available ?? null,
+        warming: missCounts?.warming ?? null,
+      },
+    });
+    return null;
+  });
+  if (refill) {
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_refill_on_miss',
+      ms: Date.now() - recoveryStartedAt,
+      graphHash: input.graphHash ?? null,
+      extra: {
+        status: 'ok',
+        available: refill.available,
+        warming: refill.warming,
+        created: refill.created,
+        promoted: refill.promoted,
+        removed: refill.removed,
+        waitedMs: refill.waitedMs,
+        waitIterations: refill.waitIterations,
+      },
+    });
+  }
+
+  let retryCount = 0;
+  const retryStartedAt = Date.now();
+  while (
+    Date.now() - retryStartedAt <
+    WORKFLOW_POOL_MISS_CLAIM_RETRY_TIMEOUT_MS
+  ) {
+    retryCount += 1;
+    pooledInstanceId = await leaseWorkflowPoolId(input.env, input.runId).catch(
+      (error) => {
+        leaseError = error instanceof Error ? error.message : String(error);
+        return null;
+      },
+    );
+    if (pooledInstanceId || leaseError) {
+      break;
+    }
+    missCounts = await workflowPoolCount(input.env).catch(() => missCounts);
+    if (!missCounts || missCounts.available + missCounts.warming <= 0) {
+      break;
+    }
+    await sleep(WORKFLOW_POOL_MISS_CLAIM_RETRY_POLL_MS);
+  }
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_claim_retry',
+    ms: Date.now() - retryStartedAt,
+    graphHash: input.graphHash ?? null,
+    extra: {
+      pooled: Boolean(pooledInstanceId),
+      retries: retryCount,
+      ...(leaseError ? { error: leaseError } : {}),
+      ...(missCounts
+        ? {
+            availableAfterRetry: missCounts.available,
+            warmingAfterRetry: missCounts.warming,
+          }
+        : {}),
+    },
+  });
+
+  return { pooledInstanceId, missCounts, leaseError };
+}
+
 async function mapRunToWorkflowInstance(input: {
   env: CoordinatorEnv;
   runId: string;
@@ -1191,56 +1310,120 @@ async function persistWorkflowRetryState(input: {
   runId: string;
   params: PlayWorkflowParams;
 }): Promise<void> {
-  const retryParams: PlayWorkflowParams = {
-    ...input.params,
-    dynamicWorkerCode: null,
-    contractSnapshot: stripRetrySourceSnapshot(input.params.contractSnapshot),
-    childPlayManifests: stripRetryChildManifestCode(
-      input.params.childPlayManifests,
-    ),
-    packagedFiles:
-      input.params.packagedFiles?.map((file) => ({
-        playPath: file.playPath,
-        storageKey: file.storageKey,
-        contentType: file.contentType,
-        bytes: file.bytes,
-      })) ?? null,
+  const retryParams = buildWorkflowRetryParams(input.params);
+  const paramsBytes = jsonByteLength(retryParams);
+  if (paramsBytes > WORKFLOW_RETRY_PARAMS_MAX_BYTES) {
+    throw new Error(
+      `workflow retry params too large: ${paramsBytes} bytes exceeds ${WORKFLOW_RETRY_PARAMS_MAX_BYTES}. Pass large payloads as staged files or ctx.csv inputs instead of inline JSON.`,
+    );
+  }
+  let body: {
+    runId: string;
+    params?: PlayWorkflowParams;
+    paramsRef?: WorkflowRetryParamsRef;
+    paramsBytes: number;
+    ttlMs: number;
   };
-  await callWorkflowPool<{ ok?: unknown }>(input.env, '/run-retry-state-put', {
-    method: 'POST',
-    body: JSON.stringify({
+  if (paramsBytes > WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES) {
+    const serialized = JSON.stringify(retryParams);
+    const hash = stableHash(serialized);
+    const storageKey = workflowRetryParamsStorageKey({
       runId: input.runId,
-      params: retryParams,
+      hash,
+    });
+    await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
+      httpMetadata: { contentType: 'application/json' },
+    });
+    body = {
+      runId: input.runId,
+      paramsRef: {
+        storageKind: 'r2',
+        storageKey,
+        bytes: paramsBytes,
+        hash,
+        expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
+      },
+      paramsBytes,
       ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
-    }),
-  }).catch((error) => {
-    console.warn('[coordinator] workflow retry state persistence skipped', {
+    };
+  } else {
+    body = {
       runId: input.runId,
-      error: error instanceof Error ? error.message : String(error),
-    });
+      params: retryParams,
+      paramsBytes,
+      ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
+    };
+  }
+  await callWorkflowPool<{ ok?: unknown }>(input.env, '/run-retry-state-put', {
+    method: 'POST',
+    body: JSON.stringify(body),
   });
 }
 
-function stripRetrySourceSnapshot(snapshot: unknown): unknown {
-  if (!isRecord(snapshot)) return snapshot;
-  const rest = { ...snapshot };
-  delete rest.sourceCode;
-  delete rest.sourceFiles;
-  return rest;
+async function hydrateWorkflowRetryParams(input: {
+  env: CoordinatorEnv;
+  params: unknown;
+  paramsRef: unknown;
+}): Promise<PlayWorkflowParams | null> {
+  if (isRecord(input.params)) {
+    return input.params as PlayWorkflowParams;
+  }
+  if (!isRecord(input.paramsRef)) {
+    return null;
+  }
+  const storageKind = input.paramsRef.storageKind;
+  const storageKey = input.paramsRef.storageKey;
+  const expectedBytes = input.paramsRef.bytes;
+  const expectedHash = input.paramsRef.hash;
+  if (
+    storageKind !== 'r2' ||
+    typeof storageKey !== 'string' ||
+    !storageKey.startsWith('plays/workflow-retry-params/') ||
+    typeof expectedBytes !== 'number' ||
+    !Number.isFinite(expectedBytes) ||
+    typeof expectedHash !== 'string' ||
+    !expectedHash
+  ) {
+    throw new Error('Invalid workflow retry params reference.');
+  }
+  const object = await input.env.PLAYS_BUCKET.get(storageKey);
+  if (!object) {
+    throw new Error(`Workflow retry params missing from R2: ${storageKey}`);
+  }
+  const text = await object.text();
+  const actualBytes = new TextEncoder().encode(text).length;
+  if (actualBytes !== expectedBytes) {
+    throw new Error(
+      `Workflow retry params byte length mismatch: expected ${expectedBytes}, got ${actualBytes}.`,
+    );
+  }
+  const actualHash = stableHash(text);
+  if (actualHash !== expectedHash) {
+    throw new Error('Workflow retry params hash mismatch.');
+  }
+  const parsed = JSON.parse(text) as unknown;
+  return isRecord(parsed) ? (parsed as PlayWorkflowParams) : null;
 }
 
-function stripRetryChildManifestCode(
-  manifests: PlayRuntimeManifestMap | null | undefined,
-): PlayRuntimeManifestMap | null {
-  if (!manifests) return null;
-  const stripped: PlayRuntimeManifestMap = {};
-  for (const [key, manifest] of Object.entries(manifests)) {
-    const rest = { ...manifest };
-    delete rest.bundledCode;
-    delete rest.sourceCode;
-    stripped[key] = rest;
-  }
-  return stripped;
+function workflowRetryStatePersistenceErrorResponse(input: {
+  runId: string;
+  error: unknown;
+}): Response {
+  const message =
+    input.error instanceof Error ? input.error.message : String(input.error);
+  return Response.json(
+    {
+      error: {
+        code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
+        message:
+          'Failed to persist workflow retry state before dispatching the play run.',
+        phase: 'coordinator_retry_state_persistence',
+        runId: input.runId,
+        cause: message,
+      },
+    },
+    { status: 503 },
+  );
 }
 
 async function claimWorkflowPlatformRetry(input: {
@@ -1255,6 +1438,7 @@ async function claimWorkflowPlatformRetry(input: {
     claimed?: unknown;
     attempts?: unknown;
     params?: unknown;
+    paramsRef?: unknown;
   }>(input.env, '/run-retry-claim', {
     method: 'POST',
     body: JSON.stringify({
@@ -1262,10 +1446,15 @@ async function claimWorkflowPlatformRetry(input: {
       maxAttempts: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
     }),
   });
+  const params = await hydrateWorkflowRetryParams({
+    env: input.env,
+    params: body.params,
+    paramsRef: body.paramsRef,
+  });
   return {
     claimed: body.claimed === true,
     attempts: typeof body.attempts === 'number' ? body.attempts : 0,
-    params: isRecord(body.params) ? (body.params as PlayWorkflowParams) : null,
+    params,
   };
 }
 
@@ -1719,17 +1908,13 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
   const leaseStartedAt = Date.now();
-  let leaseError: string | null = null;
-  const pooledInstanceId = await leaseWorkflowPoolId(
-    input.env,
-    input.params.runId,
-  ).catch((error) => {
-    leaseError = error instanceof Error ? error.message : String(error);
-    return null;
-  });
-  const missCounts = pooledInstanceId
-    ? null
-    : await workflowPoolCount(input.env).catch(() => null);
+  const { pooledInstanceId, missCounts, leaseError } =
+    await leaseWorkflowPoolIdWithMissRecovery({
+      env: input.env,
+      runId: input.params.runId,
+      recordSubmitTiming: input.recordSubmitTiming,
+      graphHash: input.params.graphHash ?? null,
+    });
   input.recordSubmitTiming({
     phase: 'coordinator.workflow_pool_lease',
     ms: Date.now() - leaseStartedAt,
@@ -1746,30 +1931,6 @@ async function submitViaPooledWorkflow(input: {
     },
   });
 
-  if (!pooledInstanceId) {
-    // A pool miss must not block the user path. Refilling is handled by the
-    // caller's waitUntil after submit, so fall through to cold create now.
-    const counts =
-      missCounts ?? (await workflowPoolCount(input.env).catch(() => null));
-    input.recordSubmitTiming({
-      phase: 'coordinator.workflow_pool_refill_on_miss',
-      ms: 0,
-      graphHash: input.params.graphHash ?? null,
-      extra: {
-        skipped: true,
-        reason: 'pool_miss_does_not_block_submit',
-        ...(counts
-          ? {
-              available: counts.available,
-              warming: counts.warming,
-              waitedMs: 0,
-              waitIterations: 0,
-            }
-          : {}),
-      },
-    });
-  }
-
   if (!pooledInstanceId) {
     return null;
   }
@@ -3606,11 +3767,40 @@ async function handleWorkflowRoute(input: {
       params,
       recordSubmitTiming,
     });
-    await persistWorkflowRetryState({
-      env,
-      runId: submittedRunId,
-      params: workflowParams,
-    });
+    try {
+      const retryStateStartedAt = Date.now();
+      await persistWorkflowRetryState({
+        env,
+        runId: submittedRunId,
+        params: workflowParams,
+      });
+      recordSubmitTiming({
+        phase: 'coordinator.retry_state_persistence',
+        ms: Date.now() - retryStateStartedAt,
+        graphHash: params.graphHash ?? null,
+      });
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      console.error('[coordinator] workflow retry state persistence failed', {
+        code: 'WORKFLOW_RETRY_STATE_PERSISTENCE_FAILED',
+        runId: submittedRunId,
+        error: errorMessage,
+      });
+      recordSubmitTiming({
+        phase: 'coordinator.retry_state_persistence',
+        ms: 0,
+        graphHash: params.graphHash ?? null,
+        extra: {
+          status: 'failed',
+          error: errorMessage,
+        },
+      });
+      return workflowRetryStatePersistenceErrorResponse({
+        runId: submittedRunId,
+        error,
+      });
+    }
     let instance: WorkflowInstance | null = null;
     try {
       const statusEventStartedAt = Date.now();

package/dist/repo/apps/play-runner-workers/src/dedup-do.ts

@@ -113,6 +113,8 @@ type WorkflowRunMapping = {
 type WorkflowRunRetryState = {
   runId: string;
   params: unknown;
+  paramsRef?: unknown;
+  paramsBytes?: number;
   retryAttempts: number;
   updatedAt: number;
   expiresAt: number;
@@ -1059,8 +1061,10 @@ export class PlayDedup implements DurableObject {
       ttlMs?: unknown;
     } | null;
     const runId = typeof body?.runId === 'string' ? body.runId : '';
-    if (!runId || !body || !('params' in body)) {
-      return new Response('runId and params are required', { status: 400 });
+    if (!runId || !body || (!('params' in body) && !('paramsRef' in body))) {
+      return new Response('runId and params or paramsRef are required', {
+        status: 400,
+      });
     }
     const now = Date.now();
     const ttlMs =
@@ -1075,7 +1079,14 @@ export class PlayDedup implements DurableObject {
       const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
       const retryState = {
         runId,
-        params: body.params,
+        params: 'params' in body ? body.params : null,
+        paramsRef: 'paramsRef' in body ? body.paramsRef : null,
+        paramsBytes:
+          typeof (body as { paramsBytes?: unknown }).paramsBytes ===
+            'number' &&
+          Number.isFinite((body as { paramsBytes?: number }).paramsBytes)
+            ? (body as { paramsBytes: number }).paramsBytes
+            : undefined,
         retryAttempts:
           existing?.runId === runId &&
           typeof existing.retryAttempts === 'number'
@@ -1128,6 +1139,8 @@ export class PlayDedup implements DurableObject {
           claimed: false,
           attempts: existing.retryAttempts,
           params: existing.params,
+          paramsRef: existing.paramsRef ?? null,
+          paramsBytes: existing.paramsBytes ?? null,
         };
         return;
       }
@@ -1142,6 +1155,8 @@ export class PlayDedup implements DurableObject {
         claimed: true,
         attempts: nextAttempts,
         params: existing.params,
+        paramsRef: existing.paramsRef ?? null,
+        paramsBytes: existing.paramsBytes ?? null,
       };
     });
     return new Response(JSON.stringify(response), {

package/dist/repo/apps/play-runner-workers/src/workflow-retry-state.ts

@@ -0,0 +1,203 @@
+import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
+import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
+import type {
+  PlayRuntimeManifest,
+  PlayRuntimeManifestMap,
+} from '../../../shared_libs/plays/compiler-manifest';
+
+export const WORKFLOW_RETRY_STATE_TARGET_BYTES = 100_000;
+export const WORKFLOW_RETRY_PARAMS_EXTERNALIZE_AFTER_BYTES =
+  WORKFLOW_RETRY_STATE_TARGET_BYTES;
+export const WORKFLOW_RETRY_PARAMS_MAX_BYTES = 1024 * 1024;
+
+export type WorkflowRetryParamsRef = {
+  storageKind: 'r2';
+  storageKey: string;
+  bytes: number;
+  hash: string;
+  expiresAt: number;
+};
+
+export type WorkflowRetryPlayParams = {
+  runId: string;
+  playId: string;
+  playName: string;
+  artifactStorageKey: string;
+  artifactHash: string;
+  graphHash: string;
+  input: Record<string, unknown>;
+  inputFile?: {
+    name?: string;
+    r2Key?: string;
+    storageKey?: string;
+    path?: string;
+    fileName?: string;
+    logicalPath?: string;
+    contentType?: string;
+    bytes?: number;
+  } | null;
+  inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
+  packagedFiles?: Array<{
+    playPath: string;
+    storageKey: string;
+    contentType?: string;
+    bytes?: number;
+    inlineText?: string;
+  }> | null;
+  contractSnapshot?: unknown;
+  executionPlan?: ExecutionPlan | null;
+  childPlayManifests?: PlayRuntimeManifestMap | null;
+  playCallGovernance?: PlayCallGovernanceSnapshot | null;
+  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
+  preloadedDbSessionRef?: {
+    runId: string;
+    sessionCount: number;
+    expiresAt: number;
+  } | null;
+  dynamicWorkerCode?: string | null;
+  executorToken: string;
+  baseUrl: string;
+  orgId: string;
+  userEmail: string;
+  userId?: string | null;
+  runtimeBackend: string;
+  dedupBackend: string;
+  totalRows?: number;
+  coordinatorUrl?: string | null;
+  coordinatorInternalToken?: string | null;
+};
+
+export type WorkflowRetryStatePayload<TParams = WorkflowRetryPlayParams> =
+  | {
+      params: TParams;
+      paramsRef?: null;
+      paramsBytes: number;
+    }
+  | {
+      params: null;
+      paramsRef: WorkflowRetryParamsRef;
+      paramsBytes: number;
+    };
+
+export function buildWorkflowRetryParams<
+  TParams extends WorkflowRetryPlayParams,
+>(params: TParams): TParams {
+  const retryParams = {
+    ...params,
+    dynamicWorkerCode: null,
+    contractSnapshot: stripRetrySourceSnapshot(params.contractSnapshot),
+    childPlayManifests: stripRetryChildManifestCode(params.childPlayManifests),
+    packagedFiles: stripRetryPackagedFiles(params.packagedFiles),
+  } satisfies WorkflowRetryPlayParams as TParams;
+  if (jsonByteLength(retryParams) <= WORKFLOW_RETRY_STATE_TARGET_BYTES) {
+    return retryParams;
+  }
+  return {
+    ...retryParams,
+    contractSnapshot: stripRetryContractSnapshotToArtifact(
+      retryParams.contractSnapshot,
+      params,
+    ),
+    childPlayManifests: stripRetryChildManifestToArtifact(
+      retryParams.childPlayManifests,
+    ),
+  } satisfies WorkflowRetryPlayParams as TParams;
+}
+
+export function jsonByteLength(value: unknown): number {
+  return new TextEncoder().encode(JSON.stringify(value)).length;
+}
+
+export function workflowRetryParamsStorageKey(input: {
+  runId: string;
+  hash: string;
+}): string {
+  const safeRunId = input.runId
+    .toLowerCase()
+    .replace(/[^a-z0-9_-]+/g, '-')
+    .slice(0, 140);
+  return `plays/workflow-retry-params/${safeRunId || 'run'}/${input.hash}.json`;
+}
+
+function stripRetryPackagedFiles(
+  files: WorkflowRetryPlayParams['packagedFiles'],
+): WorkflowRetryPlayParams['packagedFiles'] {
+  return (
+    files?.map((file) => ({
+      playPath: file.playPath,
+      storageKey: file.storageKey,
+      contentType: file.contentType,
+      bytes: file.bytes,
+    })) ?? null
+  );
+}
+
+function stripRetrySourceSnapshot(snapshot: unknown): unknown {
+  if (!isRecord(snapshot)) return snapshot;
+  const rest = { ...snapshot };
+  delete rest.sourceCode;
+  delete rest.sourceFiles;
+  delete rest.bundledCode;
+  return rest;
+}
+
+function stripRetryContractSnapshotToArtifact(
+  snapshot: unknown,
+  params: WorkflowRetryPlayParams,
+): unknown {
+  if (!isRecord(snapshot)) return snapshot;
+  return {
+    source: snapshot.source ?? 'artifact',
+    revisionVersion: snapshot.revisionVersion ?? null,
+    staticPipeline: snapshot.staticPipeline ?? null,
+    billingLimit: snapshot.billingLimit ?? null,
+    artifactMetadata: {
+      storageKey: params.artifactStorageKey,
+      artifactHash: params.artifactHash,
+      graphHash: params.graphHash,
+    },
+    codeFormat: snapshot.codeFormat ?? null,
+    compatibility: snapshot.compatibility ?? null,
+  };
+}
+
+function stripRetryChildManifestCode(
+  manifests: PlayRuntimeManifestMap | null | undefined,
+): PlayRuntimeManifestMap | null {
+  if (!manifests) return null;
+  const stripped: PlayRuntimeManifestMap = {};
+  for (const [key, manifest] of Object.entries(manifests)) {
+    const rest = { ...manifest };
+    delete rest.bundledCode;
+    delete rest.sourceCode;
+    delete (rest as Record<string, unknown>).sourceMap;
+    stripped[key] = rest;
+  }
+  return stripped;
+}
+
+function stripRetryChildManifestToArtifact(
+  manifests: PlayRuntimeManifestMap | null | undefined,
+): PlayRuntimeManifestMap | null {
+  if (!manifests) return null;
+  const stripped: PlayRuntimeManifestMap = {};
+  for (const [key, manifest] of Object.entries(manifests)) {
+    stripped[key] = {
+      playName: manifest.playName,
+      graphHash: manifest.graphHash,
+      artifactStorageKey: manifest.artifactStorageKey,
+      artifactHash: manifest.artifactHash,
+      staticPipelineHash: manifest.staticPipelineHash,
+      staticPipeline: manifest.staticPipeline,
+      compiledAt: manifest.compiledAt,
+      compilerVersion: manifest.compilerVersion,
+      maxCreditsPerRun: manifest.maxCreditsPerRun,
+    };
+  }
+  return stripped;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}

package/dist/repo/sdk/src/release.ts

@@ -50,10 +50,10 @@ export type SdkRelease = {
 };
 
 export const SDK_RELEASE = {
-  version: '0.1.78',
+  version: '0.1.79',
   apiContract: '2026-06-dataset-column-cell-stale-hard-cutover',
   supportPolicy: {
-    latest: '0.1.78',
+    latest: '0.1.79',
     minimumSupported: '0.1.53',
     deprecatedBelow: '0.1.53',
   },

package/dist/repo/shared_libs/plays/static-pipeline.ts

@@ -3,6 +3,7 @@ import { normalizeTableNamespace } from './row-identity';
 export interface PlayStaticPipeline {
   tableNamespace?: string;
   inputFields?: string[];
+  rowKeyFields?: string[];
   csvArg?: string;
   hasInlineData?: boolean;
   csvDescription?: string;
@@ -32,6 +33,7 @@ export interface PlaySheetColumnContract {
   outputSqlName?: string;
   stepId?: string;
   toolId?: string;
+  isRowKey?: boolean;
 }
 
 export interface PlaySheetContract {
@@ -39,6 +41,40 @@ export interface PlaySheetContract {
   columns: PlaySheetColumnContract[];
 }
 
+export type PlayStaticColumnProducerKind =
+  | 'tool'
+  | 'waterfall'
+  | 'stepProgram'
+  | 'playCall'
+  | 'transform';
+
+export interface PlayStaticColumnProducer {
+  id: string;
+  kind: PlayStaticColumnProducerKind;
+  field: string;
+  toolId?: string;
+  playId?: string;
+  conditional?: boolean;
+  sourceRange?: PlayStaticSourceRange;
+  steps?: PlayStaticColumnProducer[];
+  substep: PlayStaticSubstep;
+}
+
+export interface PlayStaticDatasetColumn {
+  id: string;
+  source: PlaySheetColumnSource;
+  sqlName?: string;
+  producers: PlayStaticColumnProducer[];
+}
+
+export interface PlayCompiledStaticGraph {
+  topLevel: PlayStaticSubstep[];
+  datasets: Array<{
+    tableNamespace: string;
+    columns: PlayStaticDatasetColumn[];
+  }>;
+}
+
 export function ensureCompiledSheetContract(
   pipeline: PlayStaticPipeline | null | undefined,
 ): PlayStaticPipeline | null | undefined {
@@ -92,8 +128,27 @@ function truncateStaticSubstepsForStorage(
       return {
         ...base,
         inputFields: base.inputFields ? [...base.inputFields] : undefined,
+        rowKeyFields: base.rowKeyFields ? [...base.rowKeyFields] : undefined,
         outputFields: base.outputFields ? [...base.outputFields] : undefined,
+        columns: base.columns
+          ? base.columns.map((column) => ({
+              ...column,
+              producers: column.producers.map((producer) => ({
+                ...producer,
+                sourceRange: cloneStorageSafeSourceRange(producer.sourceRange),
+                steps: producer.steps
+                  ? producer.steps.map((stepProducer) => ({
+                      ...stepProducer,
+                      sourceRange: cloneStorageSafeSourceRange(
+                        stepProducer.sourceRange,
+                      ),
+                    }))
+                  : undefined,
+              })),
+            }))
+          : undefined,
         waterfallIds: base.waterfallIds ? [...base.waterfallIds] : undefined,
+        steps: truncateStaticSubstepsForStorage(base.steps, input),
         sheetContract: cloneStorageSafeSheetContract(base.sheetContract),
       };
     }
@@ -153,6 +208,9 @@ export function truncateStaticPipelineForStorage(
   return {
     ...pipeline,
     inputFields: pipeline.inputFields ? [...pipeline.inputFields] : undefined,
+    rowKeyFields: pipeline.rowKeyFields
+      ? [...pipeline.rowKeyFields]
+      : undefined,
     fields: [...(pipeline.fields ?? [])],
     stages: truncateStaticSubstepsForStorage(pipeline.stages, {
       embeddedPlayCallPipelineDepth,
@@ -198,8 +256,11 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         name?: string;
         tableNamespace?: string;
         inputFields?: string[];
+        rowKeyFields?: string[];
         outputFields?: string[];
+        columns?: PlayStaticDatasetColumn[];
         waterfallIds?: string[];
+        steps?: PlayStaticSubstep[];
         sheetContract?: PlaySheetContract | null;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
@@ -210,6 +271,8 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         type: 'tool';
         toolId: string;
         field: string;
+        paramsSource?: string;
+        sourceText?: string;
         description?: string;
         inLoop?: boolean;
         isEventWait?: boolean;
@@ -263,6 +326,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
     | {
         type: 'run_javascript';
         alias: string;
+        sourceText?: string;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
         callDepth?: number;
@@ -271,6 +335,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
     | {
         type: 'code';
         field: string;
+        sourceText?: string;
         description?: string;
         sourceRange?: PlayStaticSourceRange;
         callDepth?: number;
@@ -289,6 +354,12 @@ export function getCompiledPipelineSubsteps(
 
 export function getTopLevelPipelineSubsteps(
   pipeline: PlayStaticPipeline | null | undefined,
+): PlayStaticSubstep[] {
+  return compileStaticGraph(pipeline).topLevel;
+}
+
+function getRawTopLevelPipelineSubsteps(
+  pipeline: PlayStaticPipeline | null | undefined,
 ): PlayStaticSubstep[] {
   if (!pipeline) {
     return [];
@@ -328,6 +399,14 @@ export function flattenStaticSubsteps(
 
   for (const substep of substeps) {
     flattened.push(substep);
+    if (substep.type === 'dataset' && substep.steps?.length) {
+      flattened.push(...flattenStaticSubsteps(substep.steps));
+      continue;
+    }
+    if (substep.type === 'step_suite') {
+      flattened.push(...flattenStaticSubsteps(substep.steps));
+      continue;
+    }
     if (substep.type === 'play_call' && substep.pipeline) {
       const nestedSubsteps = getCompiledPipelineSubsteps(substep.pipeline);
       if (nestedSubsteps.length > 0) {
@@ -345,6 +424,137 @@ export function flattenStaticPipeline(
   return flattenStaticSubsteps(getCompiledPipelineSubsteps(pipeline));
 }
 
+export function compileStaticGraph(
+  pipeline: PlayStaticPipeline | null | undefined,
+): PlayCompiledStaticGraph {
+  const rawTopLevel = getRawTopLevelPipelineSubsteps(pipeline);
+  const datasets: PlayCompiledStaticGraph['datasets'] = [];
+  const topLevel = rawTopLevel.map((substep) => {
+    if (substep.type !== 'dataset') {
+      return substep;
+    }
+    const columns = compileDatasetColumns(substep);
+    const tableNamespace = (substep.tableNamespace ?? substep.field).trim();
+    if (tableNamespace) {
+      datasets.push({ tableNamespace, columns });
+    }
+    return {
+      ...substep,
+      columns,
+    } satisfies PlayStaticSubstep;
+  });
+  return { topLevel, datasets };
+}
+
+function compileDatasetColumns(
+  dataset: Extract<PlayStaticSubstep, { type: 'dataset' }>,
+): PlayStaticDatasetColumn[] {
+  const columnsById = new Map<string, PlayStaticDatasetColumn>();
+  const ensureColumn = (
+    id: string,
+    source: PlaySheetColumnSource,
+    sqlName?: string,
+  ) => {
+    const trimmed = id.trim();
+    if (!trimmed) return null;
+    const existing = columnsById.get(trimmed);
+    if (existing) {
+      if (!existing.sqlName && sqlName) existing.sqlName = sqlName;
+      return existing;
+    }
+    const column: PlayStaticDatasetColumn = {
+      id: trimmed,
+      source,
+      ...(sqlName ? { sqlName } : {}),
+      producers: [],
+    };
+    columnsById.set(trimmed, column);
+    return column;
+  };
+
+  for (const column of dataset.sheetContract?.columns ?? []) {
+    ensureColumn(column.id, column.source, column.sqlName);
+  }
+  for (const field of dataset.inputFields ?? []) {
+    ensureColumn(field, 'input', sqlSafePlayColumnName(field));
+  }
+  for (const field of dataset.outputFields ?? []) {
+    ensureColumn(field, 'datasetColumn', sqlSafePlayColumnName(field));
+  }
+
+  for (const substep of dataset.steps ?? []) {
+    const field = fieldForColumnProducer(substep);
+    if (!field) continue;
+    const column = ensureColumn(
+      field,
+      'datasetColumn',
+      sqlSafePlayColumnName(field),
+    );
+    if (!column) continue;
+    column.producers.push(columnProducerFromSubstep(substep, field));
+  }
+
+  return [...columnsById.values()];
+}
+
+function fieldForColumnProducer(substep: PlayStaticSubstep): string | null {
+  if ('field' in substep && typeof substep.field === 'string') {
+    return substep.field.trim() || null;
+  }
+  if (substep.type === 'run_javascript') {
+    return substep.alias.trim() || null;
+  }
+  return null;
+}
+
+function columnProducerFromSubstep(
+  substep: PlayStaticSubstep,
+  field: string,
+): PlayStaticColumnProducer {
+  const steps =
+    substep.type === 'step_suite'
+      ? substep.steps
+          .map((step) => {
+            const stepField = fieldForColumnProducer(step) ?? field;
+            return columnProducerFromSubstep(step, stepField);
+          })
+          .filter((producer) => producer.field.trim())
+      : undefined;
+  const kind: PlayStaticColumnProducerKind =
+    substep.type === 'tool'
+      ? 'tool'
+      : substep.type === 'waterfall'
+        ? 'waterfall'
+        : substep.type === 'step_suite'
+          ? 'stepProgram'
+          : substep.type === 'play_call'
+            ? 'playCall'
+            : 'transform';
+  return {
+    id: producerId(substep, field),
+    kind,
+    field,
+    ...(substep.type === 'tool' ? { toolId: substep.toolId } : {}),
+    ...(substep.type === 'play_call' ? { playId: substep.playId } : {}),
+    ...(substep.conditional ? { conditional: true } : {}),
+    ...(substep.sourceRange ? { sourceRange: substep.sourceRange } : {}),
+    ...(steps && steps.length > 0 ? { steps } : {}),
+    substep,
+  };
+}
+
+function producerId(substep: PlayStaticSubstep, field: string): string {
+  if (substep.type === 'tool') return `${field}:tool:${substep.toolId}`;
+  if (substep.type === 'waterfall') {
+    return `${field}:waterfall:${substep.id ?? substep.tool ?? 'unknown'}`;
+  }
+  if (substep.type === 'play_call') return `${field}:play:${substep.playId}`;
+  if (substep.type === 'step_suite') return `${field}:steps`;
+  if (substep.type === 'run_javascript')
+    return `${field}:transform:${substep.alias}`;
+  return `${field}:transform`;
+}
+
 export function resolveSheetContractForTableNamespace(
   pipeline: PlayStaticPipeline | null | undefined,
   tableNamespace: string | null | undefined,
@@ -400,6 +610,50 @@ export function resolveSheetContractForTableNamespace(
   return resolveFromPipeline(pipeline);
 }
 
+export function resolveStaticDatasetColumnsForTableNamespace(
+  pipeline: PlayStaticPipeline | null | undefined,
+  tableNamespace: string | null | undefined,
+): PlayStaticDatasetColumn[] {
+  const requestedNamespace = tableNamespace?.trim();
+  if (!pipeline || !requestedNamespace) {
+    return [];
+  }
+  const normalizedNamespace = normalizeTableNamespace(requestedNamespace);
+  const seen = new Set<PlayStaticPipeline>();
+
+  const resolveFromPipeline = (
+    currentPipeline: PlayStaticPipeline | null | undefined,
+  ): PlayStaticDatasetColumn[] | null => {
+    if (!currentPipeline || seen.has(currentPipeline)) {
+      return null;
+    }
+    seen.add(currentPipeline);
+
+    const compiled = compileStaticGraph(currentPipeline);
+    const matchingDataset = compiled.datasets.find(
+      (dataset) =>
+        normalizeTableNamespace(dataset.tableNamespace) ===
+        normalizedNamespace,
+    );
+    if (matchingDataset) {
+      return matchingDataset.columns;
+    }
+
+    for (const substep of getCompiledPipelineSubsteps(currentPipeline)) {
+      if (substep.type === 'play_call') {
+        const nestedColumns = resolveFromPipeline(substep.pipeline);
+        if (nestedColumns) {
+          return nestedColumns;
+        }
+      }
+    }
+
+    return null;
+  };
+
+  return resolveFromPipeline(pipeline) ?? [];
+}
+
 export function sqlSafePlayColumnName(id: string): string {
   const normalized = id
     .trim()
@@ -441,8 +695,14 @@ export function compileSheetContract(pipeline: PlayStaticPipeline): {
   const inputFields = pipeline.inputFields?.length
     ? pipeline.inputFields
     : [tableNamespace];
+  const rowKeyFieldSet = new Set(pipeline.rowKeyFields ?? []);
   for (const inputField of inputFields) {
-    addColumn({ id: inputField, source: 'input', field: inputField });
+    addColumn({
+      id: inputField,
+      source: 'input',
+      field: inputField,
+      ...(rowKeyFieldSet.has(inputField) ? { isRowKey: true } : {}),
+    });
   }
 
   for (const field of pipeline.fields) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "deepline",
-  "version": "0.1.78",
+  "version": "0.1.79",
   "description": "Deepline SDK + CLI — B2B data enrichment powered by durable cloud execution",
   "license": "MIT",
   "repository": {