deepline 0.1.182 → 0.1.184

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -22,7 +22,10 @@ import {
   type WorkflowRunner,
 } from '@cloudflare/dynamic-workflows';
 import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
+import { buildChildRunId as buildSharedChildRunId } from '../../../shared_libs/play-runtime/child-run-id';
 import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
+import type { RuntimeTestPolicyOverrides } from '../../../shared_libs/play-runtime/test-runtime-seams';
+import { DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION } from '../../../shared_libs/play-runtime/dynamic-worker-version';
 import {
   DB_SESSION_DEFAULT_TTL_SECONDS,
   createDbSessionResponseSchema,
@@ -54,6 +57,15 @@ import {
   COORDINATOR_INTERNAL_TOKEN_HEADER,
   COORDINATOR_RUN_SCOPE_HEADER,
 } from '../../../shared_libs/play-runtime/coordinator-headers';
+import {
+  PLAY_RUNTIME_CONTRACT,
+  PLAY_RUNTIME_CONTRACT_HEADER,
+} from '../../../shared_libs/play-runtime/runtime-contract';
+import {
+  PLAY_RUNTIME_API_COMPAT_PATH,
+  PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
+  PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH,
+} from '../../../shared_libs/play-runtime/runtime-api-paths';
 import {
   decideWorkflowPlatformRetry,
   PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
@@ -85,6 +97,8 @@ export type PlayWorkflowParams = {
   artifactHash: string;
   graphHash: string;
   input: Record<string, unknown>;
+  force?: boolean;
+  runAttempt?: number;
   inputFile?: {
     name?: string;
     r2Key?: string;
@@ -126,6 +140,8 @@ export type PlayWorkflowParams = {
   totalRows?: number;
   coordinatorUrl?: string | null;
   coordinatorInternalToken?: string | null;
+  runtimeTestFaultHeader?: string | null;
+  testPolicyOverrides?: RuntimeTestPolicyOverrides | null;
   submittedAt?: number | null;
 };
 
@@ -649,6 +665,9 @@ const SUBMIT_INITIAL_STATE_POLL_MS = 50;
 const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
 const WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS = 55_000;
 const WORKFLOW_SUBMIT_DUPLICATE_POLL_MS = 250;
+const CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS = [
+  250, 750, 1500, 3000, 5000, 10000,
+] as const;
 const COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE =
   'Worker memory limit hit before the run could finish. For CSV enrich, use --output or --in-place for automatic batches; otherwise rerun smaller --rows ranges.';
 
@@ -668,7 +687,7 @@ function buildDynamicWorkflowMetadata(
     graphHash: params.graphHash,
     artifactStorageKey: params.artifactStorageKey,
     artifactHash: params.artifactHash ?? null,
-    dynamicWorkerCode: params.dynamicWorkerCode ?? null,
+    dynamicWorkerCode: null,
     integrationMode: normalizeIntegrationMode(params.integrationMode),
     packagedFiles: normalizePackagedFiles(params.packagedFiles),
   };
@@ -746,10 +765,11 @@ async function createDynamicWorkflowInstance(input: {
   env: CoordinatorEnv;
   id: string;
   params: PlayWorkflowParams;
+  metadataParams?: PlayWorkflowParams;
 }): Promise<WorkflowInstance> {
   const workflow = createDispatcherWorkflowBinding(
     () => input.env.PLAY_WORKFLOW as unknown as Workflow,
-    buildDynamicWorkflowMetadata(input.params),
+    buildDynamicWorkflowMetadata(input.metadataParams ?? input.params),
   );
   return workflow.create({
     id: input.id,
@@ -1127,6 +1147,96 @@ async function hydrateWorkflowDbSessions(input: {
   };
 }
 
+async function hydrateWorkflowLaunchParams(input: {
+  env: CoordinatorEnv;
+  event: unknown;
+  trace: CoordinatorPerfTraceSink;
+}): Promise<unknown> {
+  if (!isRecord(input.event)) return input.event;
+  const payload = isRecord(input.event.payload) ? input.event.payload : null;
+  const envelopeParams = isRecord(payload?.params) ? payload.params : null;
+  const params = envelopeParams ?? payload;
+  if (
+    !payload ||
+    !params ||
+    (!isRecord(params.workflowParams) && !isRecord(params.workflowParamsRef))
+  ) {
+    return input.event;
+  }
+  const startedAt = Date.now();
+  const hydrated = await hydrateWorkflowRetryParams({
+    env: input.env,
+    params: params.workflowParams,
+    paramsRef: params.workflowParamsRef,
+  });
+  if (!hydrated) return input.event;
+  const submittedAt =
+    typeof params.submittedAt === 'number' &&
+    Number.isFinite(params.submittedAt)
+      ? params.submittedAt
+      : hydrated.submittedAt;
+  const hydratedParams = {
+    ...hydrated,
+    submittedAt,
+  };
+  input.trace({
+    runId:
+      typeof hydratedParams.runId === 'string'
+        ? hydratedParams.runId
+        : 'unknown-workflow-run',
+    phase: 'coordinator.workflow_launch_params_hydrated',
+    ms: Date.now() - startedAt,
+    graphHash:
+      typeof hydratedParams.graphHash === 'string'
+        ? hydratedParams.graphHash
+        : null,
+    extra: {
+      paramsBytes: isRecord(params.workflowParamsRef)
+        ? typeof params.workflowParamsRef.bytes === 'number'
+          ? params.workflowParamsRef.bytes
+          : null
+        : jsonByteLength(params.workflowParams),
+      paramsStorage: isRecord(params.workflowParamsRef) ? 'r2' : 'inline',
+    },
+  });
+  return {
+    ...input.event,
+    payload: envelopeParams
+      ? {
+          ...payload,
+          params: hydratedParams,
+        }
+      : {
+          ...payload,
+          ...hydratedParams,
+        },
+  };
+}
+
+function workflowCreateParams(
+  params: PlayWorkflowParams,
+  launchState: {
+    params?: PlayWorkflowParams;
+    paramsRef?: WorkflowRetryParamsRef;
+  },
+): PlayWorkflowParams {
+  const launchParams = launchState.params ?? params;
+  const envelope: Record<string, unknown> = {
+    runId: params.runId,
+    playName: params.playName,
+    graphHash: params.graphHash,
+    artifactStorageKey: params.artifactStorageKey,
+    artifactHash: params.artifactHash,
+    submittedAt: params.submittedAt,
+  };
+  if (launchState.paramsRef) {
+    envelope.workflowParamsRef = launchState.paramsRef;
+  } else {
+    envelope.workflowParams = launchParams;
+  }
+  return envelope as unknown as PlayWorkflowParams;
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -1209,7 +1319,7 @@ async function markWorkflowRuntimeFailure(input: {
       } satisfies PlayRunLedgerEvent,
     ],
   });
-  const url = `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`;
+  const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
   const backoffMs = [200, 500, 1500];
   let lastError: unknown = null;
   for (let attempt = 0; attempt <= backoffMs.length; attempt += 1) {
@@ -1240,6 +1350,47 @@ async function markWorkflowRuntimeFailure(input: {
   });
 }
 
+async function appendWorkflowChildCompletedRunEvent(input: {
+  env: CoordinatorEnv;
+  event: unknown;
+  result: unknown;
+}): Promise<boolean> {
+  const payload = readWorkflowPayload(input.event);
+  if (!payload || !isRecord(payload.playCallGovernance)) return false;
+  const runId = typeof payload.runId === 'string' ? payload.runId : null;
+  const baseUrl = typeof payload.baseUrl === 'string' ? payload.baseUrl : null;
+  const executorToken =
+    typeof payload.executorToken === 'string' ? payload.executorToken : null;
+  if (!runId || !baseUrl || !executorToken) return false;
+  const headers = new Headers({
+    authorization: `Bearer ${executorToken}`,
+    'content-type': 'application/json',
+  });
+  const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
+  if (bypass) headers.set('x-vercel-protection-bypass', bypass);
+  const body = JSON.stringify({
+    action: 'append_run_events',
+    playId: runId,
+    events: [
+      {
+        type: 'run.completed',
+        runId,
+        source: 'worker',
+        occurredAt: Date.now(),
+        result: input.result,
+      } satisfies PlayRunLedgerEvent,
+    ],
+  });
+  const url = `${baseUrl.replace(/\/$/, '')}${PLAY_RUNTIME_API_COMPAT_PATH}`;
+  const response = await fetch(url, { method: 'POST', headers, body });
+  if (!response.ok) {
+    throw new Error(
+      `runtime API responded ${response.status}: ${(await response.text().catch(() => '')).slice(0, 400)}`,
+    );
+  }
+  return true;
+}
+
 type StoredPlayArtifactPayload = {
   artifact?: {
     bundledCode?: string;
@@ -1253,7 +1404,10 @@ async function persistWorkflowRetryState(input: {
   env: CoordinatorEnv;
   runId: string;
   params: PlayWorkflowParams;
-}): Promise<void> {
+}): Promise<{
+  params?: PlayWorkflowParams;
+  paramsRef?: WorkflowRetryParamsRef;
+}> {
   const retryParams = buildWorkflowRetryParams(input.params);
   const paramsBytes = jsonByteLength(retryParams);
   if (paramsBytes > WORKFLOW_RETRY_PARAMS_MAX_BYTES) {
@@ -1275,21 +1429,32 @@ async function persistWorkflowRetryState(input: {
       runId: input.runId,
       hash,
     });
+    const paramsRef: WorkflowRetryParamsRef = {
+      storageKind: 'r2',
+      storageKey,
+      bytes: paramsBytes,
+      hash,
+      expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
+    };
     await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
       httpMetadata: { contentType: 'application/json' },
     });
     body = {
       runId: input.runId,
-      paramsRef: {
-        storageKind: 'r2',
-        storageKey,
-        bytes: paramsBytes,
-        hash,
-        expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
-      },
+      paramsRef,
       paramsBytes,
       ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
     };
+    await callRunScopedControl<{ ok?: unknown }>(
+      input.env,
+      input.runId,
+      '/run-retry-state-put',
+      {
+        method: 'POST',
+        body: JSON.stringify(body),
+      },
+    );
+    return { paramsRef };
   } else {
     body = {
       runId: input.runId,
@@ -1307,6 +1472,7 @@ async function persistWorkflowRetryState(input: {
       body: JSON.stringify(body),
     },
   );
+  return { params: retryParams };
 }
 
 async function persistWorkflowLaunchState(input: {
@@ -1318,14 +1484,16 @@ async function persistWorkflowLaunchState(input: {
   retryExpiresAt?: number;
   dbSessionsExpiresAt?: number;
   sessionCount?: number;
+  params?: PlayWorkflowParams;
+  paramsRef?: WorkflowRetryParamsRef;
 }> {
   if (input.sessions.length === 0) {
-    await persistWorkflowRetryState({
+    const state = await persistWorkflowRetryState({
       env: input.env,
       runId: input.runId,
       params: input.params,
     });
-    return {};
+    return state;
   }
   const retryParams = buildWorkflowRetryParams(input.params);
   const paramsBytes = jsonByteLength(retryParams);
@@ -1345,18 +1513,19 @@ async function persistWorkflowLaunchState(input: {
       runId: input.runId,
       hash,
     });
+    const paramsRef: WorkflowRetryParamsRef = {
+      storageKind: 'r2',
+      storageKey,
+      bytes: paramsBytes,
+      hash,
+      expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
+    };
     await input.env.PLAYS_BUCKET.put(storageKey, serialized, {
       httpMetadata: { contentType: 'application/json' },
     });
     body = {
       runId: input.runId,
-      paramsRef: {
-        storageKind: 'r2',
-        storageKey,
-        bytes: paramsBytes,
-        hash,
-        expiresAt: Date.now() + WORKFLOW_RETRY_STATE_TTL_MS,
-      },
+      paramsRef,
       paramsBytes,
       sessions: input.sessions,
       retryTtlMs: WORKFLOW_RETRY_STATE_TTL_MS,
@@ -1394,24 +1563,22 @@ async function persistWorkflowLaunchState(input: {
       typeof response.sessionCount === 'number'
         ? response.sessionCount
         : undefined,
+    params: body.params,
+    paramsRef: body.paramsRef,
   };
 }
 
-async function hydrateWorkflowRetryParams(input: {
-  env: CoordinatorEnv;
-  params: unknown;
-  paramsRef: unknown;
-}): Promise<PlayWorkflowParams | null> {
-  if (isRecord(input.params)) {
-    return input.params as PlayWorkflowParams;
-  }
-  if (!isRecord(input.paramsRef)) {
+function parseWorkflowRetryParamsRef(
+  paramsRef: unknown,
+): WorkflowRetryParamsRef | null {
+  if (!isRecord(paramsRef)) {
     return null;
   }
-  const storageKind = input.paramsRef.storageKind;
-  const storageKey = input.paramsRef.storageKey;
-  const expectedBytes = input.paramsRef.bytes;
-  const expectedHash = input.paramsRef.hash;
+  const storageKind = paramsRef.storageKind;
+  const storageKey = paramsRef.storageKey;
+  const expectedBytes = paramsRef.bytes;
+  const expectedHash = paramsRef.hash;
+  const expiresAt = paramsRef.expiresAt;
   if (
     storageKind !== 'r2' ||
     typeof storageKey !== 'string' ||
@@ -1419,10 +1586,37 @@ async function hydrateWorkflowRetryParams(input: {
     typeof expectedBytes !== 'number' ||
     !Number.isFinite(expectedBytes) ||
     typeof expectedHash !== 'string' ||
-    !expectedHash
+    !expectedHash ||
+    typeof expiresAt !== 'number' ||
+    !Number.isFinite(expiresAt)
   ) {
     throw new Error('Invalid workflow retry params reference.');
   }
+  return {
+    storageKind,
+    storageKey,
+    bytes: expectedBytes,
+    hash: expectedHash,
+    expiresAt,
+  };
+}
+
+async function hydrateWorkflowRetryParams(input: {
+  env: CoordinatorEnv;
+  params: unknown;
+  paramsRef: unknown;
+}): Promise<PlayWorkflowParams | null> {
+  if (isRecord(input.params)) {
+    return input.params as PlayWorkflowParams;
+  }
+  const paramsRef = parseWorkflowRetryParamsRef(input.paramsRef);
+  if (!paramsRef) return null;
+  const storageKey = paramsRef.storageKey;
+  const expectedBytes = paramsRef.bytes;
+  const expectedHash = paramsRef.hash;
+  if (paramsRef.expiresAt <= Date.now()) {
+    throw new Error('Workflow retry params reference expired.');
+  }
   const object = await input.env.PLAYS_BUCKET.get(storageKey);
   if (!object) {
     throw new Error(`Workflow retry params missing from R2: ${storageKey}`);
@@ -1470,6 +1664,7 @@ async function claimWorkflowPlatformRetry(input: {
   claimed: boolean;
   attempts: number;
   params: PlayWorkflowParams | null;
+  paramsRef?: WorkflowRetryParamsRef;
 }> {
   const body = await callRunScopedControl<{
     claimed?: unknown;
@@ -1488,10 +1683,12 @@ async function claimWorkflowPlatformRetry(input: {
     params: body.params,
     paramsRef: body.paramsRef,
   });
+  const paramsRef = parseWorkflowRetryParamsRef(body.paramsRef) ?? undefined;
   return {
     claimed: body.claimed === true,
     attempts: typeof body.attempts === 'number' ? body.attempts : 0,
     params,
+    paramsRef,
   };
 }
 
@@ -1502,6 +1699,12 @@ function workflowRetryInstanceId(runId: string, attempt: number): string {
   return `${base.slice(0, maxBaseLength)}-${suffix}`;
 }
 
+function normalizeWorkflowRunAttempt(value: unknown): number {
+  return typeof value === 'number' && Number.isFinite(value)
+    ? Math.max(0, Math.floor(value))
+    : 0;
+}
+
 async function restartWorkflowAfterPlatformReset(input: {
   env: CoordinatorEnv;
   ctx?: ExecutionContext;
@@ -1537,11 +1740,16 @@ async function restartWorkflowAfterPlatformReset(input: {
   const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
   const retryStartedAt = Date.now();
   let retryInstance: WorkflowInstance | null = null;
+  const retryParams = { ...claim.params, runAttempt: claim.attempts };
   try {
     retryInstance = await createDynamicWorkflowInstance({
       env: input.env,
       id: retryInstanceId,
-      params: claim.params,
+      params: workflowCreateParams(retryParams, {
+        params: retryParams,
+        paramsRef: claim.paramsRef,
+      }),
+      metadataParams: retryParams,
     });
     await recordWorkflowInstanceId({
       env: input.env,
@@ -1591,10 +1799,14 @@ async function mintChildWorkflowExecutorToken(input: {
   maxCreditsPerRun?: number | null;
 }): Promise<string> {
   const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.parentExecutorToken,
     baseUrl: input.baseUrl,
-    path: '/api/v2/plays/internal/child-executor-token',
-    headers: { 'x-deepline-request-id': crypto.randomUUID() },
+    path: PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
+    headers: {
+      'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    },
     timeoutMs: 15_000,
     body: {
       parentRunId: input.parentRunId,
@@ -1703,10 +1915,14 @@ async function createChildRuntimeDbSession(input: {
 }): Promise<CreateDbSessionResponse> {
   const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
   const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.childExecutorToken,
     baseUrl: input.baseUrl,
-    path: '/api/v2/plays/internal/runtime',
-    headers: { 'x-deepline-request-id': crypto.randomUUID() },
+    path: PLAY_RUNTIME_API_COMPAT_PATH,
+    headers: {
+      'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    },
     timeoutMs: 15_000,
     body: {
       action: 'create_db_session',
@@ -1812,10 +2028,14 @@ async function registerChildRunWithRuntime(input: {
   executionProfile: string;
 }): Promise<void> {
   const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.childExecutorToken,
     baseUrl: input.baseUrl,
-    path: '/api/v2/plays/internal/runtime',
-    headers: { 'x-deepline-request-id': crypto.randomUUID() },
+    path: PLAY_RUNTIME_API_COMPAT_PATH,
+    headers: {
+      'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    },
     timeoutMs: 15_000,
     body: {
       action: 'start_inline_child_run',
@@ -1858,10 +2078,14 @@ async function markRegisteredChildRunFailed(input: {
   const message =
     input.error instanceof Error ? input.error.message : String(input.error);
   const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.childExecutorToken,
     baseUrl: input.baseUrl,
-    path: '/api/v2/plays/internal/runtime',
-    headers: { 'x-deepline-request-id': crypto.randomUUID() },
+    path: PLAY_RUNTIME_API_COMPAT_PATH,
+    headers: {
+      'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    },
     timeoutMs: 15_000,
     body: {
       action: 'append_run_events',
@@ -1885,6 +2109,55 @@ async function markRegisteredChildRunFailed(input: {
   }
 }
 
+async function appendRegisteredChildRunTerminal(input: {
+  env: CoordinatorEnv;
+  baseUrl: string;
+  childExecutorToken: string;
+  childRunId: string;
+  status: 'completed' | 'failed';
+  result?: unknown;
+  error?: string | null;
+}): Promise<void> {
+  const event =
+    input.status === 'completed'
+      ? ({
+          type: 'run.completed',
+          runId: input.childRunId,
+          source: 'worker',
+          occurredAt: Date.now(),
+          ...(input.result !== undefined ? { result: input.result } : {}),
+        } satisfies PlayRunLedgerEvent)
+      : ({
+          type: 'run.failed',
+          runId: input.childRunId,
+          source: 'worker',
+          occurredAt: Date.now(),
+          error: input.error ?? 'Inline child Worker failed.',
+        } satisfies PlayRunLedgerEvent);
+  const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
+    executorToken: input.childExecutorToken,
+    baseUrl: input.baseUrl,
+    path: PLAY_RUNTIME_API_COMPAT_PATH,
+    headers: {
+      'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    },
+    timeoutMs: 15_000,
+    body: {
+      action: 'append_run_events',
+      playId: input.childRunId,
+      events: [event],
+    },
+  });
+  if (response.status < 200 || response.status >= 300) {
+    const text = response.body ?? '';
+    throw new Error(
+      `Inline child terminal append failed ${response.status}: ${text.slice(0, 800)}`,
+    );
+  }
+}
+
 type CoordinatorRuntimeApiTiming = {
   phase: string;
   ms: number;
@@ -1924,12 +2197,14 @@ async function callRuntimeApiFromCoordinator(input: {
 
   const fetchStartedAt = Date.now();
   const response = await input.env.HARNESS.runtimeApiCall({
+    contract: PLAY_RUNTIME_CONTRACT,
     executorToken: input.executorToken,
     baseUrl: input.baseUrl,
-    path: '/api/v2/plays/internal/runtime',
+    path: PLAY_RUNTIME_API_COMPAT_PATH,
     body,
     headers: {
       'x-deepline-request-id': crypto.randomUUID(),
+      [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
     },
   });
   recordTiming('coordinator.runtime_api.fetch', fetchStartedAt);
@@ -2022,19 +2297,38 @@ async function prepareInlineChildRunWithRuntime(input: {
   };
 }
 
-function buildChildRunId(playName: string): string {
-  const slug =
-    playName
-      .toLowerCase()
-      .replace(/[^a-z0-9-]/g, '-')
-      .replace(/-+/g, '-')
-      .replace(/^-+|-+$/g, '')
-      .slice(0, 64) || 'play';
-  return `play/${slug}/run/${new Date()
-    .toISOString()
-    .replace(/[-:.]/g, '')
-    .replace('T', 't')
-    .replace('Z', '')}-${crypto.randomUUID().slice(0, 8)}`;
+function buildChildRunId(input: {
+  playName: string;
+  parentRunId: string;
+  body: Record<string, unknown>;
+}): string {
+  const { playName, parentRunId, body } = input;
+  const governance = isRecord(body.internalRunPlay)
+    ? body.internalRunPlay
+    : null;
+  // Adapt the coordinator's on-the-wire child-submit body to the shared
+  // derivation. Output stays byte-identical to the previous inline
+  // implementation (same fields → same digest → same run id), so child-submit
+  // idempotency is preserved. See shared_libs/play-runtime/child-run-id.ts.
+  return buildSharedChildRunId({
+    childPlayName: playName,
+    parentRunId,
+    explicitIdempotencyKey:
+      typeof body.childIdempotencyKey === 'string'
+        ? body.childIdempotencyKey
+        : null,
+    parentPlayName:
+      typeof governance?.parentPlayName === 'string'
+        ? governance.parentPlayName
+        : null,
+    key: typeof governance?.key === 'string' ? governance.key : null,
+    input: isRecord(body.input) ? body.input : {},
+    graphHash: isRecord(body.manifest)
+      ? typeof body.manifest.graphHash === 'string'
+        ? body.manifest.graphHash
+        : null
+      : null,
+  });
 }
 
 function normalizeRuntimeBaseUrl(value: unknown): string | null {
@@ -2087,8 +2381,7 @@ function validateChildSubmitBody(input: {
   const { parentRunId, body } = input;
   const manifest = body.manifest as PlayRuntimeManifest | undefined;
   const governance = body.internalRunPlay as
-    | PlayCallGovernanceSnapshot
-    | undefined;
+    PlayCallGovernanceSnapshot | undefined;
   const childPlayName =
     typeof body.name === 'string' && body.name.trim()
       ? body.name.trim()
@@ -2219,6 +2512,7 @@ function buildChildWorkflowParams(input: {
     executorToken: childToken,
     baseUrl,
     integrationMode: normalizeIntegrationMode(body.integrationMode),
+    force: body.force === true || body.forceToolRefresh === true,
     forceToolRefresh: body.forceToolRefresh === true,
     orgId,
     userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
@@ -2253,6 +2547,7 @@ function runRequestFromPlayWorkflowParams(
     graphHash: params.graphHash,
     userEmail: params.userEmail,
     runtimeInput: params.input,
+    runAttempt: normalizeWorkflowRunAttempt(params.runAttempt),
     inlineCsv: params.inlineCsv ?? null,
     inputFiles:
       inputStorageKey && inputFileName
@@ -2280,8 +2575,11 @@ function runRequestFromPlayWorkflowParams(
     executionPlan: params.executionPlan ?? null,
     childPlayManifests: params.childPlayManifests ?? null,
     playCallGovernance: params.playCallGovernance ?? null,
-    force: params.forceToolRefresh === true,
+    force: params.force === true || params.forceToolRefresh === true,
+    forceToolRefresh: params.forceToolRefresh === true,
     preloadedDbSessions: params.preloadedDbSessions ?? null,
+    runtimeTestFaultHeader: params.runtimeTestFaultHeader ?? null,
+    testPolicyOverrides: params.testPolicyOverrides ?? null,
     inlineChildRunRegistered:
       params.runtimeBackend === 'cf_workflows_dynamic_worker_inline_child',
     coordinatorUrl: params.coordinatorUrl ?? null,
@@ -2376,7 +2674,11 @@ async function executeChildInline(input: {
   }
   const { manifest, governance, childPlayName, orgId, parentExecutorToken } =
     validated;
-  const childRunId = buildChildRunId(childPlayName);
+  const childRunId = buildChildRunId({
+    playName: childPlayName,
+    parentRunId: input.parentRunId,
+    body: input.body,
+  });
   const timings: CoordinatorTiming[] = [];
   const trace = (
     phase: string,
@@ -2433,10 +2735,11 @@ async function executeChildInline(input: {
     input.body.parentPlayName.trim()
       ? input.body.parentPlayName.trim()
       : governance.parentPlayName;
+  const baseUrl = resolveRuntimeBaseUrl(input.env, input.body);
   const { childToken, preloadedDbSessions, prepareTimings, transportTimings } =
     await prepareInlineChildRunWithRuntime({
       env: input.env,
-      baseUrl: resolveRuntimeBaseUrl(input.env, input.body),
+      baseUrl,
       parentExecutorToken,
       parentRunId: input.parentRunId,
       parentPlayName,
@@ -2607,6 +2910,20 @@ async function executeChildInline(input: {
           error: error.message,
         },
       });
+      const terminalAppendStartedAt = Date.now();
+      await appendRegisteredChildRunTerminal({
+        env: input.env,
+        baseUrl,
+        childExecutorToken: childToken,
+        childRunId,
+        status: 'failed',
+        error: error.message,
+      });
+      trace(
+        'coordinator.inline_child_terminal_ledger_append',
+        terminalAppendStartedAt,
+        { status: 'failed' },
+      );
       return {
         workflowId: childRunId,
         runId: childRunId,
@@ -2629,6 +2946,20 @@ async function executeChildInline(input: {
         mode: 'inline_dynamic_worker',
       },
     });
+    const terminalAppendStartedAt = Date.now();
+    await appendRegisteredChildRunTerminal({
+      env: input.env,
+      baseUrl,
+      childExecutorToken: childToken,
+      childRunId,
+      status: 'completed',
+      result: parsed.result,
+    });
+    trace(
+      'coordinator.inline_child_terminal_ledger_append',
+      terminalAppendStartedAt,
+      { status: 'completed' },
+    );
     return {
       workflowId: childRunId,
       runId: childRunId,
@@ -2645,6 +2976,35 @@ async function executeChildInline(input: {
   }
 }
 
+function isRetryableChildWorkflowSubmitError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /timed out|timeout|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
+    message,
+  );
+}
+
+function isRetryableChildWorkflowSubmitResponse(
+  status: number,
+  body: string,
+): boolean {
+  if (
+    status === 408 ||
+    status === 429 ||
+    status === 502 ||
+    status === 503 ||
+    status === 504 ||
+    status === 530
+  ) {
+    return true;
+  }
+  return (
+    status === 500 &&
+    /COORDINATOR_WORKFLOW_ROUTE_FAILED|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|UND_ERR_CONNECT_TIMEOUT|internal error/i.test(
+      body,
+    )
+  );
+}
+
 async function submitChildWorkflowThroughCoordinator(input: {
   env: CoordinatorEnv;
   parentRunId: string;
@@ -2700,7 +3060,11 @@ async function submitChildWorkflowThroughCoordinator(input: {
   }
   const { manifest, governance, childPlayName, orgId, parentExecutorToken } =
     validated;
-  const childRunId = buildChildRunId(childPlayName);
+  const childRunId = buildChildRunId({
+    playName: childPlayName,
+    parentRunId: input.parentRunId,
+    body: input.body,
+  });
   const baseUrl = resolveRuntimeBaseUrl(input.env, input.body);
 
   const tokenStartedAt = Date.now();
@@ -2783,38 +3147,96 @@ async function submitChildWorkflowThroughCoordinator(input: {
   );
 
   const workflowSubmitStartedAt = Date.now();
-  let response: Response;
-  try {
-    response = await handleWorkflowRoute({
-      runId: childRunId,
-      action: 'submit',
-      request: new Request(
-        `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
-          childRunId,
-        )}/submit`,
+  let response: Response | null = null;
+  let responseText = '';
+  let lastSubmitError: unknown = null;
+  for (
+    let attempt = 0;
+    attempt <= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length;
+    attempt += 1
+  ) {
+    try {
+      response = await handleWorkflowRoute({
+        runId: childRunId,
+        action: 'submit',
+        request: new Request(
+          `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
+            childRunId,
+          )}/submit`,
+          {
+            method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify(params),
+          },
+        ),
+        env: input.env,
+      });
+    } catch (error) {
+      lastSubmitError = error;
+      trace(
+        'coordinator.child_submit_workflow_attempt',
+        workflowSubmitStartedAt,
+        manifest.graphHash,
         {
-          method: 'POST',
-          headers: { 'content-type': 'application/json' },
-          body: JSON.stringify(params),
+          childRunId,
+          attempt: attempt + 1,
+          status: 'threw',
+          error: error instanceof Error ? error.message : String(error),
         },
-      ),
-      env: input.env,
-    });
-  } catch (error) {
-    await markRegisteredChildRunFailed({
-      env: input.env,
-      baseUrl,
-      childExecutorToken: childToken,
-      childRunId,
-      error,
-    }).catch((markError) => {
-      console.error('[coordinator] child workflow submit failure mark failed', {
+      );
+      if (
+        attempt >= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length ||
+        !isRetryableChildWorkflowSubmitError(error)
+      ) {
+        await markRegisteredChildRunFailed({
+          env: input.env,
+          baseUrl,
+          childExecutorToken: childToken,
+          childRunId,
+          error,
+        }).catch((markError) => {
+          console.error(
+            '[coordinator] child workflow submit failure mark failed',
+            {
+              childRunId,
+              error:
+                markError instanceof Error
+                  ? markError.message
+                  : String(markError),
+            },
+          );
+        });
+        throw error;
+      }
+      await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
+      continue;
+    }
+
+    responseText = await response.text().catch(() => '');
+    trace(
+      'coordinator.child_submit_workflow_attempt',
+      workflowSubmitStartedAt,
+      manifest.graphHash,
+      {
         childRunId,
-        error:
-          markError instanceof Error ? markError.message : String(markError),
-      });
-    });
-    throw error;
+        attempt: attempt + 1,
+        status: response.status,
+      },
+    );
+    if (
+      response.ok ||
+      attempt >= CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS.length ||
+      !isRetryableChildWorkflowSubmitResponse(response.status, responseText)
+    ) {
+      break;
+    }
+    lastSubmitError = `workflow submit returned ${response.status}: ${responseText.slice(0, 800)}`;
+    await sleep(CHILD_WORKFLOW_SUBMIT_RETRY_DELAYS_MS[attempt] ?? 0);
+  }
+  if (!response) {
+    throw lastSubmitError instanceof Error
+      ? lastSubmitError
+      : new Error(String(lastSubmitError));
   }
   trace(
     'coordinator.child_submit_workflow',
@@ -2822,7 +3244,6 @@ async function submitChildWorkflowThroughCoordinator(input: {
     manifest.graphHash,
     { childRunId, status: response.status },
   );
-  const responseText = await response.text().catch(() => '');
   if (!response.ok) {
     await markRegisteredChildRunFailed({
       env: input.env,
@@ -3156,6 +3577,11 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
       });
     }
     let dispatchedEvent = event;
+    dispatchedEvent = await hydrateWorkflowLaunchParams({
+      env: this.env,
+      event: dispatchedEvent,
+      trace,
+    });
     try {
       dispatchedEvent = await hydrateWorkflowDbSessions({
         env: this.env,
@@ -3202,8 +3628,8 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
           await writeCoordinatorTerminalState(this.env, {
             runId: entryTrace.runId,
             status: 'failed',
-            error: normalizeCoordinatorWorkflowRuntimeFailure(hydrateError)
-              .message,
+            error:
+              normalizeCoordinatorWorkflowRuntimeFailure(hydrateError).message,
           }).catch(() => undefined);
         }
       }
@@ -3300,12 +3726,42 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
                   }
                 ).run(innerEvent, innerStep);
                 const output = isRecord(result) ? result : null;
+                const terminalResult = output?.result ?? result;
                 const terminalStartedAt = Date.now();
+                const childTerminalStartedAt = Date.now();
+                try {
+                  const childTerminalPersisted =
+                    await appendWorkflowChildCompletedRunEvent({
+                      env,
+                      event: innerEvent,
+                      result: terminalResult,
+                    });
+                  if (childTerminalPersisted) {
+                    trace({
+                      runId: runIdForTrace,
+                      phase:
+                        'coordinator.child_terminal_ledger_append_completed',
+                      ms: Date.now() - childTerminalStartedAt,
+                      graphHash,
+                    });
+                  }
+                } catch (childTerminalError) {
+                  console.warn(
+                    '[coordinator] child terminal ledger append failed; preserving coordinator terminal cache',
+                    {
+                      runId: runIdForTrace,
+                      message:
+                        childTerminalError instanceof Error
+                          ? childTerminalError.message
+                          : String(childTerminalError),
+                    },
+                  );
+                }
                 try {
                   await writeCoordinatorTerminalState(env, {
                     runId: runIdForTrace,
                     status: 'completed',
-                    result: output?.result ?? result,
+                    result: terminalResult,
                     totalRows: output?.totalRows ?? output?.outputRows ?? null,
                     durationMs: output?.durationMs ?? null,
                     playName:
@@ -3586,6 +4042,69 @@ async function coordinatorRouteFetch(
       });
     }
   }
+  if (url.pathname === '/rate-acquire') {
+    const authError = authorizeCoordinatorControlRequest({ request, env });
+    if (authError) return authError;
+    try {
+      const body = (await request.json().catch(() => null)) as {
+        bucketId?: unknown;
+        rules?: unknown;
+        requested?: unknown;
+      } | null;
+      const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
+      if (!bucketId.trim()) {
+        return Response.json(
+          { error: 'bucketId is required.' },
+          { status: 400 },
+        );
+      }
+      const result = await callRateBucketControl<{
+        granted: number;
+        waitMs: number;
+      }>(env, bucketId, '/rate-acquire', body);
+      return Response.json(result);
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.rate_acquire.error]',
+        code: 'COORDINATOR_RATE_ROUTE_FAILED',
+        phase: 'coordinator.rate.acquire',
+        runId: null,
+        error,
+      });
+    }
+  }
+  if (url.pathname === '/rate-penalize') {
+    const authError = authorizeCoordinatorControlRequest({ request, env });
+    if (authError) return authError;
+    try {
+      const body = (await request.json().catch(() => null)) as {
+        bucketId?: unknown;
+        cooldownMs?: unknown;
+      } | null;
+      const bucketId = typeof body?.bucketId === 'string' ? body.bucketId : '';
+      if (!bucketId.trim()) {
+        return Response.json(
+          { error: 'bucketId is required.' },
+          { status: 400 },
+        );
+      }
+      await callRateBucketControl<{ ok?: unknown }>(
+        env,
+        bucketId,
+        '/rate-penalize',
+        body,
+      );
+      return Response.json({ ok: true });
+    } catch (error) {
+      return coordinatorRouteErrorResponse({
+        logTag: '[coordinator.rate_penalize.error]',
+        code: 'COORDINATOR_RATE_ROUTE_FAILED',
+        phase: 'coordinator.rate.penalize',
+        runId: null,
+        error,
+      });
+    }
+  }
   // Workflow routes: /workflow/{runId}/{action}
   const wfMatch = url.pathname.match(/^\/workflow\/([^/]+)(?:\/(.+))?$/);
   if (wfMatch) {
@@ -3676,7 +4195,7 @@ async function flushTailRunLogs(
         return;
       }
       await fetch(
-        `${env.DEEPLINE_API_BASE_URL}/api/v2/plays/internal/tail-log`,
+        `${env.DEEPLINE_API_BASE_URL}${PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH}`,
         {
           method: 'POST',
           headers: {
@@ -3779,7 +4298,10 @@ function coordinatorWorkflowRouteErrorResponse(input: {
     action: input.action,
     error: message,
   });
-  if (input.action === 'submit' && isTenantStorageDegradationError(input.error)) {
+  if (
+    input.action === 'submit' &&
+    isTenantStorageDegradationError(input.error)
+  ) {
     // De-redact: a tenant-storage failure gets the loud, actionable
     // WORKSPACE_STORAGE_NOT_READY message + repair instruction instead of the
     // generic "internal error". Same contract as the runtime route.
@@ -3996,12 +4518,15 @@ async function handleWorkflowRoute(input: {
           // logs. Log the tenant-storage classification loudly so the real
           // cause (and the WORKSPACE_STORAGE_NOT_READY repair path) is visible.
           if (storageNotReady) {
-            console.error('[coordinator.harness_prewarm_postgres.storage_not_ready]', {
-              runId: submittedRunId,
-              playName: params.playName,
-              code: WORKSPACE_STORAGE_NOT_READY_CODE,
-              error: error instanceof Error ? error.message : String(error),
-            });
+            console.error(
+              '[coordinator.harness_prewarm_postgres.storage_not_ready]',
+              {
+                runId: submittedRunId,
+                playName: params.playName,
+                code: WORKSPACE_STORAGE_NOT_READY_CODE,
+                error: error instanceof Error ? error.message : String(error),
+              },
+            );
           }
         });
       input.ctx?.waitUntil(prewarmPromise);
@@ -4010,9 +4535,12 @@ async function handleWorkflowRoute(input: {
       params,
     });
     const workflowParams = dbSessionExternalization.params;
+    workflowParams.submittedAt = Date.now();
+    let launchState: Awaited<ReturnType<typeof persistWorkflowLaunchState>> =
+      {};
     try {
       const retryStateStartedAt = Date.now();
-      const launchState = await persistWorkflowLaunchState({
+      launchState = await persistWorkflowLaunchState({
         env,
         runId: submittedRunId,
         params: workflowParams,
@@ -4065,7 +4593,7 @@ async function handleWorkflowRoute(input: {
         error,
       });
     }
-    workflowParams.submittedAt = Date.now();
+    const createParams = workflowCreateParams(workflowParams, launchState);
     let instance: WorkflowInstance | null = null;
     try {
       const submitClaimStartedAt = Date.now();
@@ -4167,7 +4695,8 @@ async function handleWorkflowRoute(input: {
           createDynamicWorkflowInstance({
             env,
             id: defaultInstanceId,
-            params: workflowParams,
+            params: createParams,
+            metadataParams: workflowParams,
           }),
         getExisting: () => env.PLAY_WORKFLOW.get(defaultInstanceId),
         onCreateFailure: async () => {
@@ -4737,7 +5266,8 @@ function stableHash(value: string): string {
   return (hash >>> 0).toString(36);
 }
 
-const DYNAMIC_PLAY_WORKER_HARNESS_VERSION = 'h16-coordinator-only-prewarm';
+const DYNAMIC_PLAY_WORKER_HARNESS_VERSION =
+  DYNAMIC_PLAY_WORKER_ARTIFACT_VERSION;
 const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
 const dynamicWorkerBundledCodeCache = new Map<string, string>();
 
@@ -5057,7 +5587,7 @@ export class TenantWorkflow extends WorkflowEntrypoint {
     const runId = payload && typeof payload.runId === "string" ? payload.runId : "warmup";
     const startedAt = Date.now();
     if (this.env.RUNTIME_API) {
-      await this.env.RUNTIME_API.fetch(new Request("https://deepline.runtime.internal/api/v2/plays/internal/runtime", {
+      await this.env.RUNTIME_API.fetch(new Request("https://deepline.runtime.internal${PLAY_RUNTIME_API_COMPAT_PATH}", {
         method: "POST",
         headers: { "content-type": "application/json" },
         body: "{}"