deepline 0.1.168 → 0.1.169

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -112,6 +112,7 @@ export type PlayWorkflowParams = {
   dynamicWorkerCode?: string | null;
   executorToken: string;
   baseUrl: string;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   orgId: string;
   userEmail: string;
   userId?: string | null;
@@ -628,6 +629,7 @@ type DynamicWorkflowMetadata = {
   artifactStorageKey: string;
   artifactHash?: string | null;
   dynamicWorkerCode?: string | null;
+  integrationMode?: 'live' | 'eval_stub' | 'fixture' | null;
   packagedFiles?: Array<{
     playPath: string;
     storageKey: string;
@@ -640,9 +642,19 @@ type DynamicWorkflowMetadata = {
 const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
 const SUBMIT_INITIAL_STATE_POLL_MS = 50;
 const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
+const WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS = 55_000;
+const WORKFLOW_SUBMIT_DUPLICATE_POLL_MS = 250;
 const COORDINATOR_WORKER_MEMORY_LIMIT_MESSAGE =
   'Worker memory limit hit before the run could finish. For CSV enrich, use --output or --in-place for automatic batches; otherwise rerun smaller --rows ranges.';
 
+function normalizeIntegrationMode(
+  value: unknown,
+): 'live' | 'eval_stub' | 'fixture' | null {
+  return value === 'live' || value === 'eval_stub' || value === 'fixture'
+    ? value
+    : null;
+}
+
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
 ): DynamicWorkflowMetadata {
@@ -652,6 +664,7 @@ function buildDynamicWorkflowMetadata(
     artifactStorageKey: params.artifactStorageKey,
     artifactHash: params.artifactHash ?? null,
     dynamicWorkerCode: params.dynamicWorkerCode ?? null,
+    integrationMode: normalizeIntegrationMode(params.integrationMode),
     packagedFiles: normalizePackagedFiles(params.packagedFiles),
   };
 }
@@ -879,6 +892,72 @@ async function resolveWorkflowInstanceIdForRun(
     : workflowInstanceId(runId);
 }
 
+async function claimWorkflowSubmit(input: {
+  env: CoordinatorEnv;
+  runId: string;
+}): Promise<{
+  claimed: boolean;
+  status: 'creating' | 'created' | null;
+  instanceId: string | null;
+}> {
+  const body = await callRunScopedControl<{
+    claimed?: unknown;
+    status?: unknown;
+    instanceId?: unknown;
+  }>(input.env, input.runId, '/workflow-submit-claim', {
+    method: 'POST',
+    body: JSON.stringify({
+      runId: input.runId,
+      ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
+    }),
+  });
+  return {
+    claimed: body.claimed === true,
+    status:
+      body.status === 'creating' || body.status === 'created'
+        ? body.status
+        : null,
+    instanceId: typeof body.instanceId === 'string' ? body.instanceId : null,
+  };
+}
+
+async function releaseWorkflowSubmitClaim(input: {
+  env: CoordinatorEnv;
+  runId: string;
+}): Promise<void> {
+  await callRunScopedControl<{ ok?: unknown }>(
+    input.env,
+    input.runId,
+    '/workflow-submit-release',
+    {
+      method: 'POST',
+      body: JSON.stringify({ runId: input.runId }),
+    },
+  );
+}
+
+async function waitForWorkflowSubmitCreated(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  initialClaim: Awaited<ReturnType<typeof claimWorkflowSubmit>>;
+  timeoutMs: number;
+  pollMs: number;
+}): Promise<Awaited<ReturnType<typeof claimWorkflowSubmit>>> {
+  let claim = input.initialClaim;
+  const deadline = Date.now() + input.timeoutMs;
+  while (claim.status !== 'created' || !claim.instanceId) {
+    if (Date.now() >= deadline) {
+      return claim;
+    }
+    await sleep(input.pollMs);
+    claim = await claimWorkflowSubmit({
+      env: input.env,
+      runId: input.runId,
+    });
+  }
+  return claim;
+}
+
 function assertEncryptedPreloadedDbSessions(
   sessions: PreloadedRuntimeDbSession[],
 ): void {
@@ -1715,7 +1794,7 @@ async function preloadChildRuntimeDbSessions(input: {
   return sessions;
 }
 
-async function registerInlineChildRunWithRuntime(input: {
+async function registerChildRunWithRuntime(input: {
   env: CoordinatorEnv;
   baseUrl: string;
   childExecutorToken: string;
@@ -1723,6 +1802,9 @@ async function registerInlineChildRunWithRuntime(input: {
   childPlayName: string;
   manifest: PlayRuntimeManifest;
   governance: PlayCallGovernanceSnapshot;
+  runtimeBackend: string;
+  schedulerBackend: string;
+  executionProfile: string;
 }): Promise<void> {
   const response = await input.env.HARNESS.runtimeApiCall({
     executorToken: input.childExecutorToken,
@@ -1743,9 +1825,9 @@ async function registerInlineChildRunWithRuntime(input: {
       artifactStorageKey: input.manifest.artifactStorageKey,
       artifactHash: input.manifest.artifactHash,
       graphHash: input.manifest.graphHash,
-      runtimeBackend: 'workers_edge',
-      schedulerBackend: 'inline_child',
-      executionProfile: 'workers_edge',
+      runtimeBackend: input.runtimeBackend,
+      schedulerBackend: input.schedulerBackend,
+      executionProfile: input.executionProfile,
       ...(typeof input.manifest.maxCreditsPerRun === 'number'
         ? { maxCreditsPerRun: input.manifest.maxCreditsPerRun }
         : {}),
@@ -1761,6 +1843,43 @@ async function registerInlineChildRunWithRuntime(input: {
   }
 }
 
+async function markRegisteredChildRunFailed(input: {
+  env: CoordinatorEnv;
+  baseUrl: string;
+  childExecutorToken: string;
+  childRunId: string;
+  error: unknown;
+}): Promise<void> {
+  const message =
+    input.error instanceof Error ? input.error.message : String(input.error);
+  const response = await input.env.HARNESS.runtimeApiCall({
+    executorToken: input.childExecutorToken,
+    baseUrl: input.baseUrl,
+    path: '/api/v2/plays/internal/runtime',
+    headers: { 'x-deepline-request-id': crypto.randomUUID() },
+    timeoutMs: 15_000,
+    body: {
+      action: 'append_run_events',
+      playId: input.childRunId,
+      events: [
+        {
+          type: 'run.failed',
+          runId: input.childRunId,
+          source: 'coordinator',
+          occurredAt: Date.now(),
+          error: `Child workflow submit failed: ${message}`,
+        } satisfies PlayRunLedgerEvent,
+      ],
+    },
+  });
+  if (response.status < 200 || response.status >= 300) {
+    const text = response.body ?? '';
+    throw new Error(
+      `Inline child run failure mark failed ${response.status}: ${text.slice(0, 800)}`,
+    );
+  }
+}
+
 type CoordinatorRuntimeApiTiming = {
   phase: string;
   ms: number;
@@ -2094,6 +2213,7 @@ function buildChildWorkflowParams(input: {
     dynamicWorkerCode,
     executorToken: childToken,
     baseUrl,
+    integrationMode: normalizeIntegrationMode(body.integrationMode),
     orgId,
     userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
     userId: typeof body.userId === 'string' ? body.userId : null,
@@ -2121,6 +2241,7 @@ function runRequestFromPlayWorkflowParams(
     callbackUrl: params.baseUrl,
     executorToken: params.executorToken,
     baseUrl: params.baseUrl,
+    integrationMode: normalizeIntegrationMode(params.integrationMode),
     orgId: params.orgId,
     playName: params.playName,
     graphHash: params.graphHash,
@@ -2630,22 +2751,63 @@ async function submitChildWorkflowThroughCoordinator(input: {
       preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
   });
 
-  const workflowSubmitStartedAt = Date.now();
-  const response = await handleWorkflowRoute({
-    runId: childRunId,
-    action: 'submit',
-    request: new Request(
-      `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
-        childRunId,
-      )}/submit`,
-      {
-        method: 'POST',
-        headers: { 'content-type': 'application/json' },
-        body: JSON.stringify(params),
-      },
-    ),
+  const registerStartedAt = Date.now();
+  await registerChildRunWithRuntime({
     env: input.env,
+    baseUrl,
+    childExecutorToken: childToken,
+    childRunId,
+    childPlayName,
+    manifest,
+    governance,
+    runtimeBackend: 'cf_workflows_dynamic_worker',
+    schedulerBackend: 'cf_workflows',
+    executionProfile: 'workers_edge',
   });
+  trace(
+    'coordinator.child_submit_register_run',
+    registerStartedAt,
+    manifest.graphHash,
+    {
+      childRunId,
+      childPlayName,
+    },
+  );
+
+  const workflowSubmitStartedAt = Date.now();
+  let response: Response;
+  try {
+    response = await handleWorkflowRoute({
+      runId: childRunId,
+      action: 'submit',
+      request: new Request(
+        `https://deepline.coordinator.internal/workflow/${encodeURIComponent(
+          childRunId,
+        )}/submit`,
+        {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify(params),
+        },
+      ),
+      env: input.env,
+    });
+  } catch (error) {
+    await markRegisteredChildRunFailed({
+      env: input.env,
+      baseUrl,
+      childExecutorToken: childToken,
+      childRunId,
+      error,
+    }).catch((markError) => {
+      console.error('[coordinator] child workflow submit failure mark failed', {
+        childRunId,
+        error:
+          markError instanceof Error ? markError.message : String(markError),
+      });
+    });
+    throw error;
+  }
   trace(
     'coordinator.child_submit_workflow',
     workflowSubmitStartedAt,
@@ -2653,6 +2815,21 @@ async function submitChildWorkflowThroughCoordinator(input: {
     { childRunId, status: response.status },
   );
   const responseText = await response.text().catch(() => '');
+  if (!response.ok) {
+    await markRegisteredChildRunFailed({
+      env: input.env,
+      baseUrl,
+      childExecutorToken: childToken,
+      childRunId,
+      error: `workflow submit returned ${response.status}: ${responseText.slice(0, 800)}`,
+    }).catch((markError) => {
+      console.error('[coordinator] child workflow submit failure mark failed', {
+        childRunId,
+        error:
+          markError instanceof Error ? markError.message : String(markError),
+      });
+    });
+  }
   return {
     response,
     responseText,
@@ -3025,6 +3202,9 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
                 typeof metadata.dynamicWorkerCode === 'string'
                   ? metadata.dynamicWorkerCode
                   : null,
+              integrationMode: normalizeIntegrationMode(
+                metadata.integrationMode,
+              ),
               packagedFiles: normalizePackagedFiles(metadata.packagedFiles),
             },
             trace,
@@ -3612,6 +3792,7 @@ async function handleWorkflowRoute(input: {
       ms: Date.now() - submitStartedAt,
       graphHash: params.graphHash ?? null,
       extra: {
+        integrationMode: normalizeIntegrationMode(params.integrationMode),
         hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
         dynamicWorkerBytes:
           typeof params.dynamicWorkerCode === 'string'
@@ -3624,6 +3805,7 @@ async function handleWorkflowRoute(input: {
       ms: Date.now() - parseStartedAt,
       graphHash: params.graphHash ?? null,
       extra: {
+        integrationMode: normalizeIntegrationMode(params.integrationMode),
         hasDynamicWorkerCode: Boolean(params.dynamicWorkerCode),
         dynamicWorkerBytes:
           typeof params.dynamicWorkerCode === 'string'
@@ -3746,6 +3928,92 @@ async function handleWorkflowRoute(input: {
     workflowParams.submittedAt = Date.now();
     let instance: WorkflowInstance | null = null;
     try {
+      const submitClaimStartedAt = Date.now();
+      const submitClaim = await claimWorkflowSubmit({
+        env,
+        runId: submittedRunId,
+      });
+      recordSubmitTiming({
+        phase: 'coordinator.workflow_submit_claim',
+        ms: Date.now() - submitClaimStartedAt,
+        graphHash: params.graphHash ?? null,
+        extra: {
+          claimed: submitClaim.claimed,
+          status: submitClaim.status,
+          instanceId: submitClaim.instanceId ?? defaultInstanceId,
+        },
+      });
+      if (!submitClaim.claimed) {
+        const duplicateWaitStartedAt = Date.now();
+        const resolvedSubmitClaim = await waitForWorkflowSubmitCreated({
+          env,
+          runId: submittedRunId,
+          initialClaim: submitClaim,
+          timeoutMs: WORKFLOW_SUBMIT_DUPLICATE_WAIT_MS,
+          pollMs: WORKFLOW_SUBMIT_DUPLICATE_POLL_MS,
+        });
+        recordSubmitTiming({
+          phase: 'coordinator.workflow_submit_duplicate_wait',
+          ms: Date.now() - duplicateWaitStartedAt,
+          graphHash: params.graphHash ?? null,
+          extra: {
+            status: resolvedSubmitClaim.status,
+            instanceId: resolvedSubmitClaim.instanceId ?? null,
+          },
+        });
+        const totalMs = Date.now() - submitStartedAt;
+        if (
+          resolvedSubmitClaim.status !== 'created' ||
+          !resolvedSubmitClaim.instanceId
+        ) {
+          recordSubmitTiming({
+            phase: 'coordinator.submit_total',
+            ms: totalMs,
+            graphHash: params.graphHash ?? null,
+            extra: { duplicate: true, pending: true },
+          });
+          return Response.json(
+            {
+              runId,
+              status: 'pending',
+              workflowInstanceId: null,
+              instanceState: {
+                status: resolvedSubmitClaim.status ?? 'creating',
+              },
+              retryAfterMs: 250,
+              runtimeDeployVersion:
+                env.CF_VERSION_METADATA?.id ??
+                env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
+                null,
+              coordinatorTimings,
+            },
+            { status: 503, headers: { 'retry-after': '1' } },
+          );
+        }
+        recordSubmitTiming({
+          phase: 'coordinator.submit_total',
+          ms: totalMs,
+          graphHash: params.graphHash ?? null,
+          extra: { duplicate: true },
+        });
+        recordSubmitTiming({
+          phase: 'coordinator.submit_accepted',
+          ms: totalMs,
+          graphHash: params.graphHash ?? null,
+          extra: { duplicate: true },
+        });
+        return Response.json({
+          runId,
+          status: 'submitted',
+          workflowInstanceId: resolvedSubmitClaim.instanceId,
+          instanceState: { status: resolvedSubmitClaim.status },
+          runtimeDeployVersion:
+            env.CF_VERSION_METADATA?.id ??
+            env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ??
+            null,
+          coordinatorTimings,
+        });
+      }
       const dispatchStartedAt = Date.now();
       const createStartedAt = Date.now();
       recordSubmitTiming({
@@ -3762,6 +4030,12 @@ async function handleWorkflowRoute(input: {
             params: workflowParams,
           }),
         getExisting: () => env.PLAY_WORKFLOW.get(defaultInstanceId),
+        onCreateFailure: async () => {
+          await releaseWorkflowSubmitClaim({
+            env,
+            runId: submittedRunId,
+          });
+        },
       });
       instance = createResult.instance;
       const workflowCreatedAt = Date.now();
@@ -3771,18 +4045,35 @@ async function handleWorkflowRoute(input: {
         graphHash: params.graphHash ?? null,
         extra: { instanceId: instance.id, startMode: createResult.startMode },
       });
-      const instanceIdRecord = recordWorkflowInstanceId({
-        env,
-        runId: submittedRunId,
-        instanceId: instance.id,
-      }).catch((error) => {
+      try {
+        await recordWorkflowInstanceId({
+          env,
+          runId: submittedRunId,
+          instanceId: instance.id,
+        });
+      } catch (error) {
         console.warn('[coordinator] workflow instance id record failed', {
           runId: submittedRunId,
           instanceId: instance?.id ?? null,
           error: error instanceof Error ? error.message : String(error),
         });
-      });
-      input.ctx?.waitUntil(instanceIdRecord);
+        await releaseWorkflowSubmitClaim({
+          env,
+          runId: submittedRunId,
+        }).catch((releaseError) => {
+          console.warn(
+            '[coordinator] workflow submit claim release after instance record failure failed',
+            {
+              runId: submittedRunId,
+              error:
+                releaseError instanceof Error
+                  ? releaseError.message
+                  : String(releaseError),
+            },
+          );
+        });
+        throw error;
+      }
       recordSubmitTiming({
         phase: 'coordinator.dispatch_workflow',
         ms: Date.now() - dispatchStartedAt,
@@ -3790,7 +4081,7 @@ async function handleWorkflowRoute(input: {
         extra: {
           startMode: 'direct_workflow_create',
           workflowCreateMode: createResult.startMode,
-          instanceIdRecord: 'waitUntil',
+          instanceIdRecord: 'recorded',
         },
       });
       const initialWaitMsRaw = Number(

package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts

@@ -102,6 +102,15 @@ type WorkflowInstanceState = {
   expiresAt: number;
 };
 
+type WorkflowSubmitClaimState = {
+  runId: string;
+  status: 'creating' | 'created';
+  instanceId?: string;
+  claimedAt: number;
+  updatedAt: number;
+  expiresAt: number;
+};
+
 type WorkflowDbSessionsState = {
   runId: string;
   sessions: PreloadedRuntimeDbSession[];
@@ -357,6 +366,10 @@ export class PlayDedup implements DurableObject {
           return await this.handleWorkflowInstancePut(req);
         case '/workflow-instance-get':
           return await this.handleWorkflowInstanceGet(req);
+        case '/workflow-submit-claim':
+          return await this.handleWorkflowSubmitClaim(req);
+        case '/workflow-submit-release':
+          return await this.handleWorkflowSubmitRelease(req);
         case '/run-retry-claim':
           return await this.handleRunRetryClaim(req);
         case '/db-sessions-put':
@@ -787,12 +800,22 @@ export class PlayDedup implements DurableObject {
       body.ttlMs > 0
         ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
         : WORKFLOW_RUN_STATE_TTL_MS;
-    await this.state.storage.put('workflow-instance', {
-      runId,
-      instanceId,
-      updatedAt: now,
-      expiresAt: now + ttlMs,
-    } satisfies WorkflowInstanceState);
+    await this.state.storage.put({
+      'workflow-instance': {
+        runId,
+        instanceId,
+        updatedAt: now,
+        expiresAt: now + ttlMs,
+      } satisfies WorkflowInstanceState,
+      'workflow-submit-claim': {
+        runId,
+        status: 'created',
+        instanceId,
+        claimedAt: now,
+        updatedAt: now,
+        expiresAt: now + ttlMs,
+      } satisfies WorkflowSubmitClaimState,
+    });
     return new Response(JSON.stringify({ ok: true }), {
       headers: { 'content-type': 'application/json' },
     });
@@ -821,6 +844,76 @@ export class PlayDedup implements DurableObject {
     });
   }
 
+  private async handleWorkflowSubmitClaim(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      ttlMs?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    if (!runId) {
+      return new Response('runId is required', { status: 400 });
+    }
+    const now = Date.now();
+    const ttlMs =
+      typeof body?.ttlMs === 'number' &&
+      Number.isFinite(body.ttlMs) &&
+      body.ttlMs > 0
+        ? Math.max(60_000, Math.min(body.ttlMs, WORKFLOW_RUN_STATE_TTL_MS))
+        : WORKFLOW_RUN_STATE_TTL_MS;
+    let response: {
+      claimed: boolean;
+      status: WorkflowSubmitClaimState['status'];
+      instanceId: string | null;
+    } = { claimed: true, status: 'creating', instanceId: null };
+    await this.state.blockConcurrencyWhile(async () => {
+      const key = 'workflow-submit-claim';
+      const existing =
+        await this.state.storage.get<WorkflowSubmitClaimState>(key);
+      if (existing?.runId === runId && existing.expiresAt > now) {
+        response = {
+          claimed: false,
+          status: existing.status,
+          instanceId: existing.instanceId ?? null,
+        };
+        return;
+      }
+      const state = {
+        runId,
+        status: 'creating',
+        claimedAt: now,
+        updatedAt: now,
+        expiresAt: now + ttlMs,
+      } satisfies WorkflowSubmitClaimState;
+      await this.state.storage.put(key, state);
+    });
+    return new Response(JSON.stringify(response), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async handleWorkflowSubmitRelease(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    if (!runId) {
+      return new Response('runId is required', { status: 400 });
+    }
+    let released = false;
+    await this.state.blockConcurrencyWhile(async () => {
+      const key = 'workflow-submit-claim';
+      const existing =
+        await this.state.storage.get<WorkflowSubmitClaimState>(key);
+      if (existing?.runId === runId && existing.status === 'creating') {
+        await this.state.storage.delete(key);
+        released = true;
+      }
+    });
+    return new Response(JSON.stringify({ ok: true, released }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
   private async handleDbSessionsPut(req: Request): Promise<Response> {
     const body = (await req.json().catch(() => null)) as {
       runId?: unknown;