deepline 0.1.25 → 0.1.27

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -28,7 +28,7 @@ import type {
   PlayRuntimeManifest,
   PlayRuntimeManifestMap,
 } from '../../../shared_libs/plays/compiler-manifest';
-import { runtimeRunActions } from '../../../shared_libs/play-runtime/runtime-actions';
+import type { PlayRunLedgerEvent } from '../../../shared_libs/play-runtime/run-ledger';
 import {
   COORDINATOR_INTERNAL_TOKEN_HEADER,
   COORDINATOR_RUN_SCOPE_HEADER,
@@ -45,11 +45,22 @@ export type PlayWorkflowParams = {
   artifactHash: string;
   graphHash: string;
   input: Record<string, unknown>;
-  inputFile?: { name?: string; r2Key?: string; path?: string } | null;
+  inputFile?: {
+    name?: string;
+    r2Key?: string;
+    storageKey?: string;
+    path?: string;
+    fileName?: string;
+    logicalPath?: string;
+    contentType?: string;
+    bytes?: number;
+  } | null;
   inlineCsv?: { name: string; rows: Record<string, unknown>[] } | null;
   packagedFiles?: Array<{
     playPath: string;
     storageKey: string;
+    contentType?: string;
+    bytes?: number;
     inlineText?: string;
   }> | null;
   contractSnapshot?: unknown;
@@ -221,7 +232,7 @@ interface CoordinatorEnv {
   /**
    * Service binding to the long-lived Play Harness Worker
    * (apps/play-harness-worker). Provides typed RPC access to leaf-level
-   * helpers (zod validation, runtime-API HTTP forwarder, …) that we
+   * helpers (runtime-API HTTP forwarder, Neon dataset IO, …) that we
    * deliberately keep OUT of every per-graphHash play bundle.
    *
    * Optional: when missing (e.g. an older deploy that hasn't been wired
@@ -508,6 +519,8 @@ type DynamicWorkflowMetadata = {
   packagedFiles?: Array<{
     playPath: string;
     storageKey: string;
+    contentType?: string;
+    bytes?: number;
     inlineText?: string;
   }> | null;
 };
@@ -534,10 +547,10 @@ const WORKFLOW_POOL_READY_POLL_MS = 250;
 const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
 const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 4;
 const WORKFLOW_POOL_CONTROL_TIMEOUT_MS = 750;
+const WORKFLOW_POOL_START_ACK_TIMEOUT_MS = 750;
+const WORKFLOW_POOL_START_ACK_POLL_MS = 25;
 const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
 const SUBMIT_INITIAL_STATE_POLL_MS = 50;
-const WORKFLOW_POOL_DISABLED_REASON =
-  'Cloudflare Workflows start runs directly; waitForEvent is reserved for real durable external waits.';
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
 ): DynamicWorkflowMetadata {
@@ -610,7 +623,7 @@ function readWorkflowTraceContext(event: unknown): {
 }
 
 function workflowPoolEnabled(): boolean {
-  return false;
+  return WORKFLOW_POOL_TARGET_SIZE > 0;
 }
 
 function workflowPoolTargetSize(): number {
@@ -876,8 +889,33 @@ async function mapRunToWorkflowInstance(input: {
   env: CoordinatorEnv;
   runId: string;
   instanceId: string;
-}): Promise<void> {
-  await callWorkflowPool(input.env, '/pool-map-run', {
+  started?: boolean;
+}): Promise<boolean> {
+  const body = await callWorkflowPool<{ mapped?: unknown }>(
+    input.env,
+    '/pool-map-run',
+    {
+      method: 'POST',
+      body: JSON.stringify({
+        runId: input.runId,
+        instanceId: input.instanceId,
+        started: input.started === true,
+        version: WORKFLOW_POOL_PROTOCOL_VERSION,
+      }),
+    },
+  );
+  return body.mapped !== false;
+}
+
+async function blockWorkflowPoolRun(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  instanceId: string;
+}): Promise<{ blocked: boolean; started: boolean }> {
+  const body = await callWorkflowPool<{
+    blocked?: unknown;
+    started?: unknown;
+  }>(input.env, '/pool-block-run', {
     method: 'POST',
     body: JSON.stringify({
       runId: input.runId,
@@ -885,6 +923,82 @@ async function mapRunToWorkflowInstance(input: {
       version: WORKFLOW_POOL_PROTOCOL_VERSION,
     }),
   });
+  return {
+    blocked: body.blocked === true,
+    started: body.started === true,
+  };
+}
+
+async function readWorkflowPoolRunMapping(input: {
+  env: CoordinatorEnv;
+  runId: string;
+}): Promise<{ instanceId: string | null; startedAt: number | null }> {
+  const body = await callWorkflowPool<{
+    instanceId?: unknown;
+    startedAt?: unknown;
+  }>(
+    input.env,
+    `/pool-resolve-run?runId=${encodeURIComponent(input.runId)}&version=${encodeURIComponent(
+      WORKFLOW_POOL_PROTOCOL_VERSION,
+    )}`,
+  ).catch(() => ({ instanceId: null, startedAt: null }));
+  return {
+    instanceId:
+      typeof body.instanceId === 'string' && body.instanceId
+        ? body.instanceId
+        : null,
+    startedAt:
+      typeof body.startedAt === 'number' && Number.isFinite(body.startedAt)
+        ? body.startedAt
+        : null,
+  };
+}
+
+async function waitForWorkflowPoolStartAck(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  instanceId: string;
+  timeoutMs: number;
+}): Promise<{
+  acknowledged: boolean;
+  ms: number;
+  polls: number;
+  startedAt: number | null;
+  mappedInstanceId: string | null;
+}> {
+  const startedAt = Date.now();
+  let polls = 0;
+  let latestMapping: { instanceId: string | null; startedAt: number | null } = {
+    instanceId: null,
+    startedAt: null,
+  };
+  while (Date.now() - startedAt < input.timeoutMs) {
+    polls += 1;
+    latestMapping = await readWorkflowPoolRunMapping({
+      env: input.env,
+      runId: input.runId,
+    });
+    if (
+      latestMapping.instanceId === input.instanceId &&
+      latestMapping.startedAt !== null
+    ) {
+      return {
+        acknowledged: true,
+        ms: Date.now() - startedAt,
+        polls,
+        startedAt: latestMapping.startedAt,
+        mappedInstanceId: latestMapping.instanceId,
+      };
+    }
+    await sleep(WORKFLOW_POOL_START_ACK_POLL_MS);
+  }
+  return {
+    acknowledged: false,
+    ms: Date.now() - startedAt,
+    polls,
+    startedAt: latestMapping.startedAt,
+    mappedInstanceId: latestMapping.instanceId,
+  };
 }
 
 async function resolveWorkflowInstanceIdForRun(
@@ -894,15 +1008,8 @@ async function resolveWorkflowInstanceIdForRun(
   if (!workflowPoolEnabled()) {
     return workflowInstanceId(runId);
   }
-  const body = await callWorkflowPool<{ instanceId?: unknown }>(
-    env,
-    `/pool-resolve-run?runId=${encodeURIComponent(runId)}&version=${encodeURIComponent(
-      WORKFLOW_POOL_PROTOCOL_VERSION,
-    )}`,
-  ).catch(() => ({ instanceId: null }));
-  return typeof body.instanceId === 'string' && body.instanceId
-    ? body.instanceId
-    : workflowInstanceId(runId);
+  const mapping = await readWorkflowPoolRunMapping({ env, runId });
+  return mapping.instanceId ? mapping.instanceId : workflowInstanceId(runId);
 }
 
 async function clearWorkflowPool(env: CoordinatorEnv): Promise<number> {
@@ -914,7 +1021,10 @@ async function clearWorkflowPool(env: CoordinatorEnv): Promise<number> {
   );
   await Promise.all(
     entries.map(async (entry) => {
-      const instance = await env.PLAY_WORKFLOW.get(entry.id);
+      const instance = await getWorkflowPoolInstance(env, entry.id);
+      if (!instance) {
+        return;
+      }
       try {
         await instance.terminate().catch(() => undefined);
       } finally {
@@ -929,6 +1039,27 @@ function workflowStatusName(status: InstanceStatus | null): string {
   return typeof status?.status === 'string' ? status.status : 'unknown';
 }
 
+function isWorkflowInstanceNotFoundError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /not[ _]found|not_found|does not exist|no such instance|404/i.test(
+    message,
+  );
+}
+
+async function getWorkflowPoolInstance(
+  env: CoordinatorEnv,
+  instanceId: string,
+): Promise<WorkflowInstance | null> {
+  try {
+    return await env.PLAY_WORKFLOW.get(instanceId);
+  } catch (error) {
+    if (isWorkflowInstanceNotFoundError(error)) {
+      return null;
+    }
+    throw error;
+  }
+}
+
 function workflowPoolStatusIsReady(statusName: string): boolean {
   // This is only a liveness guard. Readiness itself comes from the pooled
   // Workflow calling /pool-ready after waitForEvent("play_start") has been
@@ -1011,7 +1142,11 @@ async function refillWorkflowPoolOnce(
   const promotedIds: string[] = [];
   const removedIds: string[] = [];
   for (const entry of warmingEntries) {
-    const instance = await env.PLAY_WORKFLOW.get(entry.id);
+    const instance = await getWorkflowPoolInstance(env, entry.id);
+    if (!instance) {
+      removedIds.push(entry.id);
+      continue;
+    }
     try {
       if (entry.state === 'ready' && entry.readyAt !== null) {
         promotedIds.push(entry.id);
@@ -1238,7 +1373,21 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
 
-  const instance = await input.env.PLAY_WORKFLOW.get(pooledInstanceId);
+  const instance = await getWorkflowPoolInstance(input.env, pooledInstanceId);
+  if (!instance) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_ready_check',
+      ms: Date.now() - leaseStartedAt,
+      graphHash: input.params.graphHash ?? null,
+      extra: { instanceId: pooledInstanceId, status: 'missing' },
+    });
+    return null;
+  }
   const readyCheckStartedAt = Date.now();
   const status = await instance.status().catch(() => null);
   const statusName = workflowStatusName(status);
@@ -1249,6 +1398,11 @@ async function submitViaPooledWorkflow(input: {
     extra: { instanceId: pooledInstanceId, status: statusName },
   });
   if (!workflowPoolStatusIsReady(statusName)) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
     await instance.terminate().catch(() => undefined);
     disposeRpcStub(instance);
     return null;
@@ -1260,6 +1414,11 @@ async function submitViaPooledWorkflow(input: {
       payload: buildDispatcherEnvelope(input.params),
     });
   } catch (error) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
     disposeRpcStub(instance);
     console.warn('[coordinator.workflow_pool] sendEvent failed; falling back', {
       runId: input.params.runId,
@@ -1274,7 +1433,63 @@ async function submitViaPooledWorkflow(input: {
     graphHash: input.params.graphHash ?? null,
     extra: { instanceId: pooledInstanceId },
   });
-  return instance;
+  const ack = await waitForWorkflowPoolStartAck({
+    env: input.env,
+    runId: input.params.runId,
+    instanceId: pooledInstanceId,
+    timeoutMs: WORKFLOW_POOL_START_ACK_TIMEOUT_MS,
+  });
+  if (ack.acknowledged) {
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_start_ack',
+      ms: ack.ms,
+      graphHash: input.params.graphHash ?? null,
+      extra: {
+        acknowledged: true,
+        instanceId: pooledInstanceId,
+        polls: ack.polls,
+        startedAt: ack.startedAt,
+      },
+    });
+    return instance;
+  }
+
+  const blockStartedAt = Date.now();
+  const block = await blockWorkflowPoolRun({
+    env: input.env,
+    runId: input.params.runId,
+    instanceId: pooledInstanceId,
+  }).catch(() => ({ blocked: false, started: false }));
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_start_ack',
+    ms: ack.ms,
+    graphHash: input.params.graphHash ?? null,
+    extra: {
+      acknowledged: block.started,
+      instanceId: pooledInstanceId,
+      polls: ack.polls,
+      startedAt: ack.startedAt,
+      mappedInstanceId: ack.mappedInstanceId,
+      blocked: block.blocked,
+      blockMs: Date.now() - blockStartedAt,
+    },
+  });
+  if (block.started) {
+    return instance;
+  }
+  await instance.terminate().catch(() => undefined);
+  disposeRpcStub(instance);
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_fallback',
+    ms: Date.now() - sendStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: {
+      reason: 'start_ack_timeout',
+      instanceId: pooledInstanceId,
+      ackTimeoutMs: WORKFLOW_POOL_START_ACK_TIMEOUT_MS,
+    },
+  });
+  return null;
 }
 
 function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
@@ -1311,16 +1526,21 @@ async function markWorkflowRuntimeFailure(input: {
   });
   const bypass = input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim();
   if (bypass) headers.set('x-vercel-protection-bypass', bypass);
-  const body = JSON.stringify(
-    runtimeRunActions.updateStatus({
-      playId: runId,
-      status: 'failed',
-      error: `DynamicWorkflow runner failed: ${errorName}: ${errorMessage}${
-        errorStack ? `\n${errorStack}` : ''
-      }`,
-      runtimeBackend: 'cf_workflows_dynamic_worker',
-    }),
-  );
+  const body = JSON.stringify({
+    action: 'append_run_events',
+    playId: runId,
+    events: [
+      {
+        type: 'run.failed',
+        runId,
+        source: 'coordinator',
+        occurredAt: Date.now(),
+        error: `DynamicWorkflow runner failed: ${errorName}: ${errorMessage}${
+          errorStack ? `\n${errorStack}` : ''
+        }`,
+      } satisfies PlayRunLedgerEvent,
+    ],
+  });
   const url = `${baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`;
   const backoffMs = [200, 500, 1500];
   let lastError: unknown = null;
@@ -1621,6 +1841,15 @@ function buildChildWorkflowParams(input: {
 function runRequestFromPlayWorkflowParams(
   params: PlayWorkflowParams,
 ): Record<string, unknown> {
+  const inputFileName = String(
+    params.inputFile?.name ??
+      params.inputFile?.fileName ??
+      params.inputFile?.logicalPath ??
+      params.inputFile?.path ??
+      '',
+  );
+  const inputStorageKey =
+    params.inputFile?.r2Key ?? params.inputFile?.storageKey ?? null;
   return {
     runId: params.runId,
     callbackUrl: params.baseUrl,
@@ -1632,12 +1861,22 @@ function runRequestFromPlayWorkflowParams(
     userEmail: params.userEmail,
     runtimeInput: params.input,
     inlineCsv: params.inlineCsv ?? null,
-    inputR2Keys:
-      params.inputFile?.r2Key &&
-      (params.inputFile.name || params.inputFile.path)
+    inputFiles:
+      inputStorageKey && inputFileName
         ? {
-            [String(params.inputFile.name ?? params.inputFile.path)]:
-              params.inputFile.r2Key,
+            [inputFileName]: {
+              logicalPath:
+                params.inputFile?.logicalPath ??
+                params.inputFile?.path ??
+                inputFileName,
+              fileName: params.inputFile?.fileName ?? inputFileName,
+              storageKey: inputStorageKey,
+              contentType: params.inputFile?.contentType ?? null,
+              bytes:
+                typeof params.inputFile?.bytes === 'number'
+                  ? params.inputFile.bytes
+                  : null,
+            },
           }
         : null,
     packagedFiles: params.packagedFiles ?? null,
@@ -1996,13 +2235,6 @@ export class RuntimeApi extends WorkerEntrypoint<CoordinatorEnv, undefined> {
         ? this.env.DEEPLINE_API_BASE_URL.trim()
         : 'https://code.deepline.com';
     const target = new URL(incoming.pathname + incoming.search, apiBaseUrl);
-    const runtimeStatusBody =
-      incoming.pathname === '/api/v2/plays/internal/runtime'
-        ? await request
-            .clone()
-            .json()
-            .catch(() => null)
-        : null;
     const forwarded = new Request(target.toString(), request);
     const bypassToken = this.env.VERCEL_PROTECTION_BYPASS_TOKEN;
     if (typeof bypassToken === 'string' && bypassToken) {
@@ -2018,41 +2250,9 @@ export class RuntimeApi extends WorkerEntrypoint<CoordinatorEnv, undefined> {
         `[RUNTIME_API] ${incoming.pathname} failed: status=${res.status} ` +
           `target=${target.toString()} body=${body.slice(0, 500)}`,
       );
-    } else {
-      await this.recordRuntimeStatusEvent(runtimeStatusBody).catch(() => null);
     }
     return res;
   }
-
-  private async recordRuntimeStatusEvent(body: unknown): Promise<void> {
-    if (!isRecord(body) || body.action !== 'update_run_status') {
-      return;
-    }
-    const runId = typeof body.playId === 'string' ? body.playId : '';
-    const status = typeof body.status === 'string' ? body.status : '';
-    if (!runId || !status) {
-      return;
-    }
-    await appendCoordinatorRunEvent(this.env, {
-      runId,
-      type: 'progress',
-      status,
-      ts: Date.now(),
-      logs: sanitizeLiveLogLines(body.liveLogs) ?? undefined,
-      activeNodeId:
-        typeof body.activeNodeId === 'string' ? body.activeNodeId : null,
-      activeArtifactTableNamespace:
-        typeof body.activeArtifactTableNamespace === 'string'
-          ? body.activeArtifactTableNamespace
-          : null,
-      updatedAt:
-        typeof body.lastCheckpointAt === 'number'
-          ? body.lastCheckpointAt
-          : null,
-      liveNodeProgress:
-        body.liveNodeProgress !== undefined ? body.liveNodeProgress : undefined,
-    });
-  }
 }
 
 export class CoordinatorControl extends WorkerEntrypoint<
@@ -2188,9 +2388,35 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
       dispatchedEvent = {
         payload: startEvent.payload,
         timestamp: startEvent.timestamp,
-        instanceId: workflowEvent.instanceId,
+        instanceId: workflowEvent.instanceId ?? pooledPayload.poolId,
       };
       const dispatchedTrace = readWorkflowTraceContext(dispatchedEvent);
+      const mapped = await mapRunToWorkflowInstance({
+        env: this.env,
+        runId: dispatchedTrace.runId,
+        instanceId: pooledPayload.poolId,
+        started: true,
+      }).catch((error) => {
+        console.warn('[coordinator.workflow_pool] start ack failed', {
+          poolId: pooledPayload.poolId,
+          runId: dispatchedTrace.runId,
+          message: error instanceof Error ? error.message : String(error),
+        });
+        return false;
+      });
+      if (!mapped) {
+        trace({
+          runId: dispatchedTrace.runId,
+          phase: 'coordinator.workflow_pool_start_blocked',
+          ms: 0,
+          graphHash: dispatchedTrace.graphHash,
+          extra: {
+            instanceId: pooledPayload.poolId,
+            eventType: startEvent.type,
+          },
+        });
+        return { ok: false, blocked: true, runId: dispatchedTrace.runId };
+      }
       const eventDeliveryMs = Math.max(
         0,
         Date.now() - startEvent.timestamp.getTime(),
@@ -2474,7 +2700,15 @@ const coordinatorEntrypoint = {
       const entries = await listWorkflowPoolEntries(env);
       const detailed = [];
       for (const entry of entries) {
-        const instance = await env.PLAY_WORKFLOW.get(entry.id);
+        const instance = await getWorkflowPoolInstance(env, entry.id);
+        if (!instance) {
+          detailed.push({
+            ...entry,
+            status: 'missing',
+            mappedStatus: 'failed',
+          });
+          continue;
+        }
         try {
           const status = await instance.status().catch(() => null);
           detailed.push({
@@ -2538,6 +2772,14 @@ const coordinatorEntrypoint = {
   async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
     await flushTailRunLogs(events, env);
   },
+  async scheduled(
+    _controller: unknown,
+    env: CoordinatorEnv,
+    ctx?: ExecutionContext,
+  ): Promise<void> {
+    if (!workflowPoolEnabled()) return;
+    ctx?.waitUntil(refillWorkflowPool(env).catch(() => undefined));
+  },
 };
 
 export default coordinatorEntrypoint;
@@ -2700,36 +2942,100 @@ async function handleWorkflowRoute(input: {
           'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
       );
     }
+    const preloadedDbSessions = Array.isArray(params.preloadedDbSessions)
+      ? params.preloadedDbSessions
+      : [];
+    if (preloadedDbSessions.length > 0 && params.executorToken) {
+      const prewarmStartedAt = Date.now();
+      recordSubmitTiming({
+        phase: 'coordinator.harness_prewarm_postgres_start',
+        ms: 0,
+        graphHash: params.graphHash ?? null,
+        extra: { sessions: preloadedDbSessions.length },
+      });
+      const prewarmPromise = env.HARNESS.prewarmPostgresSessions({
+        executorToken: params.executorToken,
+        sessions: preloadedDbSessions,
+      })
+        .then((result) => {
+          recordSubmitTiming({
+            phase: 'coordinator.harness_prewarm_postgres',
+            ms: Date.now() - prewarmStartedAt,
+            graphHash: params.graphHash ?? null,
+            extra: {
+              status: 'ok',
+              sessions: result.sessions,
+            },
+          });
+        })
+        .catch((error: unknown) => {
+          recordSubmitTiming({
+            phase: 'coordinator.harness_prewarm_postgres',
+            ms: Date.now() - prewarmStartedAt,
+            graphHash: params.graphHash ?? null,
+            extra: {
+              status: 'failed',
+              sessions: preloadedDbSessions.length,
+              error: error instanceof Error ? error.message : String(error),
+            },
+          });
+        });
+      input.ctx?.waitUntil(prewarmPromise);
+    }
     let instance: WorkflowInstance | null = null;
     try {
-      const dispatchStartedAt = Date.now();
+      const statusEventStartedAt = Date.now();
+      await appendCoordinatorRunEvent(env, {
+        runId: submittedRunId,
+        type: 'status',
+        status: 'running',
+        ts: Date.now(),
+      });
       recordSubmitTiming({
-        phase: 'coordinator.workflow_pool_attempt',
-        ms: 0,
+        phase: 'coordinator.submit_status_event',
+        ms: Date.now() - statusEventStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: {
-          usedPool: false,
-          disabled: true,
-          reason: WORKFLOW_POOL_DISABLED_REASON,
-        },
       });
-      const createStartedAt = Date.now();
-      instance = await createDynamicWorkflowInstance({
+      const dispatchStartedAt = Date.now();
+      const poolAttemptStartedAt = Date.now();
+      instance = await submitViaPooledWorkflow({
         env,
-        id: defaultInstanceId,
         params,
+        recordSubmitTiming,
       });
       recordSubmitTiming({
-        phase: 'coordinator.workflow_create',
-        ms: Date.now() - createStartedAt,
+        phase: 'coordinator.workflow_pool_attempt',
+        ms: Date.now() - poolAttemptStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: { instanceId: instance.id },
+        extra: {
+          usedPool: Boolean(instance),
+          enabled: workflowPoolEnabled(),
+        },
       });
+      if (!instance) {
+        const createStartedAt = Date.now();
+        instance = await createDynamicWorkflowInstance({
+          env,
+          id: defaultInstanceId,
+          params,
+        });
+        recordSubmitTiming({
+          phase: 'coordinator.workflow_create',
+          ms: Date.now() - createStartedAt,
+          graphHash: params.graphHash ?? null,
+          extra: { instanceId: instance.id },
+        });
+      }
       recordSubmitTiming({
         phase: 'coordinator.dispatch_workflow',
         ms: Date.now() - dispatchStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: { startMode: 'direct_workflow_create' },
+        extra: {
+          startMode:
+            instance.id === defaultInstanceId
+              ? 'direct_workflow_create'
+              : 'pooled_workflow_start_event',
+        },
       });
       const initialWaitMsRaw = Number(
         new URL(request.url).searchParams.get('initialWaitMs') ?? '0',
@@ -2763,6 +3069,9 @@ async function handleWorkflowRoute(input: {
         ms: totalMs,
         graphHash: params.graphHash ?? null,
       });
+      if (workflowPoolEnabled() && instance.id === defaultInstanceId) {
+        input.ctx?.waitUntil(refillWorkflowPool(env).catch(() => undefined));
+      }
       return Response.json({
         runId,
         status: 'submitted',
@@ -3018,11 +3327,19 @@ async function handleWorkflowRoute(input: {
       afterSeq,
       timeoutMs: waitMs,
     }).catch(() => null);
+    const rawEvents = eventResult?.events ?? [];
+    const terminalEventIndex = rawEvents.findIndex(
+      (event) => event.type === 'terminal',
+    );
+    const events =
+      terminalEventIndex >= 0
+        ? rawEvents.slice(0, terminalEventIndex + 1)
+        : rawEvents;
     const coordinatorTrace =
-      includeTrace && eventResult?.events.length
+      includeTrace && events.length
         ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
         : [];
-    const terminalEvent = eventResult?.events.find(
+    const terminalEvent = events.find(
       (event): event is Extract<CoordinatorRunEvent, { type: 'terminal' }> =>
         event.type === 'terminal',
     );
@@ -3038,7 +3355,7 @@ async function handleWorkflowRoute(input: {
         completedAt: terminalEvent.ts,
         liveLogs: sanitizeLiveLogLines(terminalEvent.liveLogs),
         liveNodeProgress: terminalEvent.liveNodeProgress ?? null,
-        events: eventResult?.events ?? [],
+        events,
         latestSeq: eventResult?.latestSeq ?? afterSeq,
         wait: null,
         coordinatorObserve: {
@@ -3054,7 +3371,7 @@ async function handleWorkflowRoute(input: {
     return Response.json({
       runId,
       status: 'running',
-      events: eventResult?.events ?? [],
+      events,
       latestSeq: eventResult?.latestSeq ?? afterSeq,
       wait: null,
       coordinatorObserve: {
@@ -3252,7 +3569,7 @@ function stableHash(value: string): string {
   return (hash >>> 0).toString(36);
 }
 
-const DYNAMIC_PLAY_WORKER_HARNESS_VERSION = 'h7-skip-high-volume-tool-traces';
+const DYNAMIC_PLAY_WORKER_HARNESS_VERSION = 'h16-coordinator-only-prewarm';
 const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
 const dynamicWorkerBundledCodeCache = new Map<string, string>();
 
@@ -3548,6 +3865,14 @@ function normalizePackagedFiles(
     .map((entry) => ({
       playPath: String(entry.playPath ?? '').replace(/^\.\//, ''),
       storageKey: String(entry.storageKey ?? ''),
+      contentType:
+        typeof entry.contentType === 'string' ? entry.contentType : undefined,
+      bytes:
+        typeof entry.bytes === 'number' &&
+        Number.isSafeInteger(entry.bytes) &&
+        entry.bytes >= 0
+          ? entry.bytes
+          : undefined,
       inlineText:
         typeof entry.inlineText === 'string' ? entry.inlineText : undefined,
     }))
@@ -4062,8 +4387,13 @@ function mapWorkflowResult(
   runId: string,
   status: InstanceStatus,
 ): Record<string, unknown> {
-  const error = readWorkflowError(status);
-  const mapped = resolveTerminalStatus(status, error);
+  const rawError = readWorkflowError(status);
+  const mapped = resolveTerminalStatus(status, rawError);
+  const error =
+    rawError ??
+    (mapped === 'failed'
+      ? `Cloudflare workflow reported ${String(status.status ?? 'failed')} without an error payload.`
+      : null);
   const output =
     status.output && typeof status.output === 'object'
       ? (status.output as Record<string, unknown>)
@@ -4131,9 +4461,11 @@ function resolveTerminalStatus(
 function readWorkflowError(status: InstanceStatus): string | null {
   const error = status.error as unknown;
   if (!error) return null;
-  if (typeof error === 'string') return error;
+  if (typeof error === 'string') return error.trim() || null;
   if (typeof error === 'object' && 'message' in error) {
-    return String((error as { message?: unknown }).message ?? '');
+    const message = String((error as { message?: unknown }).message ?? '');
+    return message.trim() || null;
   }
-  return String(error);
+  const message = String(error);
+  return message.trim() || null;
 }