deepline 0.1.26 → 0.1.28

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -232,7 +232,7 @@ interface CoordinatorEnv {
   /**
    * Service binding to the long-lived Play Harness Worker
    * (apps/play-harness-worker). Provides typed RPC access to leaf-level
-   * helpers (zod validation, runtime-API HTTP forwarder, …) that we
+   * helpers (runtime-API HTTP forwarder, Neon dataset IO, …) that we
    * deliberately keep OUT of every per-graphHash play bundle.
    *
    * Optional: when missing (e.g. an older deploy that hasn't been wired
@@ -547,10 +547,10 @@ const WORKFLOW_POOL_READY_POLL_MS = 250;
 const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
 const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 4;
 const WORKFLOW_POOL_CONTROL_TIMEOUT_MS = 750;
+const WORKFLOW_POOL_START_ACK_TIMEOUT_MS = 750;
+const WORKFLOW_POOL_START_ACK_POLL_MS = 25;
 const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
 const SUBMIT_INITIAL_STATE_POLL_MS = 50;
-const WORKFLOW_POOL_DISABLED_REASON =
-  'Cloudflare Workflows start runs directly; waitForEvent is reserved for real durable external waits.';
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
 ): DynamicWorkflowMetadata {
@@ -623,7 +623,7 @@ function readWorkflowTraceContext(event: unknown): {
 }
 
 function workflowPoolEnabled(): boolean {
-  return false;
+  return WORKFLOW_POOL_TARGET_SIZE > 0;
 }
 
 function workflowPoolTargetSize(): number {
@@ -889,8 +889,33 @@ async function mapRunToWorkflowInstance(input: {
   env: CoordinatorEnv;
   runId: string;
   instanceId: string;
-}): Promise<void> {
-  await callWorkflowPool(input.env, '/pool-map-run', {
+  started?: boolean;
+}): Promise<boolean> {
+  const body = await callWorkflowPool<{ mapped?: unknown }>(
+    input.env,
+    '/pool-map-run',
+    {
+      method: 'POST',
+      body: JSON.stringify({
+        runId: input.runId,
+        instanceId: input.instanceId,
+        started: input.started === true,
+        version: WORKFLOW_POOL_PROTOCOL_VERSION,
+      }),
+    },
+  );
+  return body.mapped !== false;
+}
+
+async function blockWorkflowPoolRun(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  instanceId: string;
+}): Promise<{ blocked: boolean; started: boolean }> {
+  const body = await callWorkflowPool<{
+    blocked?: unknown;
+    started?: unknown;
+  }>(input.env, '/pool-block-run', {
     method: 'POST',
     body: JSON.stringify({
       runId: input.runId,
@@ -898,6 +923,82 @@ async function mapRunToWorkflowInstance(input: {
       version: WORKFLOW_POOL_PROTOCOL_VERSION,
     }),
   });
+  return {
+    blocked: body.blocked === true,
+    started: body.started === true,
+  };
+}
+
+async function readWorkflowPoolRunMapping(input: {
+  env: CoordinatorEnv;
+  runId: string;
+}): Promise<{ instanceId: string | null; startedAt: number | null }> {
+  const body = await callWorkflowPool<{
+    instanceId?: unknown;
+    startedAt?: unknown;
+  }>(
+    input.env,
+    `/pool-resolve-run?runId=${encodeURIComponent(input.runId)}&version=${encodeURIComponent(
+      WORKFLOW_POOL_PROTOCOL_VERSION,
+    )}`,
+  ).catch(() => ({ instanceId: null, startedAt: null }));
+  return {
+    instanceId:
+      typeof body.instanceId === 'string' && body.instanceId
+        ? body.instanceId
+        : null,
+    startedAt:
+      typeof body.startedAt === 'number' && Number.isFinite(body.startedAt)
+        ? body.startedAt
+        : null,
+  };
+}
+
+async function waitForWorkflowPoolStartAck(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  instanceId: string;
+  timeoutMs: number;
+}): Promise<{
+  acknowledged: boolean;
+  ms: number;
+  polls: number;
+  startedAt: number | null;
+  mappedInstanceId: string | null;
+}> {
+  const startedAt = Date.now();
+  let polls = 0;
+  let latestMapping: { instanceId: string | null; startedAt: number | null } = {
+    instanceId: null,
+    startedAt: null,
+  };
+  while (Date.now() - startedAt < input.timeoutMs) {
+    polls += 1;
+    latestMapping = await readWorkflowPoolRunMapping({
+      env: input.env,
+      runId: input.runId,
+    });
+    if (
+      latestMapping.instanceId === input.instanceId &&
+      latestMapping.startedAt !== null
+    ) {
+      return {
+        acknowledged: true,
+        ms: Date.now() - startedAt,
+        polls,
+        startedAt: latestMapping.startedAt,
+        mappedInstanceId: latestMapping.instanceId,
+      };
+    }
+    await sleep(WORKFLOW_POOL_START_ACK_POLL_MS);
+  }
+  return {
+    acknowledged: false,
+    ms: Date.now() - startedAt,
+    polls,
+    startedAt: latestMapping.startedAt,
+    mappedInstanceId: latestMapping.instanceId,
+  };
 }
 
 async function resolveWorkflowInstanceIdForRun(
@@ -907,15 +1008,8 @@ async function resolveWorkflowInstanceIdForRun(
   if (!workflowPoolEnabled()) {
     return workflowInstanceId(runId);
   }
-  const body = await callWorkflowPool<{ instanceId?: unknown }>(
-    env,
-    `/pool-resolve-run?runId=${encodeURIComponent(runId)}&version=${encodeURIComponent(
-      WORKFLOW_POOL_PROTOCOL_VERSION,
-    )}`,
-  ).catch(() => ({ instanceId: null }));
-  return typeof body.instanceId === 'string' && body.instanceId
-    ? body.instanceId
-    : workflowInstanceId(runId);
+  const mapping = await readWorkflowPoolRunMapping({ env, runId });
+  return mapping.instanceId ? mapping.instanceId : workflowInstanceId(runId);
 }
 
 async function clearWorkflowPool(env: CoordinatorEnv): Promise<number> {
@@ -927,7 +1021,10 @@ async function clearWorkflowPool(env: CoordinatorEnv): Promise<number> {
   );
   await Promise.all(
     entries.map(async (entry) => {
-      const instance = await env.PLAY_WORKFLOW.get(entry.id);
+      const instance = await getWorkflowPoolInstance(env, entry.id);
+      if (!instance) {
+        return;
+      }
       try {
         await instance.terminate().catch(() => undefined);
       } finally {
@@ -942,6 +1039,27 @@ function workflowStatusName(status: InstanceStatus | null): string {
   return typeof status?.status === 'string' ? status.status : 'unknown';
 }
 
+function isWorkflowInstanceNotFoundError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return /not[ _]found|not_found|does not exist|no such instance|404/i.test(
+    message,
+  );
+}
+
+async function getWorkflowPoolInstance(
+  env: CoordinatorEnv,
+  instanceId: string,
+): Promise<WorkflowInstance | null> {
+  try {
+    return await env.PLAY_WORKFLOW.get(instanceId);
+  } catch (error) {
+    if (isWorkflowInstanceNotFoundError(error)) {
+      return null;
+    }
+    throw error;
+  }
+}
+
 function workflowPoolStatusIsReady(statusName: string): boolean {
   // This is only a liveness guard. Readiness itself comes from the pooled
   // Workflow calling /pool-ready after waitForEvent("play_start") has been
@@ -1024,7 +1142,11 @@ async function refillWorkflowPoolOnce(
   const promotedIds: string[] = [];
   const removedIds: string[] = [];
   for (const entry of warmingEntries) {
-    const instance = await env.PLAY_WORKFLOW.get(entry.id);
+    const instance = await getWorkflowPoolInstance(env, entry.id);
+    if (!instance) {
+      removedIds.push(entry.id);
+      continue;
+    }
     try {
       if (entry.state === 'ready' && entry.readyAt !== null) {
         promotedIds.push(entry.id);
@@ -1251,7 +1373,21 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
 
-  const instance = await input.env.PLAY_WORKFLOW.get(pooledInstanceId);
+  const instance = await getWorkflowPoolInstance(input.env, pooledInstanceId);
+  if (!instance) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_ready_check',
+      ms: Date.now() - leaseStartedAt,
+      graphHash: input.params.graphHash ?? null,
+      extra: { instanceId: pooledInstanceId, status: 'missing' },
+    });
+    return null;
+  }
   const readyCheckStartedAt = Date.now();
   const status = await instance.status().catch(() => null);
   const statusName = workflowStatusName(status);
@@ -1262,6 +1398,11 @@ async function submitViaPooledWorkflow(input: {
     extra: { instanceId: pooledInstanceId, status: statusName },
   });
   if (!workflowPoolStatusIsReady(statusName)) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
     await instance.terminate().catch(() => undefined);
     disposeRpcStub(instance);
     return null;
@@ -1273,6 +1414,11 @@ async function submitViaPooledWorkflow(input: {
       payload: buildDispatcherEnvelope(input.params),
     });
   } catch (error) {
+    await blockWorkflowPoolRun({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    }).catch(() => undefined);
     disposeRpcStub(instance);
     console.warn('[coordinator.workflow_pool] sendEvent failed; falling back', {
       runId: input.params.runId,
@@ -1287,7 +1433,63 @@ async function submitViaPooledWorkflow(input: {
     graphHash: input.params.graphHash ?? null,
     extra: { instanceId: pooledInstanceId },
   });
-  return instance;
+  const ack = await waitForWorkflowPoolStartAck({
+    env: input.env,
+    runId: input.params.runId,
+    instanceId: pooledInstanceId,
+    timeoutMs: WORKFLOW_POOL_START_ACK_TIMEOUT_MS,
+  });
+  if (ack.acknowledged) {
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_start_ack',
+      ms: ack.ms,
+      graphHash: input.params.graphHash ?? null,
+      extra: {
+        acknowledged: true,
+        instanceId: pooledInstanceId,
+        polls: ack.polls,
+        startedAt: ack.startedAt,
+      },
+    });
+    return instance;
+  }
+
+  const blockStartedAt = Date.now();
+  const block = await blockWorkflowPoolRun({
+    env: input.env,
+    runId: input.params.runId,
+    instanceId: pooledInstanceId,
+  }).catch(() => ({ blocked: false, started: false }));
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_start_ack',
+    ms: ack.ms,
+    graphHash: input.params.graphHash ?? null,
+    extra: {
+      acknowledged: block.started,
+      instanceId: pooledInstanceId,
+      polls: ack.polls,
+      startedAt: ack.startedAt,
+      mappedInstanceId: ack.mappedInstanceId,
+      blocked: block.blocked,
+      blockMs: Date.now() - blockStartedAt,
+    },
+  });
+  if (block.started) {
+    return instance;
+  }
+  await instance.terminate().catch(() => undefined);
+  disposeRpcStub(instance);
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_fallback',
+    ms: Date.now() - sendStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: {
+      reason: 'start_ack_timeout',
+      instanceId: pooledInstanceId,
+      ackTimeoutMs: WORKFLOW_POOL_START_ACK_TIMEOUT_MS,
+    },
+  });
+  return null;
 }
 
 function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
@@ -2186,9 +2388,35 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
       dispatchedEvent = {
         payload: startEvent.payload,
         timestamp: startEvent.timestamp,
-        instanceId: workflowEvent.instanceId,
+        instanceId: workflowEvent.instanceId ?? pooledPayload.poolId,
       };
       const dispatchedTrace = readWorkflowTraceContext(dispatchedEvent);
+      const mapped = await mapRunToWorkflowInstance({
+        env: this.env,
+        runId: dispatchedTrace.runId,
+        instanceId: pooledPayload.poolId,
+        started: true,
+      }).catch((error) => {
+        console.warn('[coordinator.workflow_pool] start ack failed', {
+          poolId: pooledPayload.poolId,
+          runId: dispatchedTrace.runId,
+          message: error instanceof Error ? error.message : String(error),
+        });
+        return false;
+      });
+      if (!mapped) {
+        trace({
+          runId: dispatchedTrace.runId,
+          phase: 'coordinator.workflow_pool_start_blocked',
+          ms: 0,
+          graphHash: dispatchedTrace.graphHash,
+          extra: {
+            instanceId: pooledPayload.poolId,
+            eventType: startEvent.type,
+          },
+        });
+        return { ok: false, blocked: true, runId: dispatchedTrace.runId };
+      }
       const eventDeliveryMs = Math.max(
         0,
         Date.now() - startEvent.timestamp.getTime(),
@@ -2472,7 +2700,15 @@ const coordinatorEntrypoint = {
       const entries = await listWorkflowPoolEntries(env);
       const detailed = [];
       for (const entry of entries) {
-        const instance = await env.PLAY_WORKFLOW.get(entry.id);
+        const instance = await getWorkflowPoolInstance(env, entry.id);
+        if (!instance) {
+          detailed.push({
+            ...entry,
+            status: 'missing',
+            mappedStatus: 'failed',
+          });
+          continue;
+        }
         try {
           const status = await instance.status().catch(() => null);
           detailed.push({
@@ -2536,6 +2772,14 @@ const coordinatorEntrypoint = {
   async tail(events: unknown[], env: CoordinatorEnv): Promise<void> {
     await flushTailRunLogs(events, env);
   },
+  async scheduled(
+    _controller: unknown,
+    env: CoordinatorEnv,
+    ctx?: ExecutionContext,
+  ): Promise<void> {
+    if (!workflowPoolEnabled()) return;
+    ctx?.waitUntil(refillWorkflowPool(env).catch(() => undefined));
+  },
 };
 
 export default coordinatorEntrypoint;
@@ -2698,36 +2942,100 @@ async function handleWorkflowRoute(input: {
           'Start apps/play-harness-worker before the coordinator or fix wrangler.toml services.',
       );
     }
+    const preloadedDbSessions = Array.isArray(params.preloadedDbSessions)
+      ? params.preloadedDbSessions
+      : [];
+    if (preloadedDbSessions.length > 0 && params.executorToken) {
+      const prewarmStartedAt = Date.now();
+      recordSubmitTiming({
+        phase: 'coordinator.harness_prewarm_postgres_start',
+        ms: 0,
+        graphHash: params.graphHash ?? null,
+        extra: { sessions: preloadedDbSessions.length },
+      });
+      const prewarmPromise = env.HARNESS.prewarmPostgresSessions({
+        executorToken: params.executorToken,
+        sessions: preloadedDbSessions,
+      })
+        .then((result) => {
+          recordSubmitTiming({
+            phase: 'coordinator.harness_prewarm_postgres',
+            ms: Date.now() - prewarmStartedAt,
+            graphHash: params.graphHash ?? null,
+            extra: {
+              status: 'ok',
+              sessions: result.sessions,
+            },
+          });
+        })
+        .catch((error: unknown) => {
+          recordSubmitTiming({
+            phase: 'coordinator.harness_prewarm_postgres',
+            ms: Date.now() - prewarmStartedAt,
+            graphHash: params.graphHash ?? null,
+            extra: {
+              status: 'failed',
+              sessions: preloadedDbSessions.length,
+              error: error instanceof Error ? error.message : String(error),
+            },
+          });
+        });
+      input.ctx?.waitUntil(prewarmPromise);
+    }
     let instance: WorkflowInstance | null = null;
     try {
-      const dispatchStartedAt = Date.now();
+      const statusEventStartedAt = Date.now();
+      await appendCoordinatorRunEvent(env, {
+        runId: submittedRunId,
+        type: 'status',
+        status: 'running',
+        ts: Date.now(),
+      });
       recordSubmitTiming({
-        phase: 'coordinator.workflow_pool_attempt',
-        ms: 0,
+        phase: 'coordinator.submit_status_event',
+        ms: Date.now() - statusEventStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: {
-          usedPool: false,
-          disabled: true,
-          reason: WORKFLOW_POOL_DISABLED_REASON,
-        },
       });
-      const createStartedAt = Date.now();
-      instance = await createDynamicWorkflowInstance({
+      const dispatchStartedAt = Date.now();
+      const poolAttemptStartedAt = Date.now();
+      instance = await submitViaPooledWorkflow({
         env,
-        id: defaultInstanceId,
         params,
+        recordSubmitTiming,
       });
       recordSubmitTiming({
-        phase: 'coordinator.workflow_create',
-        ms: Date.now() - createStartedAt,
+        phase: 'coordinator.workflow_pool_attempt',
+        ms: Date.now() - poolAttemptStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: { instanceId: instance.id },
+        extra: {
+          usedPool: Boolean(instance),
+          enabled: workflowPoolEnabled(),
+        },
       });
+      if (!instance) {
+        const createStartedAt = Date.now();
+        instance = await createDynamicWorkflowInstance({
+          env,
+          id: defaultInstanceId,
+          params,
+        });
+        recordSubmitTiming({
+          phase: 'coordinator.workflow_create',
+          ms: Date.now() - createStartedAt,
+          graphHash: params.graphHash ?? null,
+          extra: { instanceId: instance.id },
+        });
+      }
       recordSubmitTiming({
         phase: 'coordinator.dispatch_workflow',
         ms: Date.now() - dispatchStartedAt,
         graphHash: params.graphHash ?? null,
-        extra: { startMode: 'direct_workflow_create' },
+        extra: {
+          startMode:
+            instance.id === defaultInstanceId
+              ? 'direct_workflow_create'
+              : 'pooled_workflow_start_event',
+        },
       });
       const initialWaitMsRaw = Number(
         new URL(request.url).searchParams.get('initialWaitMs') ?? '0',
@@ -2761,6 +3069,9 @@ async function handleWorkflowRoute(input: {
         ms: totalMs,
         graphHash: params.graphHash ?? null,
       });
+      if (workflowPoolEnabled() && instance.id === defaultInstanceId) {
+        input.ctx?.waitUntil(refillWorkflowPool(env).catch(() => undefined));
+      }
       return Response.json({
         runId,
         status: 'submitted',
@@ -3016,11 +3327,19 @@ async function handleWorkflowRoute(input: {
       afterSeq,
       timeoutMs: waitMs,
     }).catch(() => null);
+    const rawEvents = eventResult?.events ?? [];
+    const terminalEventIndex = rawEvents.findIndex(
+      (event) => event.type === 'terminal',
+    );
+    const events =
+      terminalEventIndex >= 0
+        ? rawEvents.slice(0, terminalEventIndex + 1)
+        : rawEvents;
     const coordinatorTrace =
-      includeTrace && eventResult?.events.length
+      includeTrace && events.length
         ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
         : [];
-    const terminalEvent = eventResult?.events.find(
+    const terminalEvent = events.find(
       (event): event is Extract<CoordinatorRunEvent, { type: 'terminal' }> =>
         event.type === 'terminal',
     );
@@ -3036,7 +3355,7 @@ async function handleWorkflowRoute(input: {
         completedAt: terminalEvent.ts,
         liveLogs: sanitizeLiveLogLines(terminalEvent.liveLogs),
         liveNodeProgress: terminalEvent.liveNodeProgress ?? null,
-        events: eventResult?.events ?? [],
+        events,
         latestSeq: eventResult?.latestSeq ?? afterSeq,
         wait: null,
         coordinatorObserve: {
@@ -3052,7 +3371,7 @@ async function handleWorkflowRoute(input: {
     return Response.json({
       runId,
       status: 'running',
-      events: eventResult?.events ?? [],
+      events,
       latestSeq: eventResult?.latestSeq ?? afterSeq,
       wait: null,
       coordinatorObserve: {
@@ -3250,7 +3569,7 @@ function stableHash(value: string): string {
   return (hash >>> 0).toString(36);
 }
 
-const DYNAMIC_PLAY_WORKER_HARNESS_VERSION = 'h11-full-row-dataset-handles';
+const DYNAMIC_PLAY_WORKER_HARNESS_VERSION = 'h16-coordinator-only-prewarm';
 const DYNAMIC_WORKER_BUNDLED_CODE_CACHE_MAX_ENTRIES = 64;
 const dynamicWorkerBundledCodeCache = new Map<string, string>();