deepline 0.1.12 → 0.1.20

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -156,6 +156,7 @@ interface CoordinatorEnv {
   DEEPLINE_API_BASE_URL: string;
   DEEPLINE_INTERNAL_TOKEN?: string;
   DEEPLINE_TAIL_LOG_TOKEN?: string;
+  DEEPLINE_COORDINATOR_DEPLOY_MARKER?: string;
   VERCEL_PROTECTION_BYPASS_TOKEN?: string;
   DEEPLINE_PLAY_PREVIEW_SLUG?: string;
   /**
@@ -367,6 +368,9 @@ const WORKFLOW_POOL_TTL_MS = 8 * 60 * 1000;
 const WORKFLOW_POOL_TARGET_SIZE = 2;
 const WORKFLOW_POOL_READY_TIMEOUT_MS = 1_500;
 const WORKFLOW_POOL_READY_POLL_MS = 250;
+const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
+const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 1;
+const WORKFLOW_POOL_CONTROL_TIMEOUT_MS = 750;
 
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
@@ -467,26 +471,48 @@ function workflowPoolDurableObject(env: CoordinatorEnv): DurableObjectStub {
 async function callWorkflowPool<T>(
   env: CoordinatorEnv,
   path: string,
-  init?: RequestInit,
+  init?: RequestInit & { timeoutMs?: number },
 ): Promise<T> {
-  const response = await workflowPoolDurableObject(env).fetch(
-    `https://deepline.workflow-pool.internal${path}`,
-    {
-      ...init,
-      headers: {
-        'content-type': 'application/json',
-        ...(init?.headers ?? {}),
-      },
-    },
+  const timeoutMs = Math.max(
+    1,
+    Math.floor(init?.timeoutMs ?? WORKFLOW_POOL_CONTROL_TIMEOUT_MS),
   );
-  if (!response.ok) {
-    throw new Error(
-      `workflow pool ${path} failed ${response.status}: ${(
-        await response.text().catch(() => '')
-      ).slice(0, 400)}`,
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const fetchInit: RequestInit = { ...(init ?? {}) };
+    delete (fetchInit as { timeoutMs?: number }).timeoutMs;
+    delete fetchInit.signal;
+    const response = await workflowPoolDurableObject(env).fetch(
+      `https://deepline.workflow-pool.internal${path}`,
+      {
+        ...fetchInit,
+        signal: controller.signal,
+        headers: {
+          'content-type': 'application/json',
+          ...(init?.headers ?? {}),
+        },
+      },
     );
+    if (!response.ok) {
+      throw new Error(
+        `workflow pool ${path} failed ${response.status}: ${(
+          await response.text().catch(() => '')
+        ).slice(0, 400)}`,
+      );
+    }
+    return (await response.json()) as T;
+  } catch (error) {
+    if (
+      error instanceof Error &&
+      (error.name === 'AbortError' || error.message.includes('aborted'))
+    ) {
+      throw new Error(`workflow pool ${path} timed out after ${timeoutMs}ms`);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timer);
   }
-  return (await response.json()) as T;
 }
 
 type WorkflowPoolCounts = {
@@ -901,7 +927,11 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
   const leaseStartedAt = Date.now();
-  const pooledInstanceId = await leaseWorkflowPoolId(input.env);
+  let leaseError: string | null = null;
+  let pooledInstanceId = await leaseWorkflowPoolId(input.env).catch((error) => {
+    leaseError = error instanceof Error ? error.message : String(error);
+    return null;
+  });
   const missCounts = pooledInstanceId
     ? null
     : await workflowPoolCount(input.env).catch(() => null);
@@ -911,6 +941,7 @@ async function submitViaPooledWorkflow(input: {
     graphHash: input.params.graphHash ?? null,
     extra: {
       pooled: Boolean(pooledInstanceId),
+      ...(leaseError ? { error: leaseError } : {}),
       ...(missCounts
         ? {
             availableAfterMiss: missCounts.available,
@@ -919,43 +950,79 @@ async function submitViaPooledWorkflow(input: {
         : {}),
     },
   });
+
+  if (!pooledInstanceId) {
+    // A pool miss is often a timing gap rather than a true lack of warm
+    // capacity. Wait briefly for refill/promotion, then retry lease once
+    // before falling back to a cold workflow create.
+    const refillStartedAt = Date.now();
+    const refillResult = await refillWorkflowPool(input.env, {
+      waitReady: true,
+      minAvailable: WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE,
+      waitTimeoutMs: WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS,
+    }).catch(() => null);
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_refill_on_miss',
+      ms: Date.now() - refillStartedAt,
+      graphHash: input.params.graphHash ?? null,
+      extra:
+        refillResult === null
+          ? { ok: false }
+          : {
+              ok: true,
+              available: refillResult.available,
+              warming: refillResult.warming,
+              created: refillResult.created,
+              promoted: refillResult.promoted,
+              removed: refillResult.removed,
+              waitedMs: refillResult.waitedMs,
+              waitIterations: refillResult.waitIterations,
+            },
+    });
+    if (refillResult?.available) {
+      const retryStartedAt = Date.now();
+      let retryLeaseError: string | null = null;
+      pooledInstanceId = await leaseWorkflowPoolId(input.env).catch((error) => {
+        retryLeaseError = error instanceof Error ? error.message : String(error);
+        return null;
+      });
+      input.recordSubmitTiming({
+        phase: 'coordinator.workflow_pool_lease_retry',
+        ms: Date.now() - retryStartedAt,
+        graphHash: input.params.graphHash ?? null,
+        extra: {
+          pooled: Boolean(pooledInstanceId),
+          ...(retryLeaseError ? { error: retryLeaseError } : {}),
+        },
+      });
+    }
+  }
+
   if (!pooledInstanceId) {
     return null;
   }
 
   const instance = await input.env.PLAY_WORKFLOW.get(pooledInstanceId);
+  const readyCheckStartedAt = Date.now();
+  const status = await instance.status().catch(() => null);
+  const statusName = workflowStatusName(status);
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_ready_check',
+    ms: Date.now() - readyCheckStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: { instanceId: pooledInstanceId, status: statusName },
+  });
+  if (!workflowPoolStatusIsReady(statusName)) {
+    await instance.terminate().catch(() => undefined);
+    disposeRpcStub(instance);
+    return null;
+  }
+  const sendStartedAt = Date.now();
   try {
-    const readyCheckStartedAt = Date.now();
-    const status = await instance.status().catch(() => null);
-    const statusName = workflowStatusName(status);
-    input.recordSubmitTiming({
-      phase: 'coordinator.workflow_pool_ready_check',
-      ms: Date.now() - readyCheckStartedAt,
-      graphHash: input.params.graphHash ?? null,
-      extra: { instanceId: pooledInstanceId, status: statusName },
-    });
-    if (!workflowPoolStatusIsReady(statusName)) {
-      await instance.terminate().catch(() => undefined);
-      disposeRpcStub(instance);
-      return null;
-    }
-    const sendStartedAt = Date.now();
     await instance.sendEvent({
       type: WORKFLOW_POOL_START_EVENT_TYPE,
       payload: buildDispatcherEnvelope(input.params),
     });
-    await mapRunToWorkflowInstance({
-      env: input.env,
-      runId: input.params.runId,
-      instanceId: pooledInstanceId,
-    });
-    input.recordSubmitTiming({
-      phase: 'coordinator.workflow_pool_send_event',
-      ms: Date.now() - sendStartedAt,
-      graphHash: input.params.graphHash ?? null,
-      extra: { instanceId: pooledInstanceId },
-    });
-    return instance;
   } catch (error) {
     disposeRpcStub(instance);
     console.warn('[coordinator.workflow_pool] sendEvent failed; falling back', {
@@ -965,6 +1032,27 @@ async function submitViaPooledWorkflow(input: {
     });
     return null;
   }
+  try {
+    await mapRunToWorkflowInstance({
+      env: input.env,
+      runId: input.params.runId,
+      instanceId: pooledInstanceId,
+    });
+  } catch (error) {
+    disposeRpcStub(instance);
+    throw new Error(
+      `workflow pool mapRunToWorkflowInstance failed after pooled workflow start for ${input.params.runId}: ${
+        error instanceof Error ? error.message : String(error)
+      }`,
+    );
+  }
+  input.recordSubmitTiming({
+    phase: 'coordinator.workflow_pool_send_event',
+    ms: Date.now() - sendStartedAt,
+    graphHash: input.params.graphHash ?? null,
+    extra: { instanceId: pooledInstanceId },
+  });
+  return instance;
 }
 
 function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
@@ -1986,6 +2074,29 @@ const coordinatorEntrypoint = {
       if (authError) return authError;
       return await handleCoordinatorWarmup(request, env, ctx);
     }
+    if (url.pathname === '/tail-log-token/probe') {
+      const authError = authorizeCoordinatorControlRequest({ request, env });
+      if (authError) return authError;
+      const expectedTailLogToken = env.DEEPLINE_TAIL_LOG_TOKEN?.trim();
+      if (!expectedTailLogToken) {
+        return Response.json(
+          { ok: false, error: 'tail log token is not configured' },
+          { status: 503 },
+        );
+      }
+      const actualTailLogToken =
+        request.headers.get('x-deepline-tail-log-token')?.trim() ?? '';
+      if (actualTailLogToken !== expectedTailLogToken) {
+        return Response.json(
+          { ok: false, error: 'tail log token mismatch' },
+          { status: 401 },
+        );
+      }
+      return Response.json({
+        ok: true,
+        deployMarker: env.DEEPLINE_COORDINATOR_DEPLOY_MARKER ?? null,
+      });
+    }
     if (url.pathname === '/workflow-pool/refill') {
       const internalAuthError = authorizeCoordinatorControlRequest({
         request,
@@ -2755,7 +2866,7 @@ function loadDynamicPlayWorkerSync(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h5-runtime-api-coordinator`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2781,10 +2892,15 @@ function loadDynamicPlayWorkerSync(
         // which case the per-play stub throws a loud error on first use
         // (no silent fallbacks — see harness-stub.ts → requireBinding).
         HARNESS: env.HARNESS,
+        VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
         // In-process runtime API bridge used by the play harness for status,
         // tool execution, DB session, and artifact callbacks. This avoids a
-        // public fetch hop and is required by apps/play-runner-workers/src/entry.ts.
-        RUNTIME_API: makeRuntimeApiBinding(),
+        // public fetch hop when Cloudflare exposes the RuntimeApi export.
+        ...makeRuntimeApiEnvBinding(),
+        // In-process coordinator control bridge used by ctx.runPlay and
+        // parent terminal signals. This keeps scalar child plays inline with
+        // the parent instead of round-tripping through nested Workflow waits.
+        ...makeCoordinatorControlBinding(),
         // NOTE: We intentionally do NOT pass `env.PLAYS_BUCKET` (an R2Bucket
         // binding) through to the per-play Worker's env. Including a raw
         // R2Bucket in the dynamically-loaded Worker's env makes Cloudflare
@@ -2836,7 +2952,7 @@ async function loadDynamicPlayWorker(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h5-runtime-api-coordinator`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2858,10 +2974,11 @@ async function loadDynamicPlayWorker(
         // Mirror of the sync loader (above) — see that copy for the
         // architectural rationale. The dynamic worker env is intentionally
         // minimal; runtime callbacks use RUNTIME_API, file reads go through
-        // HARNESS, and child workflow control uses the coordinator URL in the
-        // run request.
+        // HARNESS, and child workflow control uses the COORDINATOR binding.
         HARNESS: env.HARNESS,
-        RUNTIME_API: makeRuntimeApiBinding(),
+        VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
+        ...makeRuntimeApiEnvBinding(),
+        ...makeCoordinatorControlBinding(),
       },
     };
   });
@@ -3169,11 +3286,16 @@ async function handleCoordinatorWarmup(
  * structured-cloneable. WorkerEntrypoint stubs ARE cloneable — same trick
  * `makePlayAssetsBinding` already uses.
  *
- * Falls back transparently in legacy coordinators: if the binding isn't
- * present in `env`, the harness uses its existing `fetch(req.baseUrl + path)`
- * path, which still works (just slower).
+ * Falls back transparently when Cloudflare does not expose module exports in
+ * the current execution path: if the binding is omitted from `env`, the play
+ * worker uses its existing `fetch(req.baseUrl + path)` transport.
  */
-function makeRuntimeApiBinding(): { fetch(req: Request): Promise<Response> } {
+let loggedMissingRuntimeApiExport = false;
+let loggedMissingCoordinatorControlExport = false;
+
+function makeRuntimeApiEnvBinding():
+  | { RUNTIME_API: { fetch(req: Request): Promise<Response> } }
+  | Record<string, never> {
   const exports = workersExports as unknown as {
     RuntimeApi?: (init: { props: undefined }) => {
       fetch(req: Request): Promise<Response>;
@@ -3181,27 +3303,35 @@ function makeRuntimeApiBinding(): { fetch(req: Request): Promise<Response> } {
   };
   const ctor = exports.RuntimeApi;
   if (typeof ctor !== 'function') {
-    throw new Error(
-      'RuntimeApi is not registered on cloudflare:workers exports.',
-    );
+    if (!loggedMissingRuntimeApiExport) {
+      loggedMissingRuntimeApiExport = true;
+      console.warn(
+        '[coordinator] RuntimeApi is not registered on cloudflare:workers exports; using public runtime API transport.',
+      );
+    }
+    return {};
   }
-  return ctor({ props: undefined });
+  return { RUNTIME_API: ctor({ props: undefined }) };
 }
 
-function makeCoordinatorControlBinding(): {
-  submitChild(
-    parentRunId: string,
-    body: Record<string, unknown>,
-  ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
-  signal(
-    runId: string,
-    body: Record<string, unknown>,
-  ): Promise<Record<string, unknown>>;
-  recordPerfTrace(
-    runId: string,
-    payload: CoordinatorPerfTracePayload,
-  ): Promise<void>;
-} {
+function makeCoordinatorControlBinding():
+  | {
+      COORDINATOR: {
+        submitChild(
+          parentRunId: string,
+          body: Record<string, unknown>,
+        ): Promise<{ workflowId?: string; runId?: string; error?: unknown }>;
+        signal(
+          runId: string,
+          body: Record<string, unknown>,
+        ): Promise<Record<string, unknown>>;
+        recordPerfTrace(
+          runId: string,
+          payload: CoordinatorPerfTracePayload,
+        ): Promise<void>;
+      };
+    }
+  | Record<string, never> {
   const exports = workersExports as unknown as {
     CoordinatorControl?: (init: { props: undefined }) => {
       submitChild(
@@ -3220,11 +3350,15 @@ function makeCoordinatorControlBinding(): {
   };
   const ctor = exports.CoordinatorControl;
   if (typeof ctor !== 'function') {
-    throw new Error(
-      'CoordinatorControl is not registered on cloudflare:workers exports.',
-    );
+    if (!loggedMissingCoordinatorControlExport) {
+      loggedMissingCoordinatorControlExport = true;
+      console.warn(
+        '[coordinator] CoordinatorControl is not registered on cloudflare:workers exports; using public coordinator transport.',
+      );
+    }
+    return {};
   }
-  return ctor({ props: undefined });
+  return { COORDINATOR: ctor({ props: undefined }) };
 }
 
 async function loadStoredPlayArtifactFromR2(
@@ -3261,6 +3395,9 @@ function mapWorkflowResult(
       : null;
   return {
     runId,
+    ...(typeof output?.playName === 'string' && output.playName.trim()
+      ? { playName: output.playName.trim() }
+      : {}),
     status: mapped,
     result: output?.result ?? status.output ?? null,
     error,