deepline 0.1.11 → 0.1.19

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -359,13 +359,16 @@ type PooledWorkflowBootstrapPayload = {
   createdAt: number;
 };
 
-const WORKFLOW_POOL_PROTOCOL_VERSION = 'pooled-workflow-wait-v3-dynamic-binding';
+const WORKFLOW_POOL_PROTOCOL_VERSION =
+  'pooled-workflow-wait-v4-waiting-only';
 const WORKFLOW_POOL_DO_NAME = 'workflow-pool:v2';
 const WORKFLOW_POOL_START_EVENT_TYPE = 'play_start';
 const WORKFLOW_POOL_TTL_MS = 8 * 60 * 1000;
 const WORKFLOW_POOL_TARGET_SIZE = 2;
 const WORKFLOW_POOL_READY_TIMEOUT_MS = 1_500;
 const WORKFLOW_POOL_READY_POLL_MS = 250;
+const WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS = 2_500;
+const WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE = 1;
 
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
@@ -675,11 +678,11 @@ function workflowStatusName(status: InstanceStatus | null): string {
 }
 
 function workflowPoolStatusIsReady(statusName: string): boolean {
-  // Cloudflare Workflows may report a run as "running" even while the body is
-  // blocked inside step.waitForEvent(). Treat both states as pool-candidates;
-  // submitViaPooledWorkflow still falls back to a fresh Workflow if sendEvent
-  // fails.
-  return statusName === 'waiting' || statusName === 'running';
+  // Only a Workflow explicitly blocked on waitForEvent is safe to lease.
+  // A generic "running" instance may already be executing a previous play; if
+  // we send a new start event to it, the submitted run can stay running forever
+  // while callbacks continue updating the previous runId.
+  return statusName === 'waiting';
 }
 
 async function waitForWorkflowPoolReady(instance: WorkflowInstance): Promise<{
@@ -900,7 +903,7 @@ async function submitViaPooledWorkflow(input: {
     return null;
   }
   const leaseStartedAt = Date.now();
-  const pooledInstanceId = await leaseWorkflowPoolId(input.env);
+  let pooledInstanceId = await leaseWorkflowPoolId(input.env);
   const missCounts = pooledInstanceId
     ? null
     : await workflowPoolCount(input.env).catch(() => null);
@@ -918,6 +921,47 @@ async function submitViaPooledWorkflow(input: {
         : {}),
     },
   });
+
+  if (!pooledInstanceId) {
+    // A pool miss is often a timing gap rather than a true lack of warm
+    // capacity. Wait briefly for refill/promotion, then retry lease once
+    // before falling back to a cold workflow create.
+    const refillStartedAt = Date.now();
+    const refillResult = await refillWorkflowPool(input.env, {
+      waitReady: true,
+      minAvailable: WORKFLOW_POOL_REFILL_ON_MISS_MIN_AVAILABLE,
+      waitTimeoutMs: WORKFLOW_POOL_REFILL_ON_MISS_TIMEOUT_MS,
+    }).catch(() => null);
+    input.recordSubmitTiming({
+      phase: 'coordinator.workflow_pool_refill_on_miss',
+      ms: Date.now() - refillStartedAt,
+      graphHash: input.params.graphHash ?? null,
+      extra:
+        refillResult === null
+          ? { ok: false }
+          : {
+              ok: true,
+              available: refillResult.available,
+              warming: refillResult.warming,
+              created: refillResult.created,
+              promoted: refillResult.promoted,
+              removed: refillResult.removed,
+              waitedMs: refillResult.waitedMs,
+              waitIterations: refillResult.waitIterations,
+            },
+    });
+    if (refillResult?.available) {
+      const retryStartedAt = Date.now();
+      pooledInstanceId = await leaseWorkflowPoolId(input.env);
+      input.recordSubmitTiming({
+        phase: 'coordinator.workflow_pool_lease_retry',
+        ms: Date.now() - retryStartedAt,
+        graphHash: input.params.graphHash ?? null,
+        extra: { pooled: Boolean(pooledInstanceId) },
+      });
+    }
+  }
+
   if (!pooledInstanceId) {
     return null;
   }
@@ -1497,8 +1541,8 @@ async function executeChildInline(input: {
     artifactStorageKey: manifest.artifactStorageKey,
     artifactHash: manifest.artifactHash,
     dynamicWorkerCode:
-      typeof manifest.dynamicWorkerCode === 'string'
-        ? manifest.dynamicWorkerCode
+      typeof manifest.bundledCode === 'string'
+        ? manifest.bundledCode
         : null,
     packagedFiles: null,
   });
@@ -2451,7 +2495,10 @@ async function handleWorkflowRoute(input: {
                 ? (body.childPlayManifests as PlayRuntimeManifestMap)
                 : null,
               playCallGovernance: governance,
-              dynamicWorkerCode: null,
+              dynamicWorkerCode:
+                typeof manifest.bundledCode === 'string'
+                  ? manifest.bundledCode
+                  : null,
               executorToken: childToken,
               baseUrl: resolveRuntimeBaseUrl(env, body),
               orgId,
@@ -2751,7 +2798,7 @@ function loadDynamicPlayWorkerSync(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h3`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2777,6 +2824,11 @@ function loadDynamicPlayWorkerSync(
         // which case the per-play stub throws a loud error on first use
         // (no silent fallbacks — see harness-stub.ts → requireBinding).
         HARNESS: env.HARNESS,
+        VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
+        // In-process runtime API bridge used by the play harness for status,
+        // tool execution, DB session, and artifact callbacks. This avoids a
+        // public fetch hop when Cloudflare exposes the RuntimeApi export.
+        ...makeRuntimeApiEnvBinding(),
         // NOTE: We intentionally do NOT pass `env.PLAYS_BUCKET` (an R2Bucket
         // binding) through to the per-play Worker's env. Including a raw
         // R2Bucket in the dynamically-loaded Worker's env makes Cloudflare
@@ -2828,7 +2880,7 @@ async function loadDynamicPlayWorker(
   }
   const artifactIdentity =
     metadata.artifactHash?.trim() || stableHash(artifactStorageKey);
-  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h3`;
+  const workerCacheKey = `play:${graphHash}:${artifactIdentity}:harness=h4-runtime-api`;
   const runIdForTrace = metadata.runId ?? graphHash;
   const loaderGetStartedAt = Date.now();
   const stub = env.LOADER.get(workerCacheKey, async () => {
@@ -2849,9 +2901,12 @@ async function loadDynamicPlayWorker(
       env: {
         // Mirror of the sync loader (above) — see that copy for the
         // architectural rationale. The dynamic worker env is intentionally
-        // minimal; runtime callbacks and file reads go through HARNESS, and
-        // child workflow control uses the coordinator URL in the run request.
+        // minimal; runtime callbacks use RUNTIME_API, file reads go through
+        // HARNESS, and child workflow control uses the coordinator URL in the
+        // run request.
         HARNESS: env.HARNESS,
+        VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
+        ...makeRuntimeApiEnvBinding(),
       },
     };
   });
@@ -3159,11 +3214,15 @@ async function handleCoordinatorWarmup(
  * structured-cloneable. WorkerEntrypoint stubs ARE cloneable — same trick
  * `makePlayAssetsBinding` already uses.
  *
- * Falls back transparently in legacy coordinators: if the binding isn't
- * present in `env`, the harness uses its existing `fetch(req.baseUrl + path)`
- * path, which still works (just slower).
+ * Falls back transparently when Cloudflare does not expose module exports in
+ * the current execution path: if the binding is omitted from `env`, the play
+ * worker uses its existing `fetch(req.baseUrl + path)` transport.
  */
-function makeRuntimeApiBinding(): { fetch(req: Request): Promise<Response> } {
+let loggedMissingRuntimeApiExport = false;
+
+function makeRuntimeApiEnvBinding():
+  | { RUNTIME_API: { fetch(req: Request): Promise<Response> } }
+  | Record<string, never> {
   const exports = workersExports as unknown as {
     RuntimeApi?: (init: { props: undefined }) => {
       fetch(req: Request): Promise<Response>;
@@ -3171,11 +3230,15 @@ function makeRuntimeApiBinding(): { fetch(req: Request): Promise<Response> } {
   };
   const ctor = exports.RuntimeApi;
   if (typeof ctor !== 'function') {
-    throw new Error(
-      'RuntimeApi is not registered on cloudflare:workers exports.',
-    );
+    if (!loggedMissingRuntimeApiExport) {
+      loggedMissingRuntimeApiExport = true;
+      console.warn(
+        '[coordinator] RuntimeApi is not registered on cloudflare:workers exports; using public runtime API transport.',
+      );
+    }
+    return {};
   }
-  return ctor({ props: undefined });
+  return { RUNTIME_API: ctor({ props: undefined }) };
 }
 
 function makeCoordinatorControlBinding(): {
@@ -3251,6 +3314,9 @@ function mapWorkflowResult(
       : null;
   return {
     runId,
+    ...(typeof output?.playName === 'string' && output.playName.trim()
+      ? { playName: output.playName.trim() }
+      : {}),
     status: mapped,
     result: output?.result ?? status.output ?? null,
     error,