deepline 0.1.32 → 0.1.35

package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts

@@ -33,6 +33,10 @@ import {
   COORDINATOR_INTERNAL_TOKEN_HEADER,
   COORDINATOR_RUN_SCOPE_HEADER,
 } from '../../../shared_libs/play-runtime/coordinator-headers';
+import {
+  decideWorkflowPlatformRetry,
+  PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
+} from './workflow-retry';
 import { sanitizeLiveLogLines } from './runtime/live-progress';
 
 export { DynamicWorkflowBinding };
@@ -551,6 +555,11 @@ const WORKFLOW_POOL_START_ACK_TIMEOUT_MS = 750;
 const WORKFLOW_POOL_START_ACK_POLL_MS = 25;
 const SUBMIT_INITIAL_STATE_MAX_WAIT_MS = 0;
 const SUBMIT_INITIAL_STATE_POLL_MS = 50;
+const WORKFLOW_RETRY_STATE_TTL_MS = 60 * 60 * 1000;
+const WORKFLOW_POOL_PREWARM_ESCALATE_TARGET_AFTER_MS = 250;
+const WORKFLOW_POOL_SCHEDULED_REFILL_MIN_AVAILABLE = 1;
+const WORKFLOW_POOL_SCHEDULED_REFILL_TIMEOUT_MS = 10_000;
+
 function buildDynamicWorkflowMetadata(
   params: PlayWorkflowParams,
 ): DynamicWorkflowMetadata {
@@ -954,6 +963,176 @@ async function readWorkflowPoolRunMapping(input: {
   };
 }
 
+async function persistWorkflowRetryState(input: {
+  env: CoordinatorEnv;
+  runId: string;
+  params: PlayWorkflowParams;
+}): Promise<void> {
+  const retryParams: PlayWorkflowParams = {
+    ...input.params,
+    dynamicWorkerCode: null,
+    contractSnapshot: stripRetrySourceSnapshot(input.params.contractSnapshot),
+    childPlayManifests: stripRetryChildManifestCode(
+      input.params.childPlayManifests,
+    ),
+    packagedFiles: input.params.packagedFiles?.map((file) => ({
+      playPath: file.playPath,
+      storageKey: file.storageKey,
+      contentType: file.contentType,
+      bytes: file.bytes,
+    })) ?? null,
+  };
+  await callWorkflowPool<{ ok?: unknown }>(input.env, '/run-retry-state-put', {
+    method: 'POST',
+    body: JSON.stringify({
+      runId: input.runId,
+      params: retryParams,
+      ttlMs: WORKFLOW_RETRY_STATE_TTL_MS,
+    }),
+  }).catch((error) => {
+    console.warn('[coordinator] workflow retry state persistence skipped', {
+      runId: input.runId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  });
+}
+
+function stripRetrySourceSnapshot(snapshot: unknown): unknown {
+  if (!isRecord(snapshot)) return snapshot;
+  const rest = { ...snapshot };
+  delete rest.sourceCode;
+  delete rest.sourceFiles;
+  return rest;
+}
+
+function stripRetryChildManifestCode(
+  manifests: PlayRuntimeManifestMap | null | undefined,
+): PlayRuntimeManifestMap | null {
+  if (!manifests) return null;
+  const stripped: PlayRuntimeManifestMap = {};
+  for (const [key, manifest] of Object.entries(manifests)) {
+    const rest = { ...manifest };
+    delete rest.bundledCode;
+    delete rest.sourceCode;
+    stripped[key] = rest;
+  }
+  return stripped;
+}
+
+async function claimWorkflowPlatformRetry(input: {
+  env: CoordinatorEnv;
+  runId: string;
+}): Promise<{
+  claimed: boolean;
+  attempts: number;
+  params: PlayWorkflowParams | null;
+}> {
+  const body = await callWorkflowPool<{
+    claimed?: unknown;
+    attempts?: unknown;
+    params?: unknown;
+  }>(input.env, '/run-retry-claim', {
+    method: 'POST',
+    body: JSON.stringify({
+      runId: input.runId,
+      maxAttempts: PLATFORM_DEPLOY_WORKFLOW_RETRY_LIMIT,
+    }),
+  });
+  return {
+    claimed: body.claimed === true,
+    attempts: typeof body.attempts === 'number' ? body.attempts : 0,
+    params: isRecord(body.params) ? (body.params as PlayWorkflowParams) : null,
+  };
+}
+
+function workflowRetryInstanceId(runId: string, attempt: number): string {
+  const suffix = `retry-${attempt}-${stableHash(`${runId}:${attempt}`).slice(0, 8)}`;
+  const base = workflowInstanceId(runId);
+  const maxBaseLength = Math.max(1, 100 - suffix.length - 1);
+  return `${base.slice(0, maxBaseLength)}-${suffix}`;
+}
+
+async function restartWorkflowAfterPlatformReset(input: {
+  env: CoordinatorEnv;
+  ctx?: ExecutionContext;
+  runId: string;
+  oldInstance: WorkflowInstance;
+  error: string;
+  status: InstanceStatus;
+}): Promise<{
+  retried: boolean;
+  result: Record<string, unknown> | null;
+}> {
+  const decision = decideWorkflowPlatformRetry({
+    workflowStatus: String(input.status.status ?? ''),
+    error: input.error,
+    retryAttempts: 0,
+  });
+  if (decision.action !== 'retry') {
+    return { retried: false, result: null };
+  }
+  const claim = await claimWorkflowPlatformRetry({
+    env: input.env,
+    runId: input.runId,
+  }).catch((error) => {
+    console.warn('[coordinator] workflow platform retry claim failed', {
+      runId: input.runId,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return null;
+  });
+  if (!claim?.claimed || !claim.params) {
+    return { retried: false, result: null };
+  }
+  const retryInstanceId = workflowRetryInstanceId(input.runId, claim.attempts);
+  const retryStartedAt = Date.now();
+  let retryInstance: WorkflowInstance | null = null;
+  try {
+    retryInstance = await createDynamicWorkflowInstance({
+      env: input.env,
+      id: retryInstanceId,
+      params: claim.params,
+    });
+    await mapRunToWorkflowInstance({
+      env: input.env,
+      runId: input.runId,
+      instanceId: retryInstance.id,
+      started: true,
+    });
+    input.ctx?.waitUntil(
+      input.oldInstance.terminate().catch(() => undefined),
+    );
+    recordCoordinatorPerfTraceBuffered(input.env, input.ctx, {
+      runId: input.runId,
+      phase: 'coordinator.platform_deploy_retry',
+      ms: Date.now() - retryStartedAt,
+      graphHash: claim.params.graphHash ?? null,
+      extra: {
+        retryAttempt: claim.attempts,
+        retryInstanceId: retryInstance.id,
+        reason: decision.reason,
+      },
+    });
+    return {
+      retried: true,
+      result: {
+        runId: input.runId,
+        playName: claim.params.playName,
+        status: 'running',
+        result: null,
+        error: null,
+        retry: {
+          reason: decision.reason,
+          attempt: claim.attempts,
+          message: decision.message,
+        },
+      },
+    };
+  } finally {
+    disposeRpcStub(retryInstance);
+  }
+}
+
 async function waitForWorkflowPoolStartAck(input: {
   env: CoordinatorEnv;
   runId: string;
@@ -2979,9 +3158,14 @@ async function handleWorkflowRoute(input: {
               error: error instanceof Error ? error.message : String(error),
             },
           });
-        });
+      });
       input.ctx?.waitUntil(prewarmPromise);
     }
+    await persistWorkflowRetryState({
+      env,
+      runId: submittedRunId,
+      params,
+    });
     let instance: WorkflowInstance | null = null;
     try {
       const statusEventStartedAt = Date.now();
@@ -3500,6 +3684,33 @@ async function handleWorkflowRoute(input: {
         });
       }
       const status = await instance.status();
+      const workflowError = readWorkflowError(status);
+      if (workflowError) {
+        const retry = await restartWorkflowAfterPlatformReset({
+          env,
+          ctx: input.ctx,
+          runId,
+          oldInstance: instance,
+          error: workflowError,
+          status,
+        });
+        if (retry.retried && retry.result) {
+          const coordinatorTrace = includeTrace
+            ? await listCoordinatorPerfTrace(env, runId).catch(() => [])
+            : [];
+          return Response.json({
+            ...retry.result,
+            coordinatorObserve: {
+              ms: Date.now() - statusStartedAt,
+              waitMs: 0,
+              workflowStatus: 'retrying',
+              statusPolls: 1,
+              instanceId: instance.id,
+            },
+            ...(includeTrace ? { coordinatorTrace } : {}),
+          });
+        }
+      }
       const result = mapWorkflowResult(runId, status);
       const observeMs = Date.now() - statusStartedAt;
       // If we forced a permanent-error fail-fast (status='failed' even

package/dist/repo/apps/play-runner-workers/src/dedup-do.ts

@@ -75,6 +75,14 @@ type WorkflowRunMapping = {
   expiresAt: number;
 };
 
+type WorkflowRunRetryState = {
+  runId: string;
+  params: unknown;
+  retryAttempts: number;
+  updatedAt: number;
+  expiresAt: number;
+};
+
 type CoordinatorTraceEntry = {
   ts: number;
   source: 'coordinator' | 'dynamic_worker';
@@ -157,6 +165,7 @@ type WorkflowPoolCounts = {
 const DEDUP_KEY_PREFIX = 'd:';
 const WORKFLOW_POOL_KEY_PREFIX = 'p:';
 const WORKFLOW_POOL_RUN_KEY_PREFIX = 'm:';
+const WORKFLOW_RUN_RETRY_KEY_PREFIX = 'r:';
 const COORDINATOR_TRACE_KEY_PREFIX = 't:';
 const COORDINATOR_RUN_EVENT_KEY_PREFIX = 'e:';
 const COORDINATOR_TERMINAL_KEY = 'terminal';
@@ -167,6 +176,11 @@ const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
 const WORKFLOW_POOL_DEFAULT_TTL_MS = 8 * 60 * 1000;
 const WORKFLOW_POOL_RUN_MAPPING_TTL_MS = 60 * 60 * 1000;
 const WORKFLOW_POOL_READY_MAX_AGE_MS = 7 * 60_000;
+const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
+
+function jsonByteLength(value: unknown): number {
+  return new TextEncoder().encode(JSON.stringify(value)).byteLength;
+}
 
 interface DedupEnv {
   PLAY_DEDUP: DurableObjectNamespace;
@@ -234,6 +248,10 @@ export class PlayDedup implements DurableObject {
           return await this.handlePoolBlockRun(req);
         case '/pool-resolve-run':
           return await this.handlePoolResolveRun(req);
+        case '/run-retry-state-put':
+          return await this.handleRunRetryStatePut(req);
+        case '/run-retry-claim':
+          return await this.handleRunRetryClaim(req);
         case '/pool-clear':
           return await this.handlePoolClear(req);
         case '/trace-add':
@@ -466,13 +484,16 @@ export class PlayDedup implements DurableObject {
     now = Date.now(),
     version?: string,
   ): Promise<void> {
-    const [pool, mappings] = await Promise.all([
+    const [pool, mappings, retries] = await Promise.all([
       this.state.storage.list<WorkflowPoolEntry>({
         prefix: WORKFLOW_POOL_KEY_PREFIX,
       }),
       this.state.storage.list<WorkflowRunMapping>({
         prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
       }),
+      this.state.storage.list<WorkflowRunRetryState>({
+        prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
+      }),
     ]);
     const expiredKeys: string[] = [];
     for (const [key, entry] of pool) {
@@ -496,6 +517,11 @@ export class PlayDedup implements DurableObject {
         expiredKeys.push(key);
       }
     }
+    for (const [key, retryState] of retries) {
+      if (!retryState || retryState.expiresAt <= now) {
+        expiredKeys.push(key);
+      }
+    }
     if (expiredKeys.length > 0) {
       await this.state.storage.delete(expiredKeys);
     }
@@ -963,15 +989,115 @@ export class PlayDedup implements DurableObject {
     );
   }
 
+  private async handleRunRetryStatePut(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      params?: unknown;
+      ttlMs?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    if (!runId || !body || !('params' in body)) {
+      return new Response('runId and params are required', { status: 400 });
+    }
+    const now = Date.now();
+    const ttlMs =
+      typeof body.ttlMs === 'number' && Number.isFinite(body.ttlMs)
+        ? Math.max(
+            60_000,
+            Math.min(body.ttlMs, WORKFLOW_POOL_RUN_MAPPING_TTL_MS),
+          )
+        : WORKFLOW_POOL_RUN_MAPPING_TTL_MS;
+    const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
+    await this.state.blockConcurrencyWhile(async () => {
+      const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
+      const retryState = {
+        runId,
+        params: body.params,
+        retryAttempts:
+          existing?.runId === runId &&
+          typeof existing.retryAttempts === 'number'
+            ? existing.retryAttempts
+            : 0,
+        updatedAt: now,
+        expiresAt: now + ttlMs,
+      } satisfies WorkflowRunRetryState;
+      const bytes = jsonByteLength(retryState);
+      if (bytes > WORKFLOW_RUN_RETRY_STATE_MAX_BYTES) {
+        throw new Error(
+          `workflow retry state too large: ${bytes} bytes exceeds ${WORKFLOW_RUN_RETRY_STATE_MAX_BYTES}`,
+        );
+      }
+      await this.state.storage.put(key, retryState);
+    });
+    return new Response(JSON.stringify({ ok: true }), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
+  private async handleRunRetryClaim(req: Request): Promise<Response> {
+    const body = (await req.json().catch(() => null)) as {
+      runId?: unknown;
+      maxAttempts?: unknown;
+    } | null;
+    const runId = typeof body?.runId === 'string' ? body.runId : '';
+    if (!runId) {
+      return new Response('runId is required', { status: 400 });
+    }
+    const maxAttempts =
+      typeof body?.maxAttempts === 'number' && Number.isFinite(body.maxAttempts)
+        ? Math.max(0, Math.floor(body.maxAttempts))
+        : 1;
+    const now = Date.now();
+    const key = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
+    let response: Record<string, unknown> = {
+      claimed: false,
+      attempts: 0,
+      params: null,
+    };
+    await this.state.blockConcurrencyWhile(async () => {
+      const existing = await this.state.storage.get<WorkflowRunRetryState>(key);
+      if (!existing || existing.expiresAt <= now) {
+        response = { claimed: false, attempts: 0, params: null };
+        return;
+      }
+      if (existing.retryAttempts >= maxAttempts) {
+        response = {
+          claimed: false,
+          attempts: existing.retryAttempts,
+          params: existing.params,
+        };
+        return;
+      }
+      const nextAttempts = existing.retryAttempts + 1;
+      await this.state.storage.put(key, {
+        ...existing,
+        retryAttempts: nextAttempts,
+        updatedAt: now,
+        expiresAt: now + WORKFLOW_POOL_RUN_MAPPING_TTL_MS,
+      } satisfies WorkflowRunRetryState);
+      response = {
+        claimed: true,
+        attempts: nextAttempts,
+        params: existing.params,
+      };
+    });
+    return new Response(JSON.stringify(response), {
+      headers: { 'content-type': 'application/json' },
+    });
+  }
+
   private async handlePoolClear(req: Request): Promise<Response> {
     const version = this.workflowPoolVersion(req);
-    const [pool, mappings] = await Promise.all([
+    const [pool, mappings, retries] = await Promise.all([
       this.state.storage.list<WorkflowPoolEntry>({
         prefix: WORKFLOW_POOL_KEY_PREFIX,
       }),
       this.state.storage.list<WorkflowRunMapping>({
         prefix: WORKFLOW_POOL_RUN_KEY_PREFIX,
       }),
+      this.state.storage.list<WorkflowRunRetryState>({
+        prefix: WORKFLOW_RUN_RETRY_KEY_PREFIX,
+      }),
     ]);
     const keys = [
       ...[...pool.entries()]
@@ -980,6 +1106,7 @@ export class PlayDedup implements DurableObject {
       ...[...mappings.entries()]
         .filter(([, entry]) => !version || entry.version === version)
         .map(([key]) => key),
+      ...[...retries.keys()],
     ];
     if (keys.length > 0) {
       await this.state.storage.delete(keys);