deepline 0.1.179 → 0.1.180

package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts

@@ -44,7 +44,11 @@ import type {
   PlayRuntimeManifest,
   PlayRuntimeManifestMap,
 } from '../../../shared_libs/plays/compiler-manifest';
-import { normalizePlayRunFailure } from '../../../shared_libs/play-runtime/run-failure';
+import {
+  WORKSPACE_STORAGE_NOT_READY_CODE,
+  WORKSPACE_STORAGE_NOT_READY_MESSAGE,
+  normalizePlayRunFailure,
+} from '../../../shared_libs/play-runtime/run-failure';
 import type { PlayRunLedgerEvent } from '../../../shared_libs/play-runtime/run-ledger';
 import {
   COORDINATOR_INTERNAL_TOKEN_HEADER,
@@ -3152,11 +3156,59 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
       });
     }
     let dispatchedEvent = event;
-    dispatchedEvent = await hydrateWorkflowDbSessions({
-      env: this.env,
-      event: dispatchedEvent,
-      trace,
-    });
+    try {
+      dispatchedEvent = await hydrateWorkflowDbSessions({
+        env: this.env,
+        event: dispatchedEvent,
+        trace,
+      });
+    } catch (hydrateError) {
+      // The workflow re-entered but cannot read its externalized DB sessions.
+      // A 410 (expired retention window) is known-permanent: the sessions are
+      // stored once at submit and never re-minted here, so the workflow can
+      // only keep failing. Publish a clean run.failed instead of letting the
+      // raw transport error escape uncaught — this call sits OUTSIDE the
+      // dispatchWorkflow try/catch that authors run.failed.
+      //
+      // A 404 that survived the read-retry ladder (`workflow db session
+      // lookup failed 404`) rethrows WITHOUT marking, like every other error:
+      // sessions are persisted (awaited) before the instance is created, so a
+      // 404 here is anomalous — most plausibly a framework re-entry against an
+      // ALREADY-TERMINAL run whose DO state the post-finish alarm cleaned up.
+      // Authoring run.failed there could flip a completed (billed + durable)
+      // run to failed via newest-terminal-wins. Only the 410 is safe to mark.
+      const hydrateMessage =
+        hydrateError instanceof Error
+          ? hydrateError.message
+          : String(hydrateError);
+      if (hydrateMessage.includes('workflow db session lookup failed 410')) {
+        await markWorkflowRuntimeFailure({
+          env: this.env,
+          event: dispatchedEvent,
+          error: hydrateError,
+        }).catch((markError) => {
+          console.error(
+            '[coordinator] failed to forward expired-db-session run failure',
+            {
+              runId: entryTrace.runId,
+              message:
+                markError instanceof Error
+                  ? markError.message
+                  : String(markError),
+            },
+          );
+        });
+        if (entryTrace.runId) {
+          await writeCoordinatorTerminalState(this.env, {
+            runId: entryTrace.runId,
+            status: 'failed',
+            error: normalizeCoordinatorWorkflowRuntimeFailure(hydrateError)
+              .message,
+          }).catch(() => undefined);
+        }
+      }
+      throw hydrateError;
+    }
     const dispatchTrace = readWorkflowTraceContext(dispatchedEvent);
     trace({
       runId: dispatchTrace.runId,
@@ -3685,6 +3737,35 @@ function formatTailLogPart(value: unknown): string {
   }
 }
 
+/**
+ * Recognize the tenant-Postgres degradation signatures that used to escape the
+ * coordinator submit path as an opaque 500 "internal error" with no repair
+ * instruction: revoked CONNECT / grant drift, connect stalls, and NeonDbError.
+ * Conservative on purpose — anything not matched stays a generic internal
+ * error (with its phase) so we do not mislabel unrelated failures.
+ */
+function isTenantStorageDegradationError(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return (
+    /permission denied (?:for|to grant) (?:schema|table|relation|database|sequence|role)\b/i.test(
+      message,
+    ) ||
+    /must be owner of (?:table|relation|sequence|schema|database|function|procedure|routine|type)\b/i.test(
+      message,
+    ) ||
+    /Runtime Postgres connection timed out/i.test(message) ||
+    /\bNeonDbError\b/i.test(message) ||
+    /timeout exceeded when trying to connect/i.test(message) ||
+    /connection terminated unexpectedly/i.test(message)
+    // Deliberately NOT matching bare socket/DNS codes (ECONNRESET, ETIMEDOUT,
+    // ECONNREFUSED, EAI_AGAIN): those are thrown by Cloudflare service bindings,
+    // the Workflow API, and Durable Object fetches too, not just tenant
+    // Postgres. Matching them here would mislabel a transient CF-infra blip as
+    // WORKSPACE_STORAGE_NOT_READY ("run db repair"). Every remaining pattern
+    // carries explicit Postgres/Neon context.
+  );
+}
+
 function coordinatorWorkflowRouteErrorResponse(input: {
   runId: string;
   action: string;
@@ -3692,17 +3773,46 @@ function coordinatorWorkflowRouteErrorResponse(input: {
 }): Response {
   const message =
     input.error instanceof Error ? input.error.message : String(input.error);
+  const phase = `coordinator.${input.action || 'unknown'}`;
   console.error('[coordinator.workflow_route.error]', {
     runId: input.runId,
     action: input.action,
     error: message,
   });
+  if (input.action === 'submit' && isTenantStorageDegradationError(input.error)) {
+    // De-redact: a tenant-storage failure gets the loud, actionable
+    // WORKSPACE_STORAGE_NOT_READY message + repair instruction instead of the
+    // generic "internal error". Same contract as the runtime route.
+    //
+    // Scoped to `submit` on purpose: the message states "nothing was billed",
+    // which is only true in the pre-run submit phase. The other workflow
+    // actions (status/result/tail/cancel/signal) run against an already-active,
+    // possibly-billing run, so surfacing "storage not ready, nothing billed,
+    // run db repair" there would be false and alarming on a healthy run.
+    console.error('[coordinator.workflow_route.storage_not_ready]', {
+      runId: input.runId,
+      action: input.action,
+      phase,
+      error: message,
+    });
+    return Response.json(
+      {
+        error: {
+          code: WORKSPACE_STORAGE_NOT_READY_CODE,
+          message: WORKSPACE_STORAGE_NOT_READY_MESSAGE,
+          phase,
+          runId: input.runId,
+        },
+      },
+      { status: 503 },
+    );
+  }
   return Response.json(
     {
       error: {
         code: 'COORDINATOR_WORKFLOW_ROUTE_FAILED',
         message,
-        phase: `coordinator.${input.action || 'unknown'}`,
+        phase,
         runId: input.runId,
       },
     },
@@ -3869,6 +3979,7 @@ async function handleWorkflowRoute(input: {
           });
         })
         .catch((error: unknown) => {
+          const storageNotReady = isTenantStorageDegradationError(error);
           recordSubmitTiming({
             phase: 'coordinator.harness_prewarm_postgres',
             ms: Date.now() - prewarmStartedAt,
@@ -3877,8 +3988,21 @@ async function handleWorkflowRoute(input: {
               status: 'failed',
               sessions: preloadedDbSessions.length,
               error: error instanceof Error ? error.message : String(error),
+              storageNotReady,
             },
           });
+          // The prewarm runs in waitUntil, so it cannot fail the already-issued
+          // submit. But its cause used to be fully redacted in coordinator
+          // logs. Log the tenant-storage classification loudly so the real
+          // cause (and the WORKSPACE_STORAGE_NOT_READY repair path) is visible.
+          if (storageNotReady) {
+            console.error('[coordinator.harness_prewarm_postgres.storage_not_ready]', {
+              runId: submittedRunId,
+              playName: params.playName,
+              code: WORKSPACE_STORAGE_NOT_READY_CODE,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
         });
       input.ctx?.waitUntil(prewarmPromise);
     }

package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts

@@ -17,7 +17,10 @@
  */
 
 import { sanitizeLiveLogLines } from './runtime/live-progress';
-import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
+import {
+  DB_SESSION_MAX_TTL_SECONDS,
+  type PreloadedRuntimeDbSession,
+} from '../../../shared_libs/play-runtime/db-session';
 
 type DurableObjectStorage = {
   get<T>(key: string): Promise<T | undefined>;
@@ -259,7 +262,12 @@ const COORDINATOR_RUN_EVENT_MAX_ENTRIES = 500;
 const FINISH_ALARM_DELAY_MS = 60_000; // self-evict 1 min after finish() called
 const WORKFLOW_RUN_STATE_TTL_MS = 60 * 60 * 1000;
 const WORKFLOW_RUN_RETRY_STATE_MAX_BYTES = 110_000;
-const WORKFLOW_DB_SESSIONS_TTL_MS = 10 * 60_000;
+// Invariant: DB sessions must not expire before the workflow run/retry state.
+// hydrateWorkflowDbSessions re-reads sessions on every workflow re-entry, so a
+// shorter session TTL used to kill any run that ran longer than the session
+// window (10 min) with a misleading transport 410.
+const WORKFLOW_DB_SESSIONS_TTL_MS = WORKFLOW_RUN_STATE_TTL_MS;
+const WORKFLOW_DB_SESSIONS_MAX_TTL_MS = DB_SESSION_MAX_TTL_SECONDS * 1000;
 
 function jsonByteLength(value: unknown): number {
   return new TextEncoder().encode(JSON.stringify(value)).byteLength;
@@ -673,7 +681,7 @@ export class PlayDedup implements DurableObject {
       body.dbSessionsTtlMs > 0
         ? Math.min(
             Math.max(Math.floor(body.dbSessionsTtlMs), 60_000),
-            30 * 60_000,
+            WORKFLOW_DB_SESSIONS_MAX_TTL_MS,
           )
         : WORKFLOW_DB_SESSIONS_TTL_MS;
     const retryKey = `${WORKFLOW_RUN_RETRY_KEY_PREFIX}${runId}`;
@@ -933,7 +941,10 @@ export class PlayDedup implements DurableObject {
       typeof body?.ttlMs === 'number' &&
       Number.isFinite(body.ttlMs) &&
       body.ttlMs > 0
-        ? Math.min(Math.max(Math.floor(body.ttlMs), 60_000), 30 * 60_000)
+        ? Math.min(
+            Math.max(Math.floor(body.ttlMs), 60_000),
+            WORKFLOW_DB_SESSIONS_MAX_TTL_MS,
+          )
         : WORKFLOW_DB_SESSIONS_TTL_MS;
     const state: WorkflowDbSessionsState = {
       runId,
@@ -969,10 +980,16 @@ export class PlayDedup implements DurableObject {
     }
     if (state.expiresAt <= Date.now()) {
       await this.state.storage.delete(WORKFLOW_DB_SESSIONS_KEY);
-      return new Response(JSON.stringify({ error: 'db sessions expired' }), {
-        status: 410,
-        headers: { 'content-type': 'application/json' },
-      });
+      return new Response(
+        JSON.stringify({
+          error:
+            'workflow db sessions expired: the run exceeded the session retention window before completing. This usually follows an earlier runtime persistence failure or a stalled run; re-run to resume.',
+        }),
+        {
+          status: 410,
+          headers: { 'content-type': 'application/json' },
+        },
+      );
     }
     assertEncryptedDbSessionsForStorage(state.sessions);
     return new Response(

package/dist/bundling-sources/apps/play-runner-workers/src/durable-object-deploy-handoff.ts

@@ -1,13 +1,19 @@
 export const DURABLE_OBJECT_DEPLOY_HANDOFF_STORAGE_ERROR =
   "The Durable Object's code has been updated, this version can no longer access storage";
 
-export const DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS = [50, 150, 350];
+export const DURABLE_OBJECT_DEPLOY_HANDOFF_WORKER_NOT_FOUND_ERROR =
+  'Worker not found.';
+
+export const DURABLE_OBJECT_DEPLOY_HANDOFF_RETRY_DELAYS_MS = [
+  250, 750, 1_500, 3_000, 5_000, 8_000,
+];
 
 export function durableObjectDeployHandoffMessage(
   value: unknown,
 ): string | null {
   const message = value instanceof Error ? value.message : String(value ?? '');
-  return message.includes(DURABLE_OBJECT_DEPLOY_HANDOFF_STORAGE_ERROR)
+  return message.includes(DURABLE_OBJECT_DEPLOY_HANDOFF_STORAGE_ERROR) ||
+    message.includes(DURABLE_OBJECT_DEPLOY_HANDOFF_WORKER_NOT_FOUND_ERROR)
     ? message
     : null;
 }