deepline 0.3.258 → 0.3.259

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts

@@ -14,7 +14,10 @@ import {
   readPostgresSchedulerOutboxNextDueAt,
   type PostgresSchedulerOptions,
   type PostgresSchedulerOutboxProjectionResult,
+  type PostgresSchedulerQueryClient,
+  type RunLedgerOutboxDeliveredRow,
   type RunLedgerOutboxHeadPriority,
+  type RunLedgerOutboxTokenAuthority,
 } from './postgres';
 import {
   acquirePgNotifyHub,
@@ -95,9 +98,22 @@ export type RunLedgerOutboxRecoveryController = {
   wake: (reason?: string) => void;
   /** Whether the optional direct LISTEN hint is currently usable. */
   notifyState: () => PgNotifyHubState | 'unavailable';
+  /**
+   * Liveness facts for the drain loop itself, independent of whether the
+   * app control plane is reachable. A projection retry is a settled dispatch;
+   * only a dispatch that never settles makes the drain look stuck.
+   */
+  drainHealth: () => RunLedgerOutboxDrainHealth;
   close: () => Promise<void>;
 };
 
+export type RunLedgerOutboxDrainHealth = {
+  inFlightDispatches: number;
+  /** Start of the oldest dispatch still in flight, or null when idle. */
+  oldestInFlightStartedAt: number | null;
+  lastDispatchSettledAt: number | null;
+};
+
 function boundedFailureDelayMs(failures: number): number {
   const exponent = Math.min(Math.max(0, failures - 1), 4);
   return Math.min(
@@ -139,14 +155,26 @@ export function startRunLedgerOutboxRecovery(
   let unsubscribeNotify: (() => void) | undefined;
   let unsubscribeState: (() => void) | undefined;
   let requestDrain: (reason: string) => void = () => undefined;
+  const inFlightStartedAt = new Map<number, number>();
+  let nextDispatchId = 0;
+  let lastDispatchSettledAt: number | null = null;
   let admissionTurns = 0;
-
-  const dispatchOnce = () => {
-    const oldestReady =
-      admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
-    admissionTurns = oldestReady ? 0 : admissionTurns + 1;
-    return deps.dispatchOnce(oldestReady ? 'oldest_ready' : 'admission_first');
-  };
+  const dispatchOnce =
+    async (): Promise<PostgresSchedulerOutboxProjectionResult> => {
+      const oldestReady =
+        admissionTurns === ADMISSION_TURNS_PER_OLDEST_READY_TURN;
+      admissionTurns = oldestReady ? 0 : admissionTurns + 1;
+      const id = nextDispatchId++;
+      inFlightStartedAt.set(id, now());
+      try {
+        return await deps.dispatchOnce(
+          oldestReady ? 'oldest_ready' : 'admission_first',
+        );
+      } finally {
+        inFlightStartedAt.delete(id);
+        lastDispatchSettledAt = now();
+      }
+    };
 
   const clearWakeTimer = () => {
     if (!timer) return;
@@ -392,6 +420,17 @@ export function startRunLedgerOutboxRecovery(
   return {
     wake: (reason = 'external_safety_wake') => requestDrain(reason),
     notifyState: () => hubLease?.hub.state() ?? 'unavailable',
+    drainHealth: () => {
+      let oldest: number | null = null;
+      for (const startedAt of inFlightStartedAt.values()) {
+        if (oldest === null || startedAt < oldest) oldest = startedAt;
+      }
+      return {
+        inFlightDispatches: inFlightStartedAt.size,
+        oldestInFlightStartedAt: oldest,
+        lastDispatchSettledAt,
+      };
+    },
     close: async () => {
       if (closed) return;
       closed = true;
@@ -406,6 +445,130 @@ export function startRunLedgerOutboxRecovery(
   };
 }
 
+/**
+ * Latency budget for the inline `run.created` attempt made by an admitting
+ * process after its Runtime Postgres commit. The whole attempt (claim, token,
+ * append, acknowledgement) must settle or be abandoned inside this bound so an
+ * admission response is never held longer than this for projection.
+ */
+export const RUN_CREATED_INLINE_DELIVERY_DEADLINE_MS = 1_500;
+/**
+ * The inline attempt's lease is the scheduler minimum. If the admitting
+ * process is frozen or killed mid-attempt, the durable projector can reclaim
+ * the row after this many seconds instead of the projector's default lease.
+ */
+const RUN_CREATED_INLINE_DELIVERY_LEASE_SECONDS = 5;
+
+export type RunCreatedInlineDeliveryOutcome =
+  | { status: 'delivered'; durationMs: number }
+  /** Already delivered, leased by the projector, not yet due, or not the head. */
+  | { status: 'not_claimed'; durationMs: number }
+  /** Claimed but not acknowledged; the durable projector owns the retry. */
+  | { status: 'retrying' | 'blocked'; durationMs: number }
+  /**
+   * Lease lost mid-attempt and another deliverer acknowledged the row. The
+   * shared idempotency key made any duplicate ledger append a no-op.
+   */
+  | { status: 'superseded'; durationMs: number }
+  | { status: 'timed_out'; durationMs: number }
+  | { status: 'error'; durationMs: number; error: string };
+
+/**
+ * One bounded, idempotent attempt to deliver exactly `run-created:<runId>`
+ * from the admitting process, right after the admission commit.
+ *
+ * ADR-0001: "An inline projection attempt is a latency optimization only."
+ * This uses the projector's own single-row path
+ * (`dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope`): the same
+ * `FOR UPDATE ... SKIP LOCKED` run claim and outbox lease, the same Run Ledger
+ * Projection Contract idempotency key, and the same lease-fenced
+ * acknowledgement. Concurrent inline and projector delivery therefore produce
+ * one ledger admission and one `delivered_at`. The projector remains the
+ * durable owner: every non-delivered outcome leaves the row pending for it.
+ *
+ * Never throws and never exceeds `deadlineMs`; a deadline abandons the
+ * append and releases its lease without charging a retry.
+ */
+export async function deliverRunCreatedOutboxEventInline(input: {
+  runId: string;
+  withClient: <T>(
+    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  tokenAuthority: RunLedgerOutboxTokenAuthority;
+  deadlineMs?: number;
+  now?: () => number;
+}): Promise<RunCreatedInlineDeliveryOutcome> {
+  const now = input.now ?? Date.now;
+  const startedAt = now();
+  const elapsed = () => Math.max(0, now() - startedAt);
+  const deadlineMs = Math.max(
+    1,
+    Math.min(
+      RUN_CREATED_INLINE_DELIVERY_DEADLINE_MS,
+      input.deadlineMs ?? RUN_CREATED_INLINE_DELIVERY_DEADLINE_MS,
+    ),
+  );
+  const runId = input.runId.trim();
+  if (!runId) {
+    return { status: 'error', durationMs: 0, error: 'missing run id' };
+  }
+  const controller = new AbortController();
+  const attempt = dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+    input.withClient,
+    {
+      runId,
+      headEventId: `run-created:${runId}`,
+      limit: 1,
+      leaseSeconds: RUN_CREATED_INLINE_DELIVERY_LEASE_SECONDS,
+      tokenAuthority: input.tokenAuthority,
+      signal: controller.signal,
+    },
+    input.schedulerOptions ?? {},
+  ).then(
+    (result): RunCreatedInlineDeliveryOutcome => {
+      if (result.attempted === 0) {
+        return { status: 'not_claimed', durationMs: elapsed() };
+      }
+      if (result.projected > 0) {
+        return { status: 'delivered', durationMs: elapsed() };
+      }
+      if (controller.signal.aborted) {
+        return { status: 'timed_out', durationMs: elapsed() };
+      }
+      if (result.failed === 0 && result.retrying === 0) {
+        return { status: 'superseded', durationMs: elapsed() };
+      }
+      return {
+        status: result.failed > 0 ? 'blocked' : 'retrying',
+        durationMs: elapsed(),
+      };
+    },
+    (error: unknown): RunCreatedInlineDeliveryOutcome => ({
+      status: 'error',
+      durationMs: elapsed(),
+      error: error instanceof Error ? error.message : String(error),
+    }),
+  );
+  let deadline: ReturnType<typeof setTimeout> | undefined;
+  const timedOut = new Promise<RunCreatedInlineDeliveryOutcome>((resolve) => {
+    deadline = setTimeout(() => {
+      // Abort the append; the attempt's own tail (a short, lease-fenced
+      // release) completes in the background and cannot fail the caller.
+      controller.abort(
+        new Error('Inline run.created delivery deadline elapsed.'),
+      );
+      resolve({ status: 'timed_out', durationMs: elapsed() });
+    }, deadlineMs);
+    deadline.unref?.();
+  });
+  try {
+    return await Promise.race([attempt, timedOut]);
+  } finally {
+    if (deadline) clearTimeout(deadline);
+  }
+}
+
 /**
  * Production wiring shared by each stateless Fly projector replica. Every
  * replica uses gateway authority because Fly holds no signing key; leased
@@ -416,6 +579,8 @@ export function startRunLedgerOutboxRecoveryController(input: {
   schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
   /** Direct LISTEN is a latency hint and may be disabled for isolated schemas. */
   useDirectNotify?: boolean;
+  /** Observation hook for acknowledged rows (lag metrics). */
+  onDelivered?: (deliveries: readonly RunLedgerOutboxDeliveredRow[]) => void;
 }): RunLedgerOutboxRecoveryController {
   const schedulerOptions = input.schedulerOptions ?? {};
   return startRunLedgerOutboxRecovery({
@@ -431,7 +596,15 @@ export function startRunLedgerOutboxRecoveryController(input: {
     dispatchOnce: (headPriority) =>
       dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
         (operation) => withRuntimeSchedulerClient(input.poolOptions, operation),
-        { tokenAuthority: 'gateway', limit: 8, headPriority },
+        {
+          tokenAuthority: 'gateway',
+          limit: 8,
+          headPriority,
+          // The lease holder delivers a run's terminal committed during its
+          // append, instead of leaving it for the next wave.
+          continueThroughTerminal: true,
+          ...(input.onDelivered ? { onDelivered: input.onDelivered } : {}),
+        },
         schedulerOptions,
       ),
     // `dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope` releases each

package/dist/bundling-sources/shared_libs/play-runtime/step-lifecycle-tracker.ts

@@ -192,24 +192,6 @@ export class PlayStepLifecycleTracker {
     }
   }
 
-  markStartedFailed(at = this.now(), error?: string): void {
-    for (let index = this.nodes.length - 1; index >= 0; index -= 1) {
-      const node = this.nodes[index]!;
-      const existing = this.getProgress()[node.nodeId];
-      if (existing?.startedAt && !existing.completedAt) {
-        this.failedNodeIds.add(node.nodeId);
-        this.emit({
-          nodeId: node.nodeId,
-          type: node.type,
-          transition: 'failed',
-          at,
-          ...(error?.trim() ? { error: error.trim() } : {}),
-        });
-        return;
-      }
-    }
-  }
-
   private completeStartedNonMapNodes(at: number): void {
     for (const node of this.nodes) {
       if (node.type === 'dataset') {

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -1,5 +1,6 @@
 import {
   isHardBillingToolHttpError,
+  isHtmlErrorBody,
   normalizeToolHttpErrorMessage,
   readExplicitToolHttpRetryability,
   type ToolHttpError,
@@ -145,6 +146,7 @@ export type ToolExecuteHttpRetryDecision = {
     | 'gateway_invocation_in_progress'
     | 'customer_db_storage_unavailable'
     | 'retry_safe_transient_5xx'
+    | 'edge_gateway_transient'
     | 'unsafe_transient_5xx'
     | 'hard_billing_error'
     | 'non_retryable_status';
@@ -170,6 +172,7 @@ export type ToolExecuteHttpFailureAttemptTracker = {
     status: number;
     bodyText?: string;
     transientHttpRetrySafe?: boolean;
+    executionFenced?: boolean;
   }): number;
 };
 
@@ -264,6 +267,41 @@ function isProviderPacingWaitResponse(input: {
   );
 }
 
+/**
+ * Edge statuses (Cloudflare 521/522/523/525/526/530) that mean the edge never
+ * reached the Deepline gateway: no provider request can have been dispatched.
+ * 530 is Cloudflare's origin-unreachable page (1033 "Tunnel error": no
+ * connector serves the hostname, or the origin cannot be resolved) — the
+ * request had nowhere to go. A tunnel connector restart surfaced it on a dev
+ * gateway and, unretried, it failed the row on its first attempt.
+ */
+const EDGE_ORIGIN_UNREACHED_STATUSES: ReadonlySet<number> = new Set([
+  521, 522, 523, 525, 526, 530,
+]);
+
+/**
+ * A 5xx whose body is the HTML error page of the edge in front of the Deepline
+ * gateway (a Cloudflare 524 origin timeout, a 502/503/504 proxy page) rather
+ * than a Deepline JSON envelope: the gateway produced no answer of its own, so
+ * the provider outcome is unknown or the request never arrived.
+ */
+export function isEdgeGatewayFailureResponse(input: {
+  status: number;
+  bodyText: string;
+}): boolean {
+  return (
+    input.status >= 500 && input.status < 600 && isHtmlErrorBody(input.bodyText)
+  );
+}
+
+/** Exponential backoff for edge gateway failures: 1 s, 2 s, 4 s, capped. */
+function edgeGatewayRetryDelayMs(attempt: number): number {
+  return Math.min(
+    TOOL_EXECUTE_RETRY_DELAY_MAX_MS,
+    TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * 2 ** Math.max(0, attempt - 1),
+  );
+}
+
 function idempotencyInProgressRetryDelayMs(attempt: number): number {
   return (
     TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -303,6 +341,8 @@ function decideToolExecuteHttpRetry(input: {
   hardBillingFailure?: boolean;
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
+  edgeGatewayFailure?: boolean;
+  executionFenced?: boolean;
   explicitRetryable?: boolean;
 }): ToolExecuteHttpRetryDecision {
   if (input.hardBillingFailure) {
@@ -380,6 +420,23 @@ function decideToolExecuteHttpRetry(input: {
         reason: 'non_retryable_status',
       };
     }
+    // An edge failure in front of the gateway is transient. Repeating it is
+    // safe when the edge never reached the gateway, when the call runs under
+    // the gateway's durable execution fence (a replay returns the recorded
+    // outcome and never re-dispatches), or when the operation is declared
+    // transient-HTTP safe. Otherwise it stays a row failure, like any 5xx.
+    if (
+      input.edgeGatewayFailure &&
+      (EDGE_ORIGIN_UNREACHED_STATUSES.has(input.status) ||
+        input.executionFenced === true ||
+        input.transientHttpRetrySafe === true)
+    ) {
+      return {
+        retryable: true,
+        attemptCap: TOOL_EXECUTE_TRANSIENT_HTTP_MAX_ATTEMPTS,
+        reason: 'edge_gateway_transient',
+      };
+    }
     if (!input.transientHttpRetrySafe) {
       return {
         retryable: false,
@@ -413,6 +470,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
     gateway_invocation_in_progress: 0,
     customer_db_storage_unavailable: 0,
     retry_safe_transient_5xx: 0,
+    edge_gateway_transient: 0,
     unsafe_transient_5xx: 0,
     hard_billing_error: 0,
     non_retryable_status: 0,
@@ -449,6 +507,11 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
         }),
         hasRetryAfterHeader: true,
         transientHttpRetrySafe: input.transientHttpRetrySafe === true,
+        edgeGatewayFailure: isEdgeGatewayFailureResponse({
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
+        executionFenced: input.executionFenced === true,
         explicitRetryable: readExplicitToolHttpRetryability({
           toolId: input.toolId,
           bodyText: input.bodyText ?? '',
@@ -493,6 +556,8 @@ export function classifyToolExecuteHttpFailure(input: {
   schemaVersion?: ToolExecutionErrorSchemaVersion;
   retryAfterHeader?: string | null;
   transientHttpRetrySafe?: boolean;
+  /** The call runs under the gateway's durable execution fence. */
+  executionFenced?: boolean;
   providerLatencyMs?: number;
   nowMs?: number;
 }): ToolExecuteHttpFailureOutcome {
@@ -508,6 +573,8 @@ export function classifyToolExecuteHttpFailure(input: {
   const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
   const runtimeSchedulerCapacity = isRuntimeSchedulerCapacityResponse(input);
   const providerPacingWait = isProviderPacingWaitResponse(input);
+  const edgeGatewayFailure = isEdgeGatewayFailureResponse(input);
+  const executionFenced = input.executionFenced === true;
   const explicitRetryable = readExplicitToolHttpRetryability(input);
   const initialRetryDecision = decideToolExecuteHttpRetry({
     status: input.status,
@@ -519,6 +586,8 @@ export function classifyToolExecuteHttpFailure(input: {
     providerPacingWait,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
+    edgeGatewayFailure,
+    executionFenced,
     explicitRetryable,
   });
   const providerPacingWaitBody = providerPacingWait
@@ -583,6 +652,8 @@ export function classifyToolExecuteHttpFailure(input: {
     hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
+    edgeGatewayFailure,
+    executionFenced,
     explicitRetryable,
   });
   const shouldRetry =
@@ -636,15 +707,17 @@ export function classifyToolExecuteHttpFailure(input: {
                     TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
                   ),
                 )
-            : retryDecision.reason === 'idempotency_in_progress' &&
-                !hasRetryAfterHeader
-              ? idempotencyInProgressRetryDelayMs(input.attempt)
-              : retryDecision.reason === 'gateway_invocation_in_progress' &&
+            : retryDecision.reason === 'edge_gateway_transient'
+              ? edgeGatewayRetryDelayMs(input.attempt)
+              : retryDecision.reason === 'idempotency_in_progress' &&
                   !hasRetryAfterHeader
                 ? idempotencyInProgressRetryDelayMs(input.attempt)
-                : retryAfterMs > 0
-                  ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
-                  : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+                : retryDecision.reason === 'gateway_invocation_in_progress' &&
+                    !hasRetryAfterHeader
+                  ? idempotencyInProgressRetryDelayMs(input.attempt)
+                  : retryAfterMs > 0
+                    ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
+                    : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
   return {
     ...retryDecision,
     error,

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -322,12 +322,47 @@ function formatInsufficientCreditsMessage(input: {
   return `Workspace balance ${balance} < required ${required} for ${operation}.${addSuffix}`;
 }
 
+/** An HTML document, as an edge proxy (Cloudflare, Vercel) serves for a 5xx. */
+export function isHtmlErrorBody(bodyText: string): boolean {
+  return /^\s*(?:<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>])/i.test(
+    bodyText,
+  );
+}
+
+/** The Cloudflare Ray ID an edge error page carries, when it has one. */
+export function edgeErrorPageRayId(bodyText: string): string | null {
+  const match = /Ray ID:?\s*(?:<[^>]*>\s*)*([0-9a-f]{8,32})/i.exec(bodyText);
+  return match?.[1] ?? null;
+}
+
+/**
+ * One line for an edge HTML error page: its title and Ray ID, never the
+ * document. A raw page is kilobytes of markup that floods CLI output and logs.
+ */
+export function summarizeHtmlErrorBody(bodyText: string): string {
+  const title = /<title[^>]*>([\s\S]{0,500}?)<\/title>/i
+    .exec(bodyText)?.[1]
+    ?.replace(/<[^>]*>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  const rayId = edgeErrorPageRayId(bodyText);
+  return [
+    `gateway HTML error page${title ? `: ${title.slice(0, 160)}` : ''}`,
+    rayId ? `(ray ${rayId})` : null,
+  ]
+    .filter(Boolean)
+    .join(' ');
+}
+
 function formatPublicToolErrorPayload(input: {
   parsed: Record<string, unknown> | null;
   bodyText: string;
 }): string {
   if (!input.parsed) {
-    return input.bodyText.slice(0, 500);
+    if (isHtmlErrorBody(input.bodyText)) {
+      return summarizeHtmlErrorBody(input.bodyText);
+    }
+    return input.bodyText.replace(/\s+/g, ' ').trim().slice(0, 500);
   }
 
   const selected: Record<string, unknown> = {};
@@ -461,7 +496,7 @@ export function normalizeToolHttpErrorMessage(input: {
       getStringField(parsed, 'request_id') ??
       getStringField(parsed, 'requestId') ??
       hydratedFailure?.requestId ??
-      null,
+      (parsed ? null : edgeErrorPageRayId(input.bodyText)),
     retryAfterMs: hydratedFailure?.retryAfterMs ?? input.retryAfterMs ?? null,
     networkKind:
       hydratedFailure?.networkKind ??