deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -24,6 +24,11 @@ export const TOOL_EXECUTE_BARE_RATE_LIMIT_MAX_ATTEMPTS = 2;
 // tool deadline remains the hard bound; this prevents a normal one-minute
 // queue wait from being compressed into eight five-second retries.
 export const TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_MAX_ATTEMPTS = 65;
+// Scheduler admission happens before the integration route can dispatch a
+// provider request. A typed refusal is therefore safe to park and replay even
+// for an operation that has not opted into ordinary transient-5xx retries.
+// Keep its window aligned with other local control-plane queue signals.
+export const TOOL_EXECUTE_RUNTIME_SCHEDULER_CAPACITY_MAX_ATTEMPTS = 65;
 export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
 export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
@@ -61,6 +66,7 @@ export type ToolExecuteHttpRetryDecision = {
   reason:
     | 'rate_limit'
     | 'concurrency_backpressure'
+    | 'runtime_scheduler_capacity'
     | 'idempotency_in_progress'
     | 'gateway_invocation_in_progress'
     | 'customer_db_storage_unavailable'
@@ -78,6 +84,8 @@ export type ToolExecuteHttpFailureOutcome = ToolExecuteHttpRetryDecision & {
   retryDelayMs: number;
   backpressureDelayMs: number | null;
   chargeRetryBudget: boolean;
+  /** The response proves the provider request was never dispatched. */
+  knownNotDispatched: boolean;
 };
 
 export type ToolExecuteHttpFailureAttemptTracker = {
@@ -155,6 +163,18 @@ function isConcurrencyBackpressureResponse(input: {
   );
 }
 
+function isRuntimeSchedulerCapacityResponse(input: {
+  status: number;
+  bodyText: string;
+}): boolean {
+  // This is a scheduler pre-admission outcome. Do not extend this exemption to
+  // generic 503s: they can arrive after a non-idempotent provider call.
+  return (
+    input.status === 503 &&
+    parseJsonObject(input.bodyText)?.code === 'scheduler_capacity'
+  );
+}
+
 function idempotencyInProgressRetryDelayMs(attempt: number): number {
   return (
     TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
@@ -189,6 +209,7 @@ function decideToolExecuteHttpRetry(input: {
   gatewayInvocationInProgress?: boolean;
   customerDbStorageUnavailable?: boolean;
   concurrencyBackpressure?: boolean;
+  runtimeSchedulerCapacity?: boolean;
   hardBillingFailure?: boolean;
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
@@ -222,6 +243,13 @@ function decideToolExecuteHttpRetry(input: {
       reason: 'rate_limit',
     };
   }
+  if (input.runtimeSchedulerCapacity) {
+    return {
+      retryable: true,
+      attemptCap: TOOL_EXECUTE_RUNTIME_SCHEDULER_CAPACITY_MAX_ATTEMPTS,
+      reason: 'runtime_scheduler_capacity',
+    };
+  }
   if (input.gatewayInvocationInProgress) {
     return {
       retryable: true,
@@ -274,6 +302,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
   > = {
     rate_limit: 0,
     concurrency_backpressure: 0,
+    runtime_scheduler_capacity: 0,
     idempotency_in_progress: 0,
     gateway_invocation_in_progress: 0,
     customer_db_storage_unavailable: 0,
@@ -304,6 +333,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
           status: input.status,
           bodyText: input.bodyText ?? '',
         }),
+        runtimeSchedulerCapacity: isRuntimeSchedulerCapacityResponse({
+          status: input.status,
+          bodyText: input.bodyText ?? '',
+        }),
         hasRetryAfterHeader: true,
         transientHttpRetrySafe: input.transientHttpRetrySafe === true,
       });
@@ -350,12 +383,14 @@ export function classifyToolExecuteHttpFailure(input: {
   const customerDbStorageUnavailable =
     isCustomerDbStorageUnavailableResponse(input);
   const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
+  const runtimeSchedulerCapacity = isRuntimeSchedulerCapacityResponse(input);
   const initialRetryDecision = decideToolExecuteHttpRetry({
     status: input.status,
     idempotencyInProgress,
     gatewayInvocationInProgress,
     customerDbStorageUnavailable,
     concurrencyBackpressure,
+    runtimeSchedulerCapacity,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
   });
@@ -372,7 +407,9 @@ export function classifyToolExecuteHttpFailure(input: {
     schemaVersion: input.schemaVersion,
     retryable: initialRetryDecision.retryable,
     retryAfterMs:
-      input.status === 429 && hasRetryAfterHeader ? retryAfterMs : null,
+      (input.status === 429 && hasRetryAfterHeader) || runtimeSchedulerCapacity
+        ? retryAfterMs
+        : null,
   });
   if (
     typeof input.providerLatencyMs === 'number' &&
@@ -388,6 +425,7 @@ export function classifyToolExecuteHttpFailure(input: {
     gatewayInvocationInProgress,
     customerDbStorageUnavailable,
     concurrencyBackpressure,
+    runtimeSchedulerCapacity,
     hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
@@ -414,23 +452,28 @@ export function classifyToolExecuteHttpFailure(input: {
             TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
           ),
         )
-      : input.status === 429
+      : retryDecision.reason === 'runtime_scheduler_capacity'
         ? Math.min(
-            TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
-            Math.max(
-              retryAfterMs,
-              TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
-            ),
+            TOOL_EXECUTE_RETRY_DELAY_MAX_MS,
+            Math.max(retryAfterMs, TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS),
           )
-        : retryDecision.reason === 'idempotency_in_progress' &&
-            !hasRetryAfterHeader
-          ? idempotencyInProgressRetryDelayMs(input.attempt)
-          : retryDecision.reason === 'gateway_invocation_in_progress' &&
+        : input.status === 429
+          ? Math.min(
+              TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
+              Math.max(
+                retryAfterMs,
+                TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
+              ),
+            )
+          : retryDecision.reason === 'idempotency_in_progress' &&
               !hasRetryAfterHeader
             ? idempotencyInProgressRetryDelayMs(input.attempt)
-            : retryAfterMs > 0
-              ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
-              : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
+            : retryDecision.reason === 'gateway_invocation_in_progress' &&
+                !hasRetryAfterHeader
+              ? idempotencyInProgressRetryDelayMs(input.attempt)
+              : retryAfterMs > 0
+                ? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
+                : TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
   return {
     ...retryDecision,
     error,
@@ -458,6 +501,8 @@ export function classifyToolExecuteHttpFailure(input: {
     chargeRetryBudget:
       shouldRetry &&
       retryDecision.reason !== 'gateway_invocation_in_progress' &&
-      retryDecision.reason !== 'concurrency_backpressure',
+      retryDecision.reason !== 'concurrency_backpressure' &&
+      retryDecision.reason !== 'runtime_scheduler_capacity',
+    knownNotDispatched: runtimeSchedulerCapacity,
   };
 }

package/dist/bundling-sources/shared_libs/play-runtime/tool-execution-outcome.ts

@@ -19,6 +19,11 @@ export type ToolExecutionOutcome =
   | (BaseToolExecutionOutcome & {
       kind: 'checkpoint';
     })
+  | (BaseToolExecutionOutcome & {
+      /** Exact completed receipt recorded by the user-requested source run. */
+      kind: 'source_run';
+      receiptKey: string;
+    })
   | (BaseToolExecutionOutcome & {
       kind: 'cache';
       receiptKey: string;
@@ -75,6 +80,21 @@ export function toolExecutionMetadataForOutcome(
         source: 'checkpoint',
         ...optionalCacheKey(outcome.cacheKey),
       };
+    case 'source_run': {
+      const receiptKey = trimmed(outcome.receiptKey);
+      if (!receiptKey) {
+        throw new Error('Source-run tool recovery requires a receipt key.');
+      }
+      return {
+        idempotent: true,
+        cached: true,
+        source: 'source_run',
+        ...optionalCacheKey(outcome.cacheKey),
+        receiptRole: 'follower',
+        receiptKey,
+        ...followerAttachment({ receiptKey }),
+      };
+    }
     case 'cache': {
       const receiptKey = trimmed(outcome.receiptKey);
       if (!receiptKey) {

package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts

@@ -8,6 +8,18 @@ export function transientServiceUnavailableDetail(
   if (error instanceof TypeError && /^fetch failed$/i.test(detail.trim())) {
     return detail;
   }
+  // A Convex deployment can also be briefly unreachable at its Cloudflare
+  // edge. Unlike the normal JSON envelope, the HTTP client passes that 502
+  // page through as its error message. Keep this narrow to Convex's own
+  // branded page: an arbitrary application response that happens to mention
+  // a gateway must never make a caller retryable.
+  if (
+    /convex\.cloud\s*\|\s*502\s*:\s*Bad gateway[\s\S]*Error code\s*502/i.test(
+      detail,
+    )
+  ) {
+    return detail;
+  }
   // Convex uses explicit ServiceUnavailable and ExpiredInQueue responses, as
   // well as this generic InternalServerError envelope for short control-plane
   // outages. The latter text is intentionally exact: application exceptions

package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts

@@ -114,7 +114,9 @@ export async function resolveVercelProtectionBypassCookie(input: {
     // supplies a reusable cookie or evicts itself for a later retry.
     const response = await (input.fetchImpl ?? fetch)(url.toString(), {
       headers: { 'x-vercel-protection-bypass': token },
-      signal: AbortSignal.timeout(VERCEL_PROTECTION_COOKIE_PREFLIGHT_TIMEOUT_MS),
+      signal: AbortSignal.timeout(
+        VERCEL_PROTECTION_COOKIE_PREFLIGHT_TIMEOUT_MS,
+      ),
     });
     return response ? cookieHeaderFromSetCookie(response.headers) : null;
   })();
@@ -154,3 +156,14 @@ export async function vercelProtectionBypassHeaders(input: {
     ...(cookie ? { cookie } : {}),
   };
 }
+
+export function resolveVercelProtectionBypassTokenForRequest(
+  req: Pick<Request, 'headers'>,
+): string | null {
+  const token =
+    req.headers.get('x-vercel-protection-bypass')?.trim() ||
+    process.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim() ||
+    process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim() ||
+    '';
+  return token || null;
+}

package/dist/bundling-sources/shared_libs/play-runtime/webhook-receipt-retry.ts

@@ -0,0 +1,50 @@
+import { createHash } from 'node:crypto';
+import {
+  PLAY_SECRET_UNAVAILABLE,
+  playSecretUnavailableErrorFrom,
+} from '../plays/secrets/errors';
+
+const RETRY_BASE_MS = 1_000;
+const RETRY_CAP_MS = 15 * 60 * 1_000;
+
+/**
+ * Admission has not called a customer provider or started user code. Unknown
+ * failures therefore stay retryable: an internal 5xx must not discard a
+ * source delivery. Only a proven local configuration error is blocked.
+ */
+export function classifyWebhookReceiptAdmissionFailure(error: unknown): {
+  retryable: boolean;
+  code: 'INTERNAL_ADMISSION_FAILURE' | typeof PLAY_SECRET_UNAVAILABLE;
+} {
+  const unavailableSecret = playSecretUnavailableErrorFrom(error);
+  if (unavailableSecret) {
+    return { retryable: false, code: PLAY_SECRET_UNAVAILABLE };
+  }
+  return { retryable: true, code: 'INTERNAL_ADMISSION_FAILURE' };
+}
+
+/**
+ * Stable jitter spreads a burst without making a receipt's retry timing
+ * unknowable to operators. The cap makes a prolonged platform incident cheap
+ * while the receipt remains durable and visible for recovery.
+ */
+export function webhookReceiptRetryDelayMs(input: {
+  eventId: string;
+  attempt: number;
+}): number {
+  const exponent = Math.min(Math.max(input.attempt - 1, 0), 10);
+  const ceiling = Math.min(RETRY_CAP_MS, RETRY_BASE_MS * 2 ** exponent);
+  const hash = createHash('sha256')
+    .update(input.eventId)
+    .update(':')
+    .update(String(input.attempt))
+    .digest();
+  // Keep a nonzero delay and give concurrent deliveries independent stable
+  // placement across the exponential window. The first retry spans 1–2s;
+  // otherwise a burst of first-attempt failures would synchronise again.
+  const fraction = hash.readUInt32BE(0) / 0xffff_ffff;
+  return Math.min(
+    RETRY_CAP_MS,
+    Math.max(RETRY_BASE_MS, Math.round(ceiling * (1 + fraction))),
+  );
+}

package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-storage-contract.ts

@@ -0,0 +1,165 @@
+import { deployed } from '@deepline/deployed-contracts';
+
+/**
+ * Durable work-receipt storage is shared by app and worker deployments.
+ *
+ * The key namespace and integer codes are immutable: changing either would
+ * reinterpret an already-written receipt or make it invisible. Table columns
+ * are intentionally additive, because an older writer ignores an extra column
+ * and this module gives a newer reader one canonical self-heal path.
+ */
+export const WORK_RECEIPT_KEY_FORMAT = {
+  prefix: 'ctxv2',
+  separator: ':',
+} as const;
+
+export const WORK_RECEIPT_STATUS_CODE = {
+  queued: 0,
+  pending: 0,
+  running: 1,
+  completed: 2,
+  failed: 3,
+  skipped: 4,
+} as const;
+
+export type WorkReceiptStatus = keyof typeof WORK_RECEIPT_STATUS_CODE;
+
+export const WORK_RECEIPT_FAILURE_KIND_CODE = {
+  terminal: 0,
+  repairable: 1,
+} as const;
+
+export type WorkReceiptFailureKind =
+  keyof typeof WORK_RECEIPT_FAILURE_KIND_CODE;
+
+/**
+ * Every customer-Neon receipt writer derives its DDL from this list. The
+ * deploy-skew gate permits only additive changes once this contract is live.
+ */
+export const WORK_RECEIPT_TABLE_COLUMNS = [
+  ['status', 'smallint NOT NULL DEFAULT 0'],
+  ['output', 'jsonb'],
+  ['error', 'text'],
+  ['error_payload', 'jsonb'],
+  ['failure_kind', 'smallint NOT NULL DEFAULT 0'],
+  ['run_id', 'text'],
+  ['lease_id', 'text'],
+  ['lease_owner_run_id', 'text'],
+  ['lease_owner_attempt', 'integer'],
+  ['lease_expires_at', 'timestamptz'],
+  ['updated_at', 'timestamptz NOT NULL DEFAULT now()'],
+] as const;
+
+/** The non-idempotent-work lock shares the receipt deployment boundary. */
+export const WORK_RECEIPT_EXECUTION_LOCK_TABLE =
+  '_deepline_receipt_execution_locks';
+
+export const WORK_RECEIPT_EXECUTION_LOCK_STORAGE_CONTRACT = deployed.table({
+  id: 'work-receipt-execution-lock-storage',
+  key: 'k bytea primary key',
+  columns: {
+    owner_execution_id: { type: 'text NOT NULL' },
+    expires_at: { type: 'timestamptz NOT NULL' },
+    updated_at: {
+      type: 'timestamptz NOT NULL DEFAULT now()',
+      default: 'timestamptz NOT NULL DEFAULT now()',
+    },
+  },
+});
+
+/**
+ * CI snapshots this declarative shape. A writer can add a nullable/defaulted
+ * column, but cannot silently reinterpret a receipt key or replace/delete a
+ * column an older reader may still need.
+ */
+export const WORK_RECEIPT_STORAGE_CONTRACT = deployed.table({
+  id: 'work-receipt-storage',
+  key: 'k bytea primary key',
+  columns: Object.fromEntries(
+    WORK_RECEIPT_TABLE_COLUMNS.map(([name, definition]) => [
+      name,
+      {
+        type: definition,
+        optional: !/NOT NULL/.test(definition) || undefined,
+        default: /DEFAULT/.test(definition) ? definition : undefined,
+      },
+    ]),
+  ),
+});
+
+export type WorkReceiptTableColumn =
+  (typeof WORK_RECEIPT_TABLE_COLUMNS)[number][0];
+
+export const WORK_RECEIPT_TABLE_COLUMN_NAMES = WORK_RECEIPT_TABLE_COLUMNS.map(
+  ([column]) => column,
+) as readonly WorkReceiptTableColumn[];
+
+export const WORK_RECEIPT_LEASE_COLUMNS = [
+  'lease_id',
+  'lease_owner_run_id',
+  'lease_owner_attempt',
+  'lease_expires_at',
+] as const satisfies readonly WorkReceiptTableColumn[];
+
+/** Columns a newer runtime can safely add to a legacy receipt table. */
+export const WORK_RECEIPT_SELF_HEAL_COLUMNS = [
+  ...WORK_RECEIPT_LEASE_COLUMNS,
+  'failure_kind',
+  'error_payload',
+] as const satisfies readonly WorkReceiptTableColumn[];
+
+const WORK_RECEIPT_TABLE_COLUMN_DEFINITION = new Map<
+  WorkReceiptTableColumn,
+  string
+>(WORK_RECEIPT_TABLE_COLUMNS);
+
+export function workReceiptTableColumnDefinition(
+  column: WorkReceiptTableColumn,
+): string {
+  const definition = WORK_RECEIPT_TABLE_COLUMN_DEFINITION.get(column);
+  if (!definition) {
+    throw new Error(`Unknown runtime work receipt column: ${column}`);
+  }
+  return definition;
+}
+
+export function workReceiptTableDefinitionSql(): string {
+  return [
+    'k bytea PRIMARY KEY',
+    ...WORK_RECEIPT_TABLE_COLUMNS.map(
+      ([column, definition]) => `${column} ${definition}`,
+    ),
+  ].join(',\n      ');
+}
+
+export function workReceiptExecutionLockTableDefinitionSql(): string {
+  return [
+    'k bytea PRIMARY KEY',
+    'owner_execution_id text NOT NULL',
+    'expires_at timestamptz NOT NULL',
+    'updated_at timestamptz NOT NULL DEFAULT now()',
+  ].join(',\n      ');
+}
+
+export function workReceiptAdditiveColumnsSql(
+  columns: readonly WorkReceiptTableColumn[],
+): string {
+  return columns
+    .map(
+      (column) =>
+        `ADD COLUMN IF NOT EXISTS ${column} ${workReceiptTableColumnDefinition(column)}`,
+    )
+    .join(',\n          ');
+}
+
+export function workReceiptStatusCaseSql(column = 'status'): string {
+  return [
+    `case ${column}`,
+    `when ${WORK_RECEIPT_STATUS_CODE.queued} then 'pending'`,
+    `when ${WORK_RECEIPT_STATUS_CODE.running} then 'running'`,
+    `when ${WORK_RECEIPT_STATUS_CODE.completed} then 'completed'`,
+    `when ${WORK_RECEIPT_STATUS_CODE.failed} then 'failed'`,
+    `when ${WORK_RECEIPT_STATUS_CODE.skipped} then 'skipped'`,
+    `else ${column}::text end`,
+  ].join(' ');
+}

package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts

@@ -1,14 +1,16 @@
 import { isReusableWorkReceiptState } from './work-receipt-state-machine';
 import type { ToolExecutionFailureV1 } from '../plays/tool-execution-error';
-
-export type WorkReceiptFailureKind = 'terminal' | 'repairable';
-
-export const WORK_RECEIPT_FAILURE_KIND_CODE: Record<
-  WorkReceiptFailureKind,
-  number
-> = {
-  terminal: 0,
-  repairable: 1,
+import {
+  WORK_RECEIPT_FAILURE_KIND_CODE,
+  WORK_RECEIPT_KEY_FORMAT,
+  type WorkReceiptFailureKind,
+  type WorkReceiptStatus,
+} from './work-receipt-storage-contract';
+
+export {
+  WORK_RECEIPT_FAILURE_KIND_CODE,
+  type WorkReceiptFailureKind,
+  type WorkReceiptStatus,
 };
 
 export function workReceiptFailureKindFromCode(
@@ -39,14 +41,6 @@ export function workReceiptFailureKindCodeForWrite(
   return WORK_RECEIPT_FAILURE_KIND_CODE.repairable;
 }
 
-export type WorkReceiptStatus =
-  | 'queued'
-  | 'pending'
-  | 'running'
-  | 'completed'
-  | 'failed'
-  | 'skipped';
-
 export type WorkReceipt = {
   key: string;
   status: WorkReceiptStatus;
@@ -191,8 +185,6 @@ export type WorkReceiptStore = {
   ): Promise<Array<WorkReceipt | null>>;
 };
 
-const SCOPED_WORK_RECEIPT_KEY_VERSION = 'ctxv2';
-
 function encodeWorkReceiptKeyComponent(value: string): string {
   return encodeURIComponent(value);
 }
@@ -212,11 +204,11 @@ export function buildScopedWorkReceiptKey(input: {
   const orgId = normalizeWorkReceiptScopeComponent(input.orgId, 'org');
   const playName = normalizeWorkReceiptScopeComponent(input.playName, 'play');
   return [
-    SCOPED_WORK_RECEIPT_KEY_VERSION,
+    WORK_RECEIPT_KEY_FORMAT.prefix,
     encodeWorkReceiptKeyComponent(orgId),
     encodeWorkReceiptKeyComponent(playName),
     encodeWorkReceiptKeyComponent(input.key),
-  ].join(':');
+  ].join(WORK_RECEIPT_KEY_FORMAT.separator);
 }
 
 export function scopedWorkReceiptKeyPrefix(input: {
@@ -224,11 +216,11 @@ export function scopedWorkReceiptKeyPrefix(input: {
   playName: string;
 }): string {
   return [
-    SCOPED_WORK_RECEIPT_KEY_VERSION,
+    WORK_RECEIPT_KEY_FORMAT.prefix,
     encodeWorkReceiptKeyComponent(input.orgId.trim()),
     encodeWorkReceiptKeyComponent(input.playName.trim()),
     '',
-  ].join(':');
+  ].join(WORK_RECEIPT_KEY_FORMAT.separator);
 }
 
 export function isReusableWorkReceipt(receipt: WorkReceipt): boolean {