deepline 0.1.211 → 0.1.213

package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts

@@ -0,0 +1,162 @@
+export type RuntimePostgresLane = 'receipts' | 'sheets';
+
+export type RuntimePostgresLaneTelemetry = {
+  active: number;
+  queued: number;
+  acquired: number;
+  waited: number;
+  timedOut: number;
+  rejected: number;
+  totalWaitMs: number;
+  maxWaitMs: number;
+};
+
+export type RuntimePostgresAdmissionSnapshot = Record<
+  RuntimePostgresLane,
+  RuntimePostgresLaneTelemetry
+>;
+
+export class RuntimePostgresAdmissionTimeoutError extends Error {
+  constructor(
+    readonly lane: RuntimePostgresLane,
+    readonly queueDepth: number,
+    readonly timeoutMs: number,
+  ) {
+    super(
+      `Runtime Postgres ${lane} admission timed out after ${timeoutMs}ms (${queueDepth} queued).`,
+    );
+    this.name = 'RuntimePostgresAdmissionTimeoutError';
+  }
+}
+
+export class RuntimePostgresAdmissionQueueFullError extends Error {
+  constructor(
+    readonly lane: RuntimePostgresLane,
+    readonly queueDepth: number,
+  ) {
+    super(
+      `Runtime Postgres ${lane} admission queue is full (${queueDepth} queued).`,
+    );
+    this.name = 'RuntimePostgresAdmissionQueueFullError';
+  }
+}
+
+type Waiter = {
+  enqueuedAt: number;
+  timeout: ReturnType<typeof setTimeout>;
+  resolve: (release: () => void) => void;
+  reject: (error: Error) => void;
+};
+
+type LaneState = RuntimePostgresLaneTelemetry & { waiters: Waiter[] };
+
+function createLaneState(): LaneState {
+  return {
+    active: 0,
+    queued: 0,
+    acquired: 0,
+    waited: 0,
+    timedOut: 0,
+    rejected: 0,
+    totalWaitMs: 0,
+    maxWaitMs: 0,
+    waiters: [],
+  };
+}
+
+export class RuntimePostgresAdmission {
+  readonly #maxActivePerLane: number;
+  readonly #maxQueuedPerLane: number;
+  readonly #acquireTimeoutMs: number;
+  readonly #lanes: Record<RuntimePostgresLane, LaneState> = {
+    receipts: createLaneState(),
+    sheets: createLaneState(),
+  };
+
+  constructor(input: {
+    maxActivePerLane: number;
+    maxQueuedPerLane: number;
+    acquireTimeoutMs: number;
+  }) {
+    this.#maxActivePerLane = Math.max(1, Math.floor(input.maxActivePerLane));
+    this.#maxQueuedPerLane = Math.max(0, Math.floor(input.maxQueuedPerLane));
+    this.#acquireTimeoutMs = Math.max(1, Math.floor(input.acquireTimeoutMs));
+  }
+
+  async acquire(lane: RuntimePostgresLane): Promise<() => void> {
+    const state = this.#lanes[lane];
+    if (state.active < this.#maxActivePerLane && state.waiters.length === 0) {
+      state.active += 1;
+      state.acquired += 1;
+      return this.#releaseFor(lane);
+    }
+    if (state.waiters.length >= this.#maxQueuedPerLane) {
+      state.rejected += 1;
+      throw new RuntimePostgresAdmissionQueueFullError(
+        lane,
+        state.waiters.length,
+      );
+    }
+    state.waited += 1;
+    return await new Promise<() => void>((resolve, reject) => {
+      const waiter: Waiter = {
+        enqueuedAt: Date.now(),
+        timeout: setTimeout(() => {
+          const index = state.waiters.indexOf(waiter);
+          if (index < 0) return;
+          state.waiters.splice(index, 1);
+          state.queued = state.waiters.length;
+          state.timedOut += 1;
+          reject(
+            new RuntimePostgresAdmissionTimeoutError(
+              lane,
+              state.waiters.length,
+              this.#acquireTimeoutMs,
+            ),
+          );
+        }, this.#acquireTimeoutMs),
+        resolve,
+        reject,
+      };
+      state.waiters.push(waiter);
+      state.queued = state.waiters.length;
+    });
+  }
+
+  snapshot(): RuntimePostgresAdmissionSnapshot {
+    const snapshotLane = (state: LaneState): RuntimePostgresLaneTelemetry => ({
+      active: state.active,
+      queued: state.waiters.length,
+      acquired: state.acquired,
+      waited: state.waited,
+      timedOut: state.timedOut,
+      rejected: state.rejected,
+      totalWaitMs: state.totalWaitMs,
+      maxWaitMs: state.maxWaitMs,
+    });
+    return {
+      receipts: snapshotLane(this.#lanes.receipts),
+      sheets: snapshotLane(this.#lanes.sheets),
+    };
+  }
+
+  #releaseFor(lane: RuntimePostgresLane): () => void {
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const state = this.#lanes[lane];
+      state.active -= 1;
+      const waiter = state.waiters.shift();
+      state.queued = state.waiters.length;
+      if (!waiter) return;
+      clearTimeout(waiter.timeout);
+      const waitMs = Math.max(0, Date.now() - waiter.enqueuedAt);
+      state.totalWaitMs += waitMs;
+      state.maxWaitMs = Math.max(state.maxWaitMs, waitMs);
+      state.active += 1;
+      state.acquired += 1;
+      waiter.resolve(this.#releaseFor(lane));
+    };
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/secret-capability.ts

@@ -39,10 +39,24 @@ export function isSecretAuth(value: unknown): value is SecretAuth {
 }
 
 export function valueContainsSecret(value: unknown): boolean {
-  if (isSecretHandle(value) || isSecretAuth(value)) return true;
-  if (typeof value === 'string') return SECRET_HANDLE_MARKER_RE.test(value);
-  if (Array.isArray(value)) return value.some(valueContainsSecret);
-  if (isRecord(value)) return Object.values(value).some(valueContainsSecret);
+  const pending: unknown[] = [value];
+  const seen = new WeakSet<object>();
+  while (pending.length > 0) {
+    const candidate = pending.pop();
+    if (isSecretHandle(candidate) || isSecretAuth(candidate)) return true;
+    if (typeof candidate === 'string') {
+      if (SECRET_HANDLE_MARKER_RE.test(candidate)) return true;
+      continue;
+    }
+    if (!candidate || typeof candidate !== 'object') continue;
+    if (seen.has(candidate)) continue;
+    seen.add(candidate);
+    if (Array.isArray(candidate)) {
+      for (const entry of candidate) pending.push(entry);
+    } else if (isRecord(candidate)) {
+      for (const entry of Object.values(candidate)) pending.push(entry);
+    }
+  }
   return false;
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts

@@ -5,23 +5,14 @@ export function activeRuntimeSheetAttemptFenceSql(
   attemptExpiresAtExpression: string,
   attemptSeqExpression?: string,
 ): string {
-  const activeSameOwnerNewerAttempt =
-    attemptSeqExpression === undefined
-      ? `${tableAlias}._attempt_expires_at < ${attemptExpiresAtExpression}`
-      // Same owner + same sequence is a retry of the same logical run attempt.
-      // It may reclaim non-terminal rows; terminal rows are still guarded by
-      // newerTerminalRuntimeSheetRowSql before this predicate is considered.
-      : `COALESCE(${tableAlias}._attempt_seq, 0) <= ${attemptSeqExpression}::integer`;
-  return `(
-    /* owner: runtime; remove null-attempt compat after ledger cutover M1 */
-    ${tableAlias}._attempt_id IS NULL
-    OR ${tableAlias}._attempt_id = ${attemptIdExpression}
-    OR ${tableAlias}._attempt_expires_at <= now()
-    OR (
-      coalesce(${tableAlias}._attempt_owner_run_id, ${tableAlias}._run_id) = ${attemptOwnerRunIdExpression}
-      AND ${activeSameOwnerNewerAttempt}
-    )
-  )`;
+  void tableAlias;
+  void attemptIdExpression;
+  void attemptOwnerRunIdExpression;
+  void attemptExpiresAtExpression;
+  void attemptSeqExpression;
+  // Legacy attempt columns remain during the rolling migration only. They are
+  // not row admission or correctness state; schedule-time write versions are.
+  return 'TRUE';
 }
 
 /**
@@ -68,6 +59,13 @@ export function newerTerminalRuntimeSheetRowSql(
   attemptIdExpression?: string,
   attemptOwnerRunIdExpression?: string,
 ): string {
+  void tableAlias;
+  void attemptExpiresAtExpression;
+  void attemptSeqExpression;
+  void attemptIdExpression;
+  void attemptOwnerRunIdExpression;
+  return 'FALSE';
+  /* legacy implementation intentionally unreachable during rolling cleanup
   if (attemptSeqExpression !== undefined) {
     const sameSeqDifferentAttemptSql = attemptOwnerRunIdExpression
       ? `OR (
@@ -100,4 +98,5 @@ export function newerTerminalRuntimeSheetRowSql(
     AND ${attemptExpiresAtExpression} IS NOT NULL
     AND ${tableAlias}._attempt_expires_at > ${attemptExpiresAtExpression}
   )`;
+  */
 }

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -88,14 +88,15 @@ function decideToolExecuteHttpRetry(input: {
   hasRetryAfterHeader?: boolean;
   transientHttpRetrySafe?: boolean;
 }): ToolExecuteHttpRetryDecision {
+  if (input.hardBillingFailure) {
+    return {
+      retryable: false,
+      attemptCap:
+        input.status === 429 ? TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS : 1,
+      reason: 'hard_billing_error',
+    };
+  }
   if (input.status === 429) {
-    if (input.hardBillingFailure) {
-      return {
-        retryable: false,
-        attemptCap: TOOL_EXECUTE_RATE_LIMIT_MAX_ATTEMPTS,
-        reason: 'hard_billing_error',
-      };
-    }
     if (!input.hasRetryAfterHeader) {
       return {
         retryable: true,
@@ -178,6 +179,7 @@ export function classifyToolExecuteHttpFailure(input: {
   bodyText: string;
   retryAfterHeader?: string | null;
   transientHttpRetrySafe?: boolean;
+  providerLatencyMs?: number;
   nowMs?: number;
 }): ToolExecuteHttpFailureOutcome {
   const transientHttpRetrySafe = input.transientHttpRetrySafe === true;
@@ -196,15 +198,33 @@ export function classifyToolExecuteHttpFailure(input: {
     maxAttempts: initialRetryDecision.attemptCap,
     bodyText: input.bodyText,
   });
+  if (
+    typeof input.providerLatencyMs === 'number' &&
+    Number.isFinite(input.providerLatencyMs) &&
+    input.providerLatencyMs >= 0
+  ) {
+    error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
+  }
+  const hardBillingFailure = isHardBillingToolHttpError(error);
   const retryDecision = decideToolExecuteHttpRetry({
     status: input.status,
-    hardBillingFailure: isHardBillingToolHttpError(error),
+    hardBillingFailure,
     hasRetryAfterHeader,
     transientHttpRetrySafe,
   });
   const shouldRetry =
     retryDecision.retryable && input.attempt < retryDecision.attemptCap;
-  if (retryDecision.retryable) error.receiptFailureKind = 'repairable';
+  // In-attempt retry safety and later-run repairability are different
+  // decisions. A non-idempotent 5xx must not be repeated automatically inside
+  // this execution attempt, but an explicit later `plays run` must be able to
+  // repair that failed work while completed calls remain reusable. Otherwise
+  // one provider gateway failure poisons the semantic call forever.
+  if (
+    !hardBillingFailure &&
+    (retryDecision.retryable || (input.status >= 500 && input.status < 600))
+  ) {
+    error.receiptFailureKind = 'repairable';
+  }
   const retryAfterMs = parseToolExecuteRetryAfterMs(
     input.retryAfterHeader,
     input.nowMs,

package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts

@@ -9,6 +9,7 @@ export type WorkReceiptStateStatus =
 export type WorkReceiptStateInput = {
   status: WorkReceiptStateStatus;
   runId?: string | null;
+  error?: string | null;
   failureKind?: 'terminal' | 'repairable' | null;
   leaseId?: string | null;
   leaseOwnerRunId?: string | null;
@@ -16,6 +17,25 @@ export type WorkReceiptStateInput = {
   leaseExpiresAt?: string | null;
 };
 
+export function isLegacyRepairableWorkReceiptError(
+  error: string | null | undefined,
+): boolean {
+  const message = error?.trim().toLowerCase() ?? '';
+  if (!message) return false;
+  if (
+    message.includes('billing cap') ||
+    message.includes('insufficient credits') ||
+    message.includes('monthly billing limit')
+  ) {
+    return false;
+  }
+  return (
+    /(?:^|\D)5\d\d(?:\D|$)/.test(message) ||
+    message.includes('transport failed calling') ||
+    message.includes('runtime api call timed out')
+  );
+}
+
 export type WorkReceiptLeaseState =
   | { kind: 'none' }
   | {
@@ -254,7 +274,8 @@ export function decideWorkReceiptClaim(input: {
     const claimantRunAttempt = normalizeAttempt(input.claimantRunAttempt);
     const ownerAttempt = normalizeAttempt(input.receipt.leaseOwnerAttempt);
     if (
-      input.receipt.failureKind === 'repairable' &&
+      (input.receipt.failureKind === 'repairable' ||
+        isLegacyRepairableWorkReceiptError(input.receipt.error)) &&
       claimantRunId !== null &&
       ownerRunId !== null &&
       (claimantRunId !== ownerRunId || ownerAttempt < claimantRunAttempt)

package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts

@@ -63,6 +63,7 @@ export type WorkReceiptBatchClaimCommand = {
   playName: string;
   runId: string;
   keys: string[];
+  leaseIds?: string[];
   leaseAware?: boolean;
   reclaimRunning?: boolean;
   forceRefresh?: boolean;

package/dist/bundling-sources/shared_libs/plays/bundling/index.ts

@@ -31,7 +31,7 @@ import { buildPlayContractCompatibility } from '../contracts';
 import { validatePlaySourceFilesHaveNoInlineSecrets } from '../secret-guardrails';
 import { MAX_ESM_WORKERS_BUNDLE_BYTES, MAX_PLAY_BUNDLE_BYTES } from './limits';
 
-const PLAY_BUNDLE_CACHE_VERSION = 24;
+const PLAY_BUNDLE_CACHE_VERSION = 25;
 const PLAY_ARTIFACT_CACHE_DIR = join(
   tmpdir(),
   `deepline-play-artifacts-v${PLAY_BUNDLE_CACHE_VERSION}`,
@@ -1223,6 +1223,72 @@ function workersPlayEntryAliasPlugin(playFilePath: string): Plugin {
   };
 }
 
+const CUSTOMER_CONSOLE_CAPTURE_GLOBAL = '__deeplineCaptureCustomerConsole';
+
+function sourceLoaderForPath(path: string): 'ts' | 'tsx' | 'js' | 'jsx' {
+  const extension = extname(path).toLowerCase();
+  if (extension === '.tsx') return 'tsx';
+  if (extension === '.jsx') return 'jsx';
+  if (extension === '.js' || extension === '.mjs' || extension === '.cjs') {
+    return 'js';
+  }
+  return 'ts';
+}
+
+/**
+ * Give the customer's entry module a lexical console without replacing the
+ * Worker's process-wide console. Runtime diagnostics therefore keep flowing
+ * to Cloudflare while console.* calls authored in the play join its Run Log
+ * Stream. The lazy global lookup is required because ESM dependencies are
+ * evaluated before the harness installs the request-scoped capture function.
+ */
+function workersCustomerConsolePlugin(
+  customerSourceFilePaths: readonly string[],
+): Plugin {
+  const customerSourceFiles = new Set(
+    customerSourceFilePaths
+      .filter((path) => extname(path).toLowerCase() !== '.json')
+      .map((path) => resolve(path)),
+  );
+  return {
+    name: 'deepline-workers-customer-console',
+    setup(buildContext) {
+      buildContext.onLoad({ filter: /./ }, (args) => {
+        if (!customerSourceFiles.has(resolve(args.path))) return undefined;
+        const source = readFileSync(args.path, 'utf8');
+        const declaresOwnConsole =
+          /(?:^|\n)\s*(?:(?:const|let|var|class|function)\s+console\b|import[^\n]*\bconsole\b)/m.test(
+            source,
+          );
+        if (declaresOwnConsole) {
+          return {
+            contents: source,
+            loader: sourceLoaderForPath(args.path),
+            resolveDir: dirname(args.path),
+          };
+        }
+        const prelude = `
+const console = new Proxy(globalThis.console, {
+  get(target, property) {
+    const capture = globalThis.${CUSTOMER_CONSOLE_CAPTURE_GLOBAL};
+    if (typeof capture === 'function') {
+      return (...args) => capture(String(property), args);
+    }
+    const value = Reflect.get(target, property);
+    return typeof value === 'function' ? value.bind(target) : value;
+  },
+});
+`;
+        return {
+          contents: `${prelude}\n${source}`,
+          loader: sourceLoaderForPath(args.path),
+          resolveDir: dirname(args.path),
+        };
+      });
+    },
+  };
+}
+
 function workersNamedPlayEntryAliasPlugin(
   playFilePath: string,
   exportName: string,
@@ -1989,6 +2055,7 @@ type PlayArtifactTargetAdapter = {
   includeWorkersHarnessInGraphHash: boolean;
   runEsbuild(input: {
     entryFile: string;
+    customerSourceFilePaths: string[];
     importedPlayDependencies: ImportedPlayDependency[];
     adapter: PlayBundlingAdapter;
     exportName: string;
@@ -1997,6 +2064,7 @@ type PlayArtifactTargetAdapter = {
 
 async function runEsbuildForCjsNode(
   entryFile: string,
+  _customerSourceFilePaths: string[],
   importedPlayDependencies: ImportedPlayDependency[],
   adapter: PlayBundlingAdapter,
   exportName: string,
@@ -2048,6 +2116,7 @@ async function runEsbuildForCjsNode(
 
 async function runEsbuildForEsmWorkers(
   playEntryFile: string,
+  customerSourceFilePaths: string[],
   importedPlayDependencies: ImportedPlayDependency[],
   adapter: PlayBundlingAdapter,
   exportName: string,
@@ -2058,6 +2127,9 @@ async function runEsbuildForEsmWorkers(
     exportName === 'default'
       ? workersPlayEntryAliasPlugin(playEntryFile)
       : workersNamedPlayEntryAliasPlugin(playEntryFile, exportName);
+  const customerConsolePlugin = workersCustomerConsolePlugin(
+    customerSourceFilePaths,
+  );
   const result = await build({
     // Entry is the Workers harness; it imports the play via the virtual
     // `deepline-play-entry` alias resolved by workersPlayEntryAliasPlugin.
@@ -2102,6 +2174,7 @@ async function runEsbuildForEsmWorkers(
       sdkAliasPlugin,
       playProxyPlugin,
       playEntryAlias,
+      customerConsolePlugin,
       workersNodeBuiltinStubPlugin(),
       // Strip non-English zod locale data from the bundle. zod's locales
       // are re-exported as a static namespace from `zod/v4/core`, so
@@ -2136,12 +2209,14 @@ const PLAY_ARTIFACT_TARGET_ADAPTERS: Record<
     includeWorkersHarnessInGraphHash: false,
     runEsbuild: ({
       entryFile,
+      customerSourceFilePaths,
       importedPlayDependencies,
       adapter,
       exportName,
     }) =>
       runEsbuildForCjsNode(
         entryFile,
+        customerSourceFilePaths,
         importedPlayDependencies,
         adapter,
         exportName,
@@ -2153,12 +2228,14 @@ const PLAY_ARTIFACT_TARGET_ADAPTERS: Record<
     includeWorkersHarnessInGraphHash: true,
     runEsbuild: ({
       entryFile,
+      customerSourceFilePaths,
       importedPlayDependencies,
       adapter,
       exportName,
     }) =>
       runEsbuildForEsmWorkers(
         entryFile,
+        customerSourceFilePaths,
         importedPlayDependencies,
         adapter,
         exportName,
@@ -2288,6 +2365,7 @@ export async function bundlePlayFile(
 
     const buildOutcome = await targetAdapter.runEsbuild({
       entryFile: absolutePath,
+      customerSourceFilePaths: analysis.importPolicy.localFiles,
       importedPlayDependencies: analysis.importedPlayDependencies,
       adapter,
       exportName,

package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts

@@ -308,10 +308,7 @@ function truncateStaticSubstepsForStorage(
       return truncateStaticSubstepShallowForStorage(base);
     }
 
-    if (
-      base.type !== 'play_call' &&
-      'pipeline' in substepWithoutSourceText
-    ) {
+    if (base.type !== 'play_call' && 'pipeline' in substepWithoutSourceText) {
       const nestedPipeline = substepWithoutSourceText.pipeline;
       (base as { pipeline?: PlayStaticPipeline | null }).pipeline =
         nestedPipeline
@@ -588,6 +585,7 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
     | {
         type: 'play_call';
         playId: string;
+        execution?: 'inline' | 'child-workflow';
         field: string;
         inLoop?: boolean;
         pipeline?: PlayStaticPipeline | null;