deepline 0.1.227 → 0.1.229

package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts

@@ -3,10 +3,7 @@ import {
   type RuntimeExecutionCapability,
 } from './execution-capabilities';
 
-export type RunExecutionPlacementKind =
-  | 'root'
-  | 'inline_child'
-  | 'scheduled_child';
+export type RunExecutionPlacementKind = 'root' | 'inline_child';
 
 export type RunExecutionAuthorityInput = {
   readonly orgId: string;
@@ -59,21 +56,12 @@ export type RootRunExecutionScopeInput = {
   readonly authority: RunExecutionAuthorityInput;
 };
 
-export type ChildRunExecutionScopeInput =
-  | {
-      readonly placement: 'inline';
-      readonly runId: string;
-      readonly playId: string;
-      readonly receiptNamespace: string;
-    }
-  | {
-      readonly placement: 'scheduled';
-      readonly runId: string;
-      readonly playId: string;
-      readonly attempt: number;
-      readonly receiptNamespace: string;
-      readonly authority?: Partial<Omit<RunExecutionAuthorityInput, 'orgId'>>;
-    };
+export type ChildRunExecutionScopeInput = {
+  readonly placement: 'inline';
+  readonly runId: string;
+  readonly playId: string;
+  readonly receiptNamespace: string;
+};
 
 function requireIdentity(value: string, field: string): string {
   if (
@@ -203,54 +191,18 @@ export function deriveChildRunExecutionScope(
     rootPlayId: parent.logical.rootPlayId,
   };
 
-  if (input.placement === 'inline') {
-    return freezeScope({
-      logical,
-      receipt: {
-        ownerRunId: parent.receipt.ownerRunId,
-        ownerAttempt: parent.receipt.ownerAttempt,
-        namespace,
-      },
-      placement: {
-        kind: 'inline_child',
-        executorRunId: parent.placement.executorRunId,
-        executorPlayId: parent.placement.executorPlayId,
-      },
-      authority: parent.authority,
-    });
-  }
-
-  const attempt = requireAttempt(input.attempt, 'attempt');
   return freezeScope({
     logical,
-    receipt: { ownerRunId: runId, ownerAttempt: attempt, namespace },
+    receipt: {
+      ownerRunId: parent.receipt.ownerRunId,
+      ownerAttempt: parent.receipt.ownerAttempt,
+      namespace,
+    },
     placement: {
-      kind: 'scheduled_child',
-      executorRunId: runId,
-      executorPlayId: playId,
+      kind: 'inline_child',
+      executorRunId: parent.placement.executorRunId,
+      executorPlayId: parent.placement.executorPlayId,
     },
-    authority: authorityMetadata({
-      runId,
-      playId,
-      orgId: parent.authority.orgId,
-      authority: {
-        workflowId: input.authority?.workflowId,
-        billingRunId:
-          input.authority?.billingRunId === undefined
-            ? parent.authority.billingRunId
-            : input.authority.billingRunId,
-        maxCreditsPerRun:
-          input.authority?.maxCreditsPerRun === undefined
-            ? parent.authority.maxCreditsPerRun
-            : input.authority.maxCreditsPerRun,
-        integrationMode:
-          input.authority?.integrationMode === undefined
-            ? parent.authority.integrationMode
-            : input.authority.integrationMode,
-        synthetic: input.authority?.synthetic ?? parent.authority.synthetic,
-        capabilities:
-          input.authority?.capabilities ?? parent.authority.capabilities,
-      },
-    }),
+    authority: parent.authority,
   });
 }

package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts

@@ -166,6 +166,12 @@ export type PlayRunLedgerEvent =
       playName?: string | null;
       runtimeBackend?: string | null;
     })
+  | (PlayRunLedgerBaseEvent & {
+      type: 'run.waiting';
+    })
+  | (PlayRunLedgerBaseEvent & {
+      type: 'run.resumed';
+    })
   | (PlayRunLedgerBaseEvent & {
       type: 'run.completed';
       result?: unknown;
@@ -1025,6 +1031,16 @@ export function reducePlayRunLedgerEvent(
   if (event.runId !== snapshot.runId) {
     return snapshot;
   }
+  // The scheduler's outbox can replay an older wait transition after its run
+  // reached a terminal state. That transition is no longer meaningful and
+  // must be byte-for-byte inert: even advancing `updatedAt` would re-publish
+  // a stale terminal snapshot to every observer.
+  if (
+    (event.type === 'run.waiting' || event.type === 'run.resumed') &&
+    shouldIgnoreNonTerminalAfterStickyTerminal(snapshot)
+  ) {
+    return snapshot;
+  }
   const occurredAt = Math.max(0, event.occurredAt);
   const base: PlayRunLedgerSnapshot = {
     ...snapshot,
@@ -1062,6 +1078,16 @@ export function reducePlayRunLedgerEvent(
           : 'running',
         startedAt: base.startedAt ?? occurredAt,
       });
+    case 'run.waiting':
+      return withTiming({
+        ...base,
+        status: 'waiting',
+      });
+    case 'run.resumed':
+      return withTiming({
+        ...base,
+        status: 'running',
+      });
     case 'run.completed':
       return (
         conflictingTerminalSnapshot(base, event.type, occurredAt, null, {

package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts

@@ -1,6 +1,7 @@
 export type PlayRunLifecycleStatus =
   | 'queued'
   | 'running'
+  | 'waiting'
   | 'completed'
   | 'failed'
   | 'cancelled'
@@ -35,8 +36,9 @@ export function normalizePlayRunLifecycleStatus(
       return 'running';
     case 'running':
     case 'started':
-    case 'waiting':
       return 'running';
+    case 'waiting':
+      return 'waiting';
     case 'completed':
     case 'complete':
     case 'succeeded':
@@ -62,7 +64,8 @@ export function isTerminalPlayRunLifecycleStatus(status: unknown): boolean {
 }
 
 export function isActivePlayRunLifecycleStatus(status: unknown): boolean {
-  return normalizePlayRunLifecycleStatus(status) === 'running';
+  const normalized = normalizePlayRunLifecycleStatus(status);
+  return normalized === 'running' || normalized === 'waiting';
 }
 
 export function isRecoverableDatasetRowStatus(status: unknown): boolean {

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts

@@ -2,10 +2,6 @@ export const PLAY_RUNTIME_API_COMPAT_PATH = '/api/v2/plays/internal/runtime';
 export const PLAY_RUNTIME_API_CURRENT_PATH = '/api/v2/internal/play-runtime';
 export const PLAY_RUNTIME_ATTEMPT_EXECUTOR_TOKEN_PATH =
   '/api/v2/internal/play-runtime/executor-token';
-export const PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH =
-  '/api/v2/plays/internal/child-executor-token';
-export const PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_CURRENT_PATH =
-  '/api/v2/internal/play-runtime/child-executor-token';
 export const PLAY_RUNTIME_TAIL_LOG_COMPAT_PATH =
   '/api/v2/plays/internal/tail-log';
 export const PLAY_RUNTIME_TAIL_LOG_CURRENT_PATH =

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -371,7 +371,22 @@ export type ResolvedRuntimePlay = {
   sourceCode?: string | null;
   artifact?: PlayBundleArtifact | null;
   codeFormat?: 'function' | 'cjs_module' | 'esm_module';
+  /**
+   * The runtime must receive the child contract with its artifact. Inline
+   * composition rejects unresolved or dataset-backed children before running
+   * their code, so dropping this field at the API boundary makes valid scalar
+   * children indistinguishable from unknown ones.
+   */
+  staticPipeline?: PlayStaticPipeline | null;
   contractSnapshot?: Record<string, unknown> | null;
+  /** Preview-only contract propagation evidence, emitted by the server and
+   * logged by the runner while diagnosing a cross-runtime child resolution
+   * failure. It intentionally contains no source, inputs, or credentials. */
+  resolutionDiagnostics?: {
+    serverPlayPipeline: boolean;
+    manifestPipeline: boolean;
+    responsePipeline: boolean;
+  } | null;
 };
 
 export type PrepareRuntimeSheetResult = {
@@ -522,6 +537,14 @@ function resolveRuntimeApiHeaders(
     'content-type': 'application/json',
     authorization: `Bearer ${token}`,
     [PLAY_RUNTIME_CONTRACT_HEADER]: String(PLAY_RUNTIME_CONTRACT),
+    // A run has many callback HTTP attempts. Preserve the logical run
+    // correlation in headers while Vercel keeps its own per-attempt request id.
+    ...(context.runId?.trim()
+      ? {
+          'x-deepline-run-id': context.runId.trim(),
+          'x-deepline-correlation-id': `run:${context.runId.trim()}`,
+        }
+      : {}),
     ...vercelHeaders,
     ...(context.runtimeTestFaultHeader?.trim()
       ? {
@@ -677,7 +700,7 @@ function runtimeApiMaxAttempts(
 }
 
 export function isTransientRuntimeApiErrorMessage(message: string): boolean {
-  return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access|RUNTIME_SCHEDULER_SATURATED|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open/i.test(
+  return /timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|requested endpoint could not be found, or you don't have access|RUNTIME_SCHEDULER_(?:SATURATED|UNAVAILABLE)|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open/i.test(
     message,
   );
 }
@@ -1779,11 +1802,22 @@ function isMissingRelationError(error: unknown): boolean {
 }
 
 function isPostgresPermissionDeniedError(error: unknown): boolean {
-  return (
-    error !== null &&
-    typeof error === 'object' &&
-    'code' in error &&
-    String(error.code) === '42501'
+  if (error === null || typeof error !== 'object') {
+    return false;
+  }
+  if ('code' in error && String(error.code) === '42501') {
+    return true;
+  }
+  const message =
+    'message' in error && typeof error.message === 'string'
+      ? error.message
+      : '';
+  // Postgres errors raised by a direct session retain SQLSTATE 42501. The
+  // same failure crossing the runtime API boundary is intentionally rendered
+  // as a safe Error message, so preserve the exact permission-denied shape as
+  // the repair signal without treating arbitrary 500s as storage drift.
+  return /permission denied for (?:database|schema|relation|table|sequence)/i.test(
+    message,
   );
 }
 
@@ -2932,15 +2966,28 @@ export async function resolveRuntimeReferencedPlay(
   playRef: string,
   options?: { artifactKind?: PlayArtifactKind },
 ): Promise<ResolvedRuntimePlay | null> {
-  const response = await postRuntimeApi<{ play: ResolvedRuntimePlay | null }>(
-    context,
-    {
-      action: 'resolve_play',
-      playRef,
-      ...(options?.artifactKind ? { artifactKind: options.artifactKind } : {}),
-    },
-  );
-  return response.play;
+  const response = await postRuntimeApi<{
+    play: ResolvedRuntimePlay | null;
+    manifest?: { staticPipeline?: PlayStaticPipeline | null } | null;
+  }>(context, {
+    action: 'resolve_play',
+    playRef,
+    ...(options?.artifactKind ? { artifactKind: options.artifactKind } : {}),
+  });
+  if (!response.play) {
+    return null;
+  }
+
+  // The server builds the manifest and child artifact together. Its manifest
+  // carries the canonical empty scalar contract when a published scalar's
+  // revision snapshot has no pipeline. Keep that contract on the object the
+  // runner passes to ctx.runPlay; otherwise a valid scalar leaf arrives as an
+  // indistinguishable "missing static contract" at runtime.
+  return {
+    ...response.play,
+    staticPipeline:
+      response.play.staticPipeline ?? response.manifest?.staticPipeline ?? null,
+  };
 }
 
 export async function ensureRuntimeSheet(
@@ -2951,12 +2998,28 @@ export async function ensureRuntimeSheet(
     sheetContract: PlaySheetContract;
   },
 ): Promise<void> {
-  await postRuntimeApi<{ ok: true }>(context, {
-    action: 'ensure_sheet',
+  const request = {
+    action: 'ensure_sheet' as const,
     ...input,
     runId: context.runId ?? null,
     userEmail: normalizeRuntimeUserEmail(context.userEmail),
-  });
+  };
+  try {
+    await postRuntimeApi<{ ok: true }>(context, request);
+  } catch (error) {
+    if (!isPostgresPermissionDeniedError(error)) {
+      throw error;
+    }
+    // Fresh tenant databases can become visible through the pooled endpoint
+    // before their runtime-role grants are usable. Non-preloaded file runs hit
+    // ensure_sheet before any direct Postgres operation, so the existing
+    // operation-level self-heal cannot observe this boundary. Repair the
+    // canonical storage contract once, then retry the exact ensure request.
+    await repairRuntimeStorageGrants(context, {
+      playName: input.playName,
+    });
+    await postRuntimeApi<{ ok: true }>(context, request);
+  }
 }
 
 async function prepareRuntimeSheetDatasetRows(

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts

@@ -17,7 +17,6 @@ import type {
   PlayRowUpdate,
 } from './ctx-types';
 import type { ExecutionPlan } from './execution-plan';
-import type { PlayRuntimeManifestMap } from '../plays/compiler-manifest';
 import type { PreloadedRuntimeDbSession } from './db-session';
 import type { RuntimeAuthorityDescriptor } from './execution-capabilities';
 import type { PlayRunnerRuntimeTiming } from './protocol';
@@ -42,39 +41,6 @@ export type PlaySchedulerBackendId =
 
 export type { PlayRunInputPayload };
 
-export type PlayCallGovernanceSnapshot = {
-  rootRunId: string;
-  parentRunId: string;
-  parentPlayName: string;
-  key: string;
-  ancestryPlayIds: string[];
-  callDepth: number;
-  /**
-   * Cumulative lineage-global budget counters consumed by ancestors at the
-   * moment this child was launched. The child seeds its own counters from these
-   * so the corresponding budgets (`maxPlayCallCount`, `maxToolCallCount`,
-   * `maxRetryCount`, `maxDescendants`, `maxWaterfallStepExecutions`) accumulate
-   * down the dispatch lineage instead of resetting to 0 in each worker isolate —
-   * matching the cjs forkChild path, which threads all of them. The Governor
-   * documents these budgets as global across the nested-play tree, not
-   * per-worker (see policy.ts / rate-state-backend.ts); threading them here is
-   * what makes that true on `esm_workers`.
-   *
-   * `descendantCount` is load-bearing for fan-out: forkChild charges
-   * `descendant` on every child launch, so without threading it a deep/wide tree
-   * would reset descendant accounting at each isolate and never converge on the
-   * lineage-global cap.
-   *
-   * All optional and fail-safe: if absent (older callers / dropped in transit)
-   * the child falls back to 0, i.e. prior behavior.
-   */
-  playCallCount?: number;
-  toolCallCount?: number;
-  retryCount?: number;
-  descendantCount?: number;
-  waterfallStepExecutions?: number;
-};
-
 export type PlaySchedulerSubmitInput = {
   runId: string;
   playId: string;
@@ -122,8 +88,6 @@ export type PlaySchedulerSubmitInput = {
   }> | null;
   contractSnapshot?: unknown;
   executionPlan?: ExecutionPlan | null;
-  childPlayManifests?: PlayRuntimeManifestMap | null;
-  playCallGovernance?: PlayCallGovernanceSnapshot | null;
   preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
   /**
    * Secret that decrypts `preloadedDbSessions` postgres URLs. Equals the run's

package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts

@@ -18,20 +18,6 @@ export type PlayExecutionSuspension =
         timeoutMs: number;
       }>;
     }
-  | {
-      kind: 'child_play';
-      boundaryId: string;
-      childRunId: string;
-      childPlayName: string;
-    }
-  | {
-      kind: 'child_play_batch';
-      boundaries: Array<{
-        boundaryId: string;
-        childRunId: string;
-        childPlayName: string;
-      }>;
-    }
   | {
       /**
        * Push-execution park (B2-final): the Daytona runner command was started
@@ -72,13 +58,9 @@ export class PlayExecutionSuspendedError extends Error {
         ? `Play execution suspended for durable sleep (${suspension.delayMs}ms).`
         : suspension.kind === 'integration_event_batch'
           ? `Play execution suspended waiting for ${suspension.boundaries.length} integration events.`
-          : suspension.kind === 'child_play_batch'
-            ? `Play execution suspended waiting for ${suspension.boundaries.length} child plays.`
-            : suspension.kind === 'child_play'
-              ? `Play execution suspended waiting for child play ${JSON.stringify(suspension.childPlayName)}.`
-              : suspension.kind === 'detached_runner'
-                ? `Play execution parked on detached runner ${suspension.cmdId} (sandbox ${suspension.sandboxId}).`
-                : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
+          : suspension.kind === 'detached_runner'
+            ? `Play execution parked on detached runner ${suspension.cmdId} (sandbox ${suspension.sandboxId}).`
+            : `Play execution suspended waiting for integration event ${JSON.stringify(suspension.eventKey)}.`,
     );
     this.name = 'PlayExecutionSuspendedError';
     this.suspension = suspension;

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -5,7 +5,14 @@ export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
 export type RuntimeTestFaultName =
   | 'receipt_complete_write_fail'
   | 'receipt_fail_write_fail'
-  | 'worker_receipt_complete_write_fail';
+  | 'worker_receipt_complete_write_fail'
+  /**
+   * Holds an already checked-out receipt-gateway scheduler client for the
+   * supplied bounded millisecond value. Preview/CI only: this creates a
+   * deterministic real-Pg-pool pressure window without changing an action's
+   * SQL or outcome.
+   */
+  | 'receipt_gateway_hold_ms';
 
 export type RuntimeTestFaultRegistry = {
   consume(name: RuntimeTestFaultName): boolean;
@@ -63,6 +70,7 @@ const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
   'receipt_complete_write_fail',
   'receipt_fail_write_fail',
   'worker_receipt_complete_write_fail',
+  'receipt_gateway_hold_ms',
 ]);
 
 const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([

package/dist/bundling-sources/shared_libs/play-runtime/transport-error-diagnostics.ts

@@ -0,0 +1,63 @@
+import { redactLedgerString } from './ledger-safe-payload';
+
+type ErrorLike = Record<string, unknown>;
+
+export type TransportErrorDiagnostic = {
+  name: string | null;
+  message: string | null;
+  code: string | null;
+  errno: string | null;
+  syscall: string | null;
+  cause: TransportErrorDiagnostic | null;
+};
+
+function asErrorLike(value: unknown): ErrorLike | null {
+  return value !== null && typeof value === 'object'
+    ? (value as ErrorLike)
+    : null;
+}
+
+function safeString(value: unknown): string | null {
+  if (typeof value !== 'string' && typeof value !== 'number') return null;
+  return redactLedgerString(String(value))?.slice(0, 500) ?? null;
+}
+
+export function transportGatewayOriginForDiagnostic(url: string): string {
+  try {
+    return new URL(url).origin;
+  } catch {
+    return '<invalid-gateway-origin>';
+  }
+}
+
+/**
+ * Make a small, secret-redacted transport diagnostic that can safely cross the
+ * runner -> gateway -> durable run-log boundary. Keep stacks in process logs:
+ * serializing them into run state leaks implementation details and is rarely
+ * useful without the original error's code/errno/cause chain.
+ */
+export function describeTransportError(
+  error: unknown,
+  depth = 0,
+): TransportErrorDiagnostic {
+  const value = asErrorLike(error);
+  const errorValue = error instanceof Error ? error : null;
+  const name =
+    safeString(errorValue?.name ?? value?.name) ?? (value ? 'Error' : null);
+  const message = safeString(errorValue?.message ?? value?.message ?? error);
+  const cause = value?.cause;
+
+  return {
+    name,
+    message,
+    code: safeString(value?.code),
+    errno: safeString(value?.errno),
+    syscall: safeString(value?.syscall),
+    // A short cause chain is enough for fetch/undici errors and prevents a
+    // pathological custom error graph from making durable traces unbounded.
+    cause:
+      cause !== undefined && depth < 2
+        ? describeTransportError(cause, depth + 1)
+        : null,
+  };
+}

package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts

@@ -586,6 +586,8 @@ export type PlayStaticSubstep = PlayStaticSubstepMetadata &
         type: 'play_call';
         playId: string;
         execution?: 'inline' | 'child-workflow';
+        timeoutMs?: number;
+        hasExplicitTimeout?: boolean;
         field: string;
         inLoop?: boolean;
         pipeline?: PlayStaticPipeline | null;