deepline 0.3.88 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/postgres-fence-adapter.ts

@@ -0,0 +1,263 @@
+import { randomUUID } from 'node:crypto';
+
+import {
+  allocatePostgresSchedulerNextAttempt,
+  assignPostgresSchedulerRunAttempt,
+  deferPostgresSchedulerAttemptForCapacity,
+  deferPostgresSchedulerAttemptForRecovery,
+  handoffPostgresSchedulerAttemptContinuation,
+  markPostgresSchedulerAttemptRunning,
+  markPostgresSchedulerAttemptWaiting,
+  readPostgresSchedulerResult,
+  resumePostgresSchedulerDetachedRunnerAttempt,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerQueryClient,
+} from '../../scheduler-backends/postgres';
+import { PLAY_RUN_RECOVERY_DEFER_POLICY } from '../../play-run-recovery-policy';
+import type {
+  PlayRunAttemptClaim,
+  PlayRunAttemptTerminalEnvelope,
+} from '../../play-runs/attempts/contracts';
+import type { PlayRunAttemptStore } from '../../play-runs/attempts/ports';
+
+/**
+ * Runtime-Postgres Adapter for Play Run Attempt.
+ *
+ * The Adapter intentionally owns the scheduler's real transition vocabulary:
+ * fresh versus detached-fence claim, finite pre-code recovery, typed parking,
+ * ordinary wake, and detached continuation handoff. It does not execute
+ * sandboxes or shape runner-returned terminal output; those remain the
+ * Sandbox Runs and Terminal Modules behind their own seams. A bounded
+ * pre-code recovery exhaustion is the one scheduler-owned terminal envelope
+ * this Adapter constructs.
+ */
+export type PostgresPlayRunAttemptStore = PlayRunAttemptStore;
+
+export function createPostgresPlayRunAttemptStore(input: {
+  client: PostgresSchedulerQueryClient;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+  /** Injectable only to make Adapter conformance vectors deterministic. */
+  createFenceId?: () => string;
+  nowMs?: () => number;
+}): PostgresPlayRunAttemptStore {
+  const options = input.schedulerOptions;
+  const createFenceId = input.createFenceId ?? randomUUID;
+  const nowMs = input.nowMs ?? Date.now;
+  const freshClaim = async (run: { runId: string }) => {
+    // This intentionally retains the worker's allocate-then-assign law.
+    // Races may choose the same generation, but the assignment SQL admits
+    // only one of them; a loser observes null and must retire without work.
+    const number = await allocatePostgresSchedulerNextAttempt(
+      input.client,
+      run.runId,
+      options,
+    );
+    const fenceId = createFenceId();
+    const assigned = await assignPostgresSchedulerRunAttempt(
+      input.client,
+      { runId: run.runId, attempt: number, taskToken: fenceId },
+      options,
+    );
+    if (assigned === null) return null;
+    return {
+      runId: run.runId,
+      number,
+      fenceId,
+      checkpoint: assigned.checkpoint ?? null,
+      origin: 'fresh' as const,
+    };
+  };
+  return {
+    async readTerminal({ runId }) {
+      const terminal = await readPostgresSchedulerResult(
+        input.client,
+        runId,
+        options,
+      );
+      if (terminal === null) return null;
+      return {
+        status: terminal.status,
+        ...(Object.hasOwn(terminal, 'output')
+          ? { output: terminal.output }
+          : {}),
+        ...(Object.hasOwn(terminal, 'error') ? { error: terminal.error } : {}),
+      };
+    },
+    async resumeDetached(run) {
+      const resumed = await resumePostgresSchedulerDetachedRunnerAttempt(
+        input.client,
+        { runId: run.runId },
+        options,
+      );
+      if (!resumed) return null;
+      const suspension = detachedRunnerSuspension(resumed.suspension);
+      return {
+        runId: run.runId,
+        number: resumed.attempt,
+        fenceId: resumed.leaseToken,
+        checkpoint: resumed.checkpoint ?? null,
+        origin: 'detached_runner_resume' as const,
+        suspension,
+      };
+    },
+    claim: freshClaim,
+    async deferBeforeCode({ claim, deferral }) {
+      if (deferral.kind === 'capacity') {
+        const deferred = await deferPostgresSchedulerAttemptForCapacity(
+          input.client,
+          {
+            runId: claim.runId,
+            attempt: claim.number,
+            leaseToken: claim.fenceId,
+            reason: deferral.reason,
+          },
+          options,
+        );
+        return deferred
+          ? { kind: 'deferred' as const }
+          : { kind: 'fence_lost' as const };
+      }
+      const decision = deferral.decision;
+      if (decision.stage !== 'pre_code' || decision.action !== 'defer') {
+        throw new Error(
+          'Play Run Attempt recovery deferral requires a pre-code defer decision.',
+        );
+      }
+      const recovery = await deferPostgresSchedulerAttemptForRecovery(
+        input.client,
+        {
+          runId: claim.runId,
+          attempt: claim.number,
+          leaseToken: claim.fenceId,
+          reason: deferral.reason,
+          dependency: decision.dependency,
+          openCircuit:
+            decision.dependency === 'modal' &&
+            decision.reason !== 'modal_circuit_open_before_execution',
+          nowMs: nowMs(),
+          maxDefers: PLAY_RUN_RECOVERY_DEFER_POLICY.maxDefers,
+          horizonMs: PLAY_RUN_RECOVERY_DEFER_POLICY.horizonMs,
+          deferBaseMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferBaseSeconds * 1_000,
+          deferMaxMs: PLAY_RUN_RECOVERY_DEFER_POLICY.deferMaxSeconds * 1_000,
+        },
+        options,
+      );
+      if (recovery.deferred) return { kind: 'deferred' as const };
+      if (!recovery.exhausted) return { kind: 'fence_lost' as const };
+      return {
+        kind: 'exhausted' as const,
+        terminal: exhaustedRecoveryTerminal({
+          dependency: decision.dependency,
+          checkpoint: claim.checkpoint,
+        }),
+      };
+    },
+    async park({ claim, checkpoint, suspension }) {
+      const parked = await markPostgresSchedulerAttemptWaiting(
+        input.client,
+        {
+          runId: claim.runId,
+          attempt: claim.number,
+          leaseToken: claim.fenceId,
+          checkpoint,
+          suspension,
+          preserveHeartbeatAt: suspension.kind === 'detached_runner',
+        },
+        options,
+      );
+      return parked
+        ? { kind: 'parked', lifecycle: parked }
+        : { kind: 'fence_lost' };
+    },
+    async resumeAfterPark({ claim, suspension }) {
+      const resumed = await markPostgresSchedulerAttemptRunning(
+        input.client,
+        {
+          runId: claim.runId,
+          attempt: claim.number,
+          leaseToken: claim.fenceId,
+          preserveHeartbeatAt: suspension.kind === 'detached_runner',
+        },
+        options,
+      );
+      return resumed
+        ? { kind: 'resumed', lifecycle: resumed }
+        : { kind: 'fence_lost' };
+    },
+    async handoffDetachedContinuation({ claim, checkpoint }) {
+      if (claim.origin !== 'detached_runner_resume') {
+        throw new Error(
+          'Only a resumed detached-runner attempt may hand off a fresh continuation.',
+        );
+      }
+      const handedOff = await handoffPostgresSchedulerAttemptContinuation(
+        input.client,
+        {
+          runId: claim.runId,
+          attempt: claim.number,
+          leaseToken: claim.fenceId,
+          checkpoint,
+        },
+        options,
+      );
+      return handedOff
+        ? { kind: 'handed_off', lifecycle: handedOff }
+        : { kind: 'fence_lost' };
+    },
+  };
+}
+
+function detachedRunnerSuspension(
+  value: unknown,
+): NonNullable<PlayRunAttemptClaim['suspension']> {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    (value as { kind?: unknown }).kind !== 'detached_runner'
+  ) {
+    throw new Error(
+      'Detached Attempt resume was selected without a detached_runner suspension.',
+    );
+  }
+  const candidate = value as Record<string, unknown>;
+  if (
+    !Number.isSafeInteger(candidate.runnerAttempt) ||
+    typeof candidate.boundaryId !== 'string' ||
+    typeof candidate.sandboxId !== 'string' ||
+    typeof candidate.sessionId !== 'string' ||
+    typeof candidate.cmdId !== 'string' ||
+    typeof candidate.outputPath !== 'string' ||
+    typeof candidate.exitCodePath !== 'string' ||
+    !Number.isFinite(candidate.startedAtMs) ||
+    !Number.isFinite(candidate.heartbeatTimeoutMs) ||
+    !Number.isFinite(candidate.ceilingMs)
+  ) {
+    throw new Error(
+      'Detached Attempt resume found an invalid persisted detached_runner suspension.',
+    );
+  }
+  return value as NonNullable<PlayRunAttemptClaim['suspension']>;
+}
+
+/** @deprecated Use `createPostgresPlayRunAttemptStore`; the old name retained
+ * for the migration while callers move from the claim-only Interface. */
+export const createPostgresPlayRunAttemptFence =
+  createPostgresPlayRunAttemptStore;
+
+function exhaustedRecoveryTerminal(input: {
+  dependency: 'modal' | 'runtime';
+  checkpoint: unknown;
+}): PlayRunAttemptTerminalEnvelope {
+  // The checkpoint is intentionally not returned to customer output. It is
+  // already durable on the run row; the Terminal Module only needs a typed
+  // failure envelope to settle first-write-wins truth.
+  void input.checkpoint;
+  return {
+    status: 'failed',
+    error:
+      `PLATFORM_RECOVERY_EXHAUSTED: ${input.dependency} remained unavailable ` +
+      'before customer execution began through the bounded recovery horizon. ' +
+      'No customer code was started; inspect and redrive this run after the dependency is healthy.',
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/attempts/worker-terminal-projection.ts

@@ -0,0 +1,64 @@
+import type { PlayRunAttemptTerminalEnvelope } from '../../play-runs/attempts/contracts';
+
+/** The committed terminal fact used by worker-side read-model projection. */
+export type PlayRunAttemptTerminalCommit =
+  | { status: 'completed'; output: unknown }
+  | { status: 'failed'; error: string };
+
+/**
+ * Post-commit effects are deliberately separate from terminal truth. Terminal
+ * has already won first-write-wins before this Interface is invoked; these
+ * effects improve observability and release ephemeral receipt leases, but
+ * cannot alter customer-visible terminal state.
+ */
+export type PlayRunAttemptWorkerTerminalProjectionPort = {
+  observeFenceLost(): Promise<void>;
+  projectCommitted(input: {
+    runnerResult: PlayRunAttemptTerminalEnvelope;
+    terminal: PlayRunAttemptTerminalCommit;
+  }): Promise<void>;
+};
+
+export type PlayRunAttemptWorkerTerminalProjectionPorts = {
+  reconcileTerminalTruth(input: {
+    runnerResult: PlayRunAttemptTerminalEnvelope;
+    terminal: PlayRunAttemptTerminalCommit;
+  }): Promise<void>;
+  releaseFailedAttempt(input: {
+    runnerResult: PlayRunAttemptTerminalEnvelope;
+    terminal: Extract<PlayRunAttemptTerminalCommit, { status: 'failed' }>;
+  }): Promise<void>;
+  queueProgressFinalization(input: {
+    runnerResult: PlayRunAttemptTerminalEnvelope;
+    terminal: PlayRunAttemptTerminalCommit;
+  }): void;
+  requestTerminalProjection(input: {
+    terminal: PlayRunAttemptTerminalCommit;
+  }): void;
+  observeFenceLost(): Promise<void>;
+};
+
+/**
+ * Builds the worker projection Module. The ordering is its Interface:
+ * reconcile durable truth, release failed-attempt leases, then enqueue
+ * best-effort progress/outbox projection. A stale terminal fence observes the
+ * winner only; it never emits a second projection sequence.
+ */
+export function createPlayRunAttemptWorkerTerminalProjection(
+  ports: PlayRunAttemptWorkerTerminalProjectionPorts,
+): PlayRunAttemptWorkerTerminalProjectionPort {
+  return {
+    observeFenceLost: ports.observeFenceLost,
+    async projectCommitted(input) {
+      await ports.reconcileTerminalTruth(input);
+      if (input.terminal.status === 'failed') {
+        await ports.releaseFailedAttempt({
+          runnerResult: input.runnerResult,
+          terminal: input.terminal,
+        });
+      }
+      ports.queueProgressFinalization(input);
+      ports.requestTerminalProjection({ terminal: input.terminal });
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts

@@ -0,0 +1,29 @@
+/**
+ * Run Ledger Projection is the delivery job for already-committed Runtime
+ * Postgres outbox facts. It never decides a Play Run's terminal state and it
+ * never re-executes a Play.
+ */
+
+/** A caller may ask the durable outbox to advance through one exact event. */
+export type RunLedgerProjectionInput = {
+  /** Omit to advance the next claimable outbox fact. */
+  targetEventId?: string;
+};
+
+/**
+ * Durable delivery counters. `targetDelivered` is meaningful only for a
+ * targeted call; callers must not infer it from `projected`.
+ */
+export type RunLedgerProjectionStats = {
+  attempted: number;
+  projected: number;
+  failed: number;
+  retrying: number;
+  failedEventIds: string[];
+  targetEventId?: string;
+  targetDelivered?: boolean;
+};
+
+export type RunLedgerProjectionJob = {
+  deliver(input?: RunLedgerProjectionInput): Promise<RunLedgerProjectionStats>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -0,0 +1,56 @@
+import type { WorkerRuntimeApiContext } from '../../app-runtime-api';
+
+import {
+  dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope,
+  dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerOutboxDrainResult,
+  type PostgresSchedulerQueryClient,
+} from '../../scheduler-backends/postgres';
+
+import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports';
+
+export type GatewayRunLedgerProjectionAdapterInput = {
+  runId: string;
+  runtimeApi: WorkerRuntimeApiContext;
+  withClient: <T>(
+    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+};
+
+/**
+ * Gateway Adapter for the durable Run Ledger outbox. It retains the worker's
+ * no-signing-key rule and turns both normal and targeted delivery into the
+ * one `RunLedgerProjection.deliver` port.
+ */
+export function createGatewayRunLedgerProjectionAdapter(
+  input: GatewayRunLedgerProjectionAdapterInput,
+): RunLedgerProjectionPorts['outbox'] {
+  const options = input.schedulerOptions ?? {};
+  const dispatchInput = {
+    runId: input.runId,
+    runtimeApi: input.runtimeApi,
+    tokenAuthority: 'gateway' as const,
+  };
+  return {
+    async deliver(request) {
+      if (request.targetEventId) {
+        return await dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope(
+          input.withClient,
+          { ...dispatchInput, targetEventId: request.targetEventId },
+          options,
+        );
+      }
+      return await dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+        input.withClient,
+        dispatchInput,
+        options,
+      );
+    },
+  };
+}
+
+/** The targeted scheduler result has all public projection counters. */
+export type GatewayRunLedgerProjectionResult =
+  PostgresSchedulerOutboxDrainResult;

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts

@@ -0,0 +1,15 @@
+export { createRunLedgerProjectionJob } from '../../play-runs/projection/job';
+export {
+  createGatewayRunLedgerProjectionAdapter,
+  type GatewayRunLedgerProjectionAdapterInput,
+  type GatewayRunLedgerProjectionResult,
+} from '../../play-runs/projection/gateway-adapter';
+export type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionJob,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+export type {
+  RunLedgerOutboxDelivery,
+  RunLedgerProjectionPorts,
+} from '../../play-runs/projection/ports';

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts

@@ -0,0 +1,65 @@
+import type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionJob,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports';
+
+/**
+ * Creates the Run Ledger Projection job.
+ *
+ * A target is a delivery observation, not a second queue. In particular, a
+ * target that remains pending must return the adapter's durable retry result
+ * rather than causing the caller to append or retry an event itself.
+ */
+export function createRunLedgerProjectionJob(
+  ports: RunLedgerProjectionPorts,
+): RunLedgerProjectionJob {
+  return {
+    async deliver(
+      input: RunLedgerProjectionInput = {},
+    ): Promise<RunLedgerProjectionStats> {
+      if (input.targetEventId !== undefined && !input.targetEventId.trim()) {
+        throw new Error(
+          'Run Ledger Projection target event id must be non-empty when supplied.',
+        );
+      }
+
+      const result = await ports.outbox.deliver(input);
+      validateStats(result, input);
+      return result;
+    },
+  };
+}
+
+function validateStats(
+  result: RunLedgerProjectionStats,
+  input: RunLedgerProjectionInput,
+): void {
+  for (const value of [
+    result.attempted,
+    result.projected,
+    result.failed,
+    result.retrying,
+  ]) {
+    if (!Number.isSafeInteger(value) || value < 0) {
+      throw new Error(
+        'Run Ledger Projection adapter returned invalid delivery counters.',
+      );
+    }
+  }
+  if (!Array.isArray(result.failedEventIds)) {
+    throw new Error(
+      'Run Ledger Projection adapter returned invalid failed event ids.',
+    );
+  }
+  if (
+    input.targetEventId !== undefined &&
+    result.targetEventId !== undefined &&
+    result.targetEventId !== input.targetEventId
+  ) {
+    throw new Error(
+      'Run Ledger Projection adapter returned delivery stats for a different target event.',
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts

@@ -0,0 +1,17 @@
+import type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+
+/**
+ * The only infrastructure capability this job needs. The adapter owns the
+ * Postgres claim, ordered append, delivery idempotency, and durable retry
+ * clock; this job gives callers one stable Interface over that capability.
+ */
+export type RunLedgerOutboxDelivery = {
+  deliver(input: RunLedgerProjectionInput): Promise<RunLedgerProjectionStats>;
+};
+
+export type RunLedgerProjectionPorts = {
+  outbox: RunLedgerOutboxDelivery;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts

@@ -0,0 +1,42 @@
+/**
+ * Sandbox Cleanup reconciles one already-claimed physical resource. It begins
+ * after a resource fact and cleanup obligation exist; it neither creates a
+ * sandbox nor decides a Play Run terminal state.
+ */
+
+export type SandboxCleanupResource = {
+  provider: 'daytona' | 'modal';
+  sandboxId: string;
+  routingDomain: string | null;
+  runId: string;
+  attempt: number;
+  cleanupAttempt: number;
+  failureCount: number;
+};
+
+export type SandboxCleanupDeleteOutcome =
+  | { kind: 'deleted'; routingDomain: string | null }
+  | { kind: 'already_absent'; routingDomain: string | null }
+  /** The provider outcome is ambiguous or failed; its resource remains live. */
+  | { kind: 'failed'; code: string; detail: string };
+
+export type SandboxCleanupInput = {
+  resource: SandboxCleanupResource;
+  timeoutMs: number;
+};
+
+export type SandboxCleanupOutcome =
+  | {
+      kind: 'completed';
+      providerOutcome: 'deleted' | 'already_absent';
+      /** The durable claim transition which settled this provider fact. */
+      completion: 'completed' | 'late_success_accepted' | 'already_deleted';
+    }
+  | { kind: 'completion_rejected'; completion: 'missing_obligation' }
+  | { kind: 'blocked'; reason: string }
+  | { kind: 'retry_scheduled'; retryAfterMs: number }
+  | { kind: 'retry_rejected' };
+
+export type SandboxCleanupJob = {
+  reconcile(input: SandboxCleanupInput): Promise<SandboxCleanupOutcome>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts

@@ -0,0 +1,17 @@
+export { createSandboxCleanupJob } from '../../play-runs/sandbox-cleanup/job';
+export {
+  createPostgresSandboxCleanupStore,
+  type SandboxCleanupPostgresAdapterInput,
+} from '../../play-runs/sandbox-cleanup/postgres-adapter';
+export type {
+  SandboxCleanupDeleteOutcome,
+  SandboxCleanupInput,
+  SandboxCleanupJob,
+  SandboxCleanupOutcome,
+  SandboxCleanupResource,
+} from '../../play-runs/sandbox-cleanup/contracts';
+export type {
+  SandboxCleanupPorts,
+  SandboxCleanupStore,
+  SandboxDeletion,
+} from '../../play-runs/sandbox-cleanup/ports';