deepline 0.3.89 → 0.3.90

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/contracts.ts

@@ -0,0 +1,29 @@
+/**
+ * Run Ledger Projection is the delivery job for already-committed Runtime
+ * Postgres outbox facts. It never decides a Play Run's terminal state and it
+ * never re-executes a Play.
+ */
+
+/** A caller may ask the durable outbox to advance through one exact event. */
+export type RunLedgerProjectionInput = {
+  /** Omit to advance the next claimable outbox fact. */
+  targetEventId?: string;
+};
+
+/**
+ * Durable delivery counters. `targetDelivered` is meaningful only for a
+ * targeted call; callers must not infer it from `projected`.
+ */
+export type RunLedgerProjectionStats = {
+  attempted: number;
+  projected: number;
+  failed: number;
+  retrying: number;
+  failedEventIds: string[];
+  targetEventId?: string;
+  targetDelivered?: boolean;
+};
+
+export type RunLedgerProjectionJob = {
+  deliver(input?: RunLedgerProjectionInput): Promise<RunLedgerProjectionStats>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/gateway-adapter.ts

@@ -0,0 +1,56 @@
+import type { WorkerRuntimeApiContext } from '../../app-runtime-api';
+
+import {
+  dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope,
+  dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerOutboxDrainResult,
+  type PostgresSchedulerQueryClient,
+} from '../../scheduler-backends/postgres';
+
+import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports';
+
+export type GatewayRunLedgerProjectionAdapterInput = {
+  runId: string;
+  runtimeApi: WorkerRuntimeApiContext;
+  withClient: <T>(
+    operation: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+};
+
+/**
+ * Gateway Adapter for the durable Run Ledger outbox. It retains the worker's
+ * no-signing-key rule and turns both normal and targeted delivery into the
+ * one `RunLedgerProjection.deliver` port.
+ */
+export function createGatewayRunLedgerProjectionAdapter(
+  input: GatewayRunLedgerProjectionAdapterInput,
+): RunLedgerProjectionPorts['outbox'] {
+  const options = input.schedulerOptions ?? {};
+  const dispatchInput = {
+    runId: input.runId,
+    runtimeApi: input.runtimeApi,
+    tokenAuthority: 'gateway' as const,
+  };
+  return {
+    async deliver(request) {
+      if (request.targetEventId) {
+        return await dispatchRunLedgerOutboxToLedgerThroughEventWithClientScope(
+          input.withClient,
+          { ...dispatchInput, targetEventId: request.targetEventId },
+          options,
+        );
+      }
+      return await dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
+        input.withClient,
+        dispatchInput,
+        options,
+      );
+    },
+  };
+}
+
+/** The targeted scheduler result has all public projection counters. */
+export type GatewayRunLedgerProjectionResult =
+  PostgresSchedulerOutboxDrainResult;

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/index.ts

@@ -0,0 +1,15 @@
+export { createRunLedgerProjectionJob } from '../../play-runs/projection/job';
+export {
+  createGatewayRunLedgerProjectionAdapter,
+  type GatewayRunLedgerProjectionAdapterInput,
+  type GatewayRunLedgerProjectionResult,
+} from '../../play-runs/projection/gateway-adapter';
+export type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionJob,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+export type {
+  RunLedgerOutboxDelivery,
+  RunLedgerProjectionPorts,
+} from '../../play-runs/projection/ports';

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/job.ts

@@ -0,0 +1,65 @@
+import type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionJob,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+import type { RunLedgerProjectionPorts } from '../../play-runs/projection/ports';
+
+/**
+ * Creates the Run Ledger Projection job.
+ *
+ * A target is a delivery observation, not a second queue. In particular, a
+ * target that remains pending must return the adapter's durable retry result
+ * rather than causing the caller to append or retry an event itself.
+ */
+export function createRunLedgerProjectionJob(
+  ports: RunLedgerProjectionPorts,
+): RunLedgerProjectionJob {
+  return {
+    async deliver(
+      input: RunLedgerProjectionInput = {},
+    ): Promise<RunLedgerProjectionStats> {
+      if (input.targetEventId !== undefined && !input.targetEventId.trim()) {
+        throw new Error(
+          'Run Ledger Projection target event id must be non-empty when supplied.',
+        );
+      }
+
+      const result = await ports.outbox.deliver(input);
+      validateStats(result, input);
+      return result;
+    },
+  };
+}
+
+function validateStats(
+  result: RunLedgerProjectionStats,
+  input: RunLedgerProjectionInput,
+): void {
+  for (const value of [
+    result.attempted,
+    result.projected,
+    result.failed,
+    result.retrying,
+  ]) {
+    if (!Number.isSafeInteger(value) || value < 0) {
+      throw new Error(
+        'Run Ledger Projection adapter returned invalid delivery counters.',
+      );
+    }
+  }
+  if (!Array.isArray(result.failedEventIds)) {
+    throw new Error(
+      'Run Ledger Projection adapter returned invalid failed event ids.',
+    );
+  }
+  if (
+    input.targetEventId !== undefined &&
+    result.targetEventId !== undefined &&
+    result.targetEventId !== input.targetEventId
+  ) {
+    throw new Error(
+      'Run Ledger Projection adapter returned delivery stats for a different target event.',
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/projection/ports.ts

@@ -0,0 +1,17 @@
+import type {
+  RunLedgerProjectionInput,
+  RunLedgerProjectionStats,
+} from '../../play-runs/projection/contracts';
+
+/**
+ * The only infrastructure capability this job needs. The adapter owns the
+ * Postgres claim, ordered append, delivery idempotency, and durable retry
+ * clock; this job gives callers one stable Interface over that capability.
+ */
+export type RunLedgerOutboxDelivery = {
+  deliver(input: RunLedgerProjectionInput): Promise<RunLedgerProjectionStats>;
+};
+
+export type RunLedgerProjectionPorts = {
+  outbox: RunLedgerOutboxDelivery;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/contracts.ts

@@ -0,0 +1,42 @@
+/**
+ * Sandbox Cleanup reconciles one already-claimed physical resource. It begins
+ * after a resource fact and cleanup obligation exist; it neither creates a
+ * sandbox nor decides a Play Run terminal state.
+ */
+
+export type SandboxCleanupResource = {
+  provider: 'daytona' | 'modal';
+  sandboxId: string;
+  routingDomain: string | null;
+  runId: string;
+  attempt: number;
+  cleanupAttempt: number;
+  failureCount: number;
+};
+
+export type SandboxCleanupDeleteOutcome =
+  | { kind: 'deleted'; routingDomain: string | null }
+  | { kind: 'already_absent'; routingDomain: string | null }
+  /** The provider outcome is ambiguous or failed; its resource remains live. */
+  | { kind: 'failed'; code: string; detail: string };
+
+export type SandboxCleanupInput = {
+  resource: SandboxCleanupResource;
+  timeoutMs: number;
+};
+
+export type SandboxCleanupOutcome =
+  | {
+      kind: 'completed';
+      providerOutcome: 'deleted' | 'already_absent';
+      /** The durable claim transition which settled this provider fact. */
+      completion: 'completed' | 'late_success_accepted' | 'already_deleted';
+    }
+  | { kind: 'completion_rejected'; completion: 'missing_obligation' }
+  | { kind: 'blocked'; reason: string }
+  | { kind: 'retry_scheduled'; retryAfterMs: number }
+  | { kind: 'retry_rejected' };
+
+export type SandboxCleanupJob = {
+  reconcile(input: SandboxCleanupInput): Promise<SandboxCleanupOutcome>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/index.ts

@@ -0,0 +1,17 @@
+export { createSandboxCleanupJob } from '../../play-runs/sandbox-cleanup/job';
+export {
+  createPostgresSandboxCleanupStore,
+  type SandboxCleanupPostgresAdapterInput,
+} from '../../play-runs/sandbox-cleanup/postgres-adapter';
+export type {
+  SandboxCleanupDeleteOutcome,
+  SandboxCleanupInput,
+  SandboxCleanupJob,
+  SandboxCleanupOutcome,
+  SandboxCleanupResource,
+} from '../../play-runs/sandbox-cleanup/contracts';
+export type {
+  SandboxCleanupPorts,
+  SandboxCleanupStore,
+  SandboxDeletion,
+} from '../../play-runs/sandbox-cleanup/ports';

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/job.ts

@@ -0,0 +1,142 @@
+import type {
+  SandboxCleanupInput,
+  SandboxCleanupJob,
+  SandboxCleanupOutcome,
+} from '../../play-runs/sandbox-cleanup/contracts';
+import type { SandboxCleanupPorts } from '../../play-runs/sandbox-cleanup/ports';
+
+/**
+ * Creates the Sandbox Cleanup job.
+ *
+ * Capacity ends only after a provider-confirmed deletion or documented
+ * already-absent result. Every other provider result retains the durable
+ * cleanup obligation and schedules a fenced retry.
+ */
+export function createSandboxCleanupJob(
+  ports: SandboxCleanupPorts,
+): SandboxCleanupJob {
+  return {
+    async reconcile(
+      input: SandboxCleanupInput,
+    ): Promise<SandboxCleanupOutcome> {
+      validate(input);
+      if (
+        input.resource.provider === 'modal' &&
+        !input.resource.routingDomain?.trim()
+      ) {
+        const reason =
+          'missing_routing_domain: Modal cleanup requires its persisted app id.';
+        const blocked = await ports.store.block({
+          resource: input.resource,
+          reason,
+        });
+        return blocked === 'blocked'
+          ? { kind: 'blocked', reason }
+          : { kind: 'completion_rejected', completion: 'missing_obligation' };
+      }
+
+      try {
+        const deletion = await ports.deletion.delete(input);
+        if (deletion.kind === 'failed') {
+          if (deletion.code === 'missing_routing_domain') {
+            const reason = `missing_routing_domain: ${deletion.detail}`.slice(
+              0,
+              2_000,
+            );
+            const blocked = await ports.store.block({
+              resource: input.resource,
+              reason,
+            });
+            return blocked === 'blocked'
+              ? { kind: 'blocked', reason }
+              : {
+                  kind: 'completion_rejected',
+                  completion: 'missing_obligation',
+                };
+          }
+          return await scheduleRetry(ports, input, deletion.detail);
+        }
+
+        // A control-plane retry must never keep a provider-confirmed-deleted
+        // resource consuming capacity. This precedes cleanup row completion.
+        await ports.store.releaseCapacity({ resource: input.resource });
+        const completion = await ports.store.complete({
+          resource: input.resource,
+          providerOutcome: deletion.kind,
+          observedRoutingDomain: deletion.routingDomain,
+        });
+        if (
+          completion === 'completed' ||
+          completion === 'late_success_accepted' ||
+          completion === 'already_deleted'
+        ) {
+          return {
+            kind: 'completed',
+            providerOutcome: deletion.kind,
+            completion,
+          };
+        }
+        // A provider confirmation from the wrong routing domain is not a
+        // completion rejection: it is a real external-side-effect ambiguity.
+        // Preserve the obligation and retry through its existing claim fence.
+        if (completion === 'wrong_routing_domain') {
+          return await scheduleRetry(
+            ports,
+            input,
+            `${input.resource.provider} sandbox ${input.resource.sandboxId} cleanup confirmation used the wrong routing domain.`,
+          );
+        }
+        return { kind: 'completion_rejected', completion };
+      } catch (error) {
+        return await scheduleRetry(
+          ports,
+          input,
+          error instanceof Error ? error.message : String(error),
+        );
+      }
+    },
+  };
+}
+
+async function scheduleRetry(
+  ports: SandboxCleanupPorts,
+  input: SandboxCleanupInput,
+  error: string,
+): Promise<SandboxCleanupOutcome> {
+  const retryAfterMs = retryDelay(input.resource.failureCount + 1);
+  const scheduled = await ports.store.retry({
+    resource: input.resource,
+    error,
+    retryAfterMs,
+  });
+  return scheduled === 'retry_scheduled'
+    ? { kind: 'retry_scheduled', retryAfterMs }
+    : { kind: 'retry_rejected' };
+}
+
+function retryDelay(nextFailureCount: number): number {
+  return Math.min(
+    5 * 60_000,
+    1_000 * 2 ** Math.min(8, Math.max(0, nextFailureCount - 1)),
+  );
+}
+
+function validate(input: SandboxCleanupInput): void {
+  const { resource } = input;
+  if (!resource.sandboxId.trim() || !resource.runId.trim()) {
+    throw new Error('Sandbox Cleanup requires non-empty sandbox and run ids.');
+  }
+  if (!Number.isSafeInteger(resource.attempt) || resource.attempt < 1) {
+    throw new Error('Sandbox Cleanup requires a positive attempt number.');
+  }
+  if (
+    !Number.isSafeInteger(resource.cleanupAttempt) ||
+    resource.cleanupAttempt < 1 ||
+    !Number.isSafeInteger(resource.failureCount) ||
+    resource.failureCount < 0 ||
+    !Number.isFinite(input.timeoutMs) ||
+    input.timeoutMs <= 0
+  ) {
+    throw new Error('Sandbox Cleanup received invalid cleanup state.');
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/ports.ts

@@ -0,0 +1,42 @@
+import type {
+  SandboxCleanupDeleteOutcome,
+  SandboxCleanupResource,
+} from '../../play-runs/sandbox-cleanup/contracts';
+
+/** Provider delete proof. A timeout must be reported as `failed`, never cast to deletion. */
+export type SandboxDeletion = {
+  delete(input: {
+    resource: SandboxCleanupResource;
+    timeoutMs: number;
+  }): Promise<SandboxCleanupDeleteOutcome>;
+};
+
+/** Durable cleanup state transitions, all fenced by the claim represented in `resource`. */
+export type SandboxCleanupStore = {
+  releaseCapacity(input: { resource: SandboxCleanupResource }): Promise<void>;
+  complete(input: {
+    resource: SandboxCleanupResource;
+    providerOutcome: 'deleted' | 'already_absent';
+    observedRoutingDomain: string | null;
+  }): Promise<
+    | 'completed'
+    | 'late_success_accepted'
+    | 'already_deleted'
+    | 'missing_obligation'
+    | 'wrong_routing_domain'
+  >;
+  retry(input: {
+    resource: SandboxCleanupResource;
+    error: string;
+    retryAfterMs: number;
+  }): Promise<'retry_scheduled' | 'rejected'>;
+  block(input: {
+    resource: SandboxCleanupResource;
+    reason: string;
+  }): Promise<'blocked' | 'rejected'>;
+};
+
+export type SandboxCleanupPorts = {
+  deletion: SandboxDeletion;
+  store: SandboxCleanupStore;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/sandbox-cleanup/postgres-adapter.ts

@@ -0,0 +1,102 @@
+import {
+  blockPostgresSchedulerSandboxCleanupJob,
+  completePostgresSchedulerSandboxCleanupJob,
+  retryPostgresSchedulerSandboxCleanupJob,
+  type PostgresSchedulerOptions,
+  type PostgresSchedulerQueryClient,
+} from '../../scheduler-backends/postgres';
+import { releaseRuntimeSandboxCapacityLeaseForSandbox } from '../../scheduler-backends/runtime-sandbox-capacity';
+
+import type { SandboxCleanupPorts } from '../../play-runs/sandbox-cleanup/ports';
+
+export type SandboxCleanupPostgresAdapterInput = {
+  runWithClient: <T>(
+    fn: (client: PostgresSchedulerQueryClient) => Promise<T>,
+  ) => Promise<T>;
+  leaseOwner: string;
+  schedulerOptions?: Pick<PostgresSchedulerOptions, 'schema'>;
+};
+
+/**
+ * Runtime-Postgres Adapter for SandboxCleanup. The Job owns the state
+ * machine; this Adapter only translates its fenced transitions into the
+ * existing durable tables and capacity ledger.
+ */
+export function createPostgresSandboxCleanupStore(
+  input: SandboxCleanupPostgresAdapterInput,
+): SandboxCleanupPorts['store'] {
+  const options = input.schedulerOptions ?? {};
+  return {
+    async releaseCapacity({ resource }) {
+      await input.runWithClient(async (client) => {
+        await releaseRuntimeSandboxCapacityLeaseForSandbox(
+          client,
+          {
+            runId: resource.runId,
+            provider: resource.provider,
+            sandboxId: resource.sandboxId,
+            routingDomain: resource.routingDomain,
+          },
+          options,
+        );
+      });
+    },
+    async complete({ resource, providerOutcome, observedRoutingDomain }) {
+      return await input.runWithClient(
+        async (client) =>
+          await completePostgresSchedulerSandboxCleanupJob(
+            client,
+            {
+              provider: resource.provider,
+              sandboxId: resource.sandboxId,
+              routingDomain: resource.routingDomain,
+              runId: resource.runId,
+              attempt: resource.attempt,
+              cleanupAttempt: resource.cleanupAttempt,
+              leaseOwner: input.leaseOwner,
+              providerOutcome,
+              observedRoutingDomain,
+            },
+            options,
+          ),
+      );
+    },
+    async retry({ resource, error, retryAfterMs }) {
+      const outcome = await input.runWithClient(
+        async (client) =>
+          await retryPostgresSchedulerSandboxCleanupJob(
+            client,
+            {
+              provider: resource.provider,
+              sandboxId: resource.sandboxId,
+              routingDomain: resource.routingDomain,
+              leaseOwner: input.leaseOwner,
+              claimGeneration: resource.cleanupAttempt,
+              error,
+              retryAfterMs,
+            },
+            options,
+          ),
+      );
+      return outcome === 'retry_scheduled' ? outcome : 'rejected';
+    },
+    async block({ resource, reason }) {
+      const outcome = await input.runWithClient(
+        async (client) =>
+          await blockPostgresSchedulerSandboxCleanupJob(
+            client,
+            {
+              provider: resource.provider,
+              sandboxId: resource.sandboxId,
+              routingDomain: resource.routingDomain,
+              leaseOwner: input.leaseOwner,
+              claimGeneration: resource.cleanupAttempt,
+              reason,
+            },
+            options,
+          ),
+      );
+      return outcome === 'blocked' ? outcome : 'rejected';
+    },
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts

@@ -0,0 +1,60 @@
+import {
+  directCompletedTerminalContract,
+  type DirectCompletedTerminalContract,
+} from '../../scheduler-backends/terminal-contract';
+
+/**
+ * The authenticated authority for one runner attempt to write the Play Run's
+ * terminal state. It is intentionally a terminal concept: it carries neither
+ * scheduler-row handles nor transport headers.
+ */
+export type PlayRunTerminalAuthority = {
+  terminalContract: DirectCompletedTerminalContract;
+  leaseToken: string;
+};
+
+/** Parse ingress-only values into the authority the terminal job accepts. */
+export function resolvePlayRunTerminalAuthority(input: {
+  terminalContract?: unknown;
+  leaseToken?: unknown;
+}): PlayRunTerminalAuthority | null {
+  const terminalContract = directCompletedTerminalContract(
+    input.terminalContract,
+  );
+  if (
+    terminalContract === null ||
+    typeof input.leaseToken !== 'string' ||
+    !input.leaseToken.trim()
+  ) {
+    return null;
+  }
+  return {
+    terminalContract,
+    leaseToken: input.leaseToken.trim(),
+  };
+}
+
+/** The only terminal results visible to an ingress caller. */
+export type PlayRunTerminalOutcome =
+  | { kind: 'direct_completed' }
+  | { kind: 'stale_attempt_cleaned' }
+  | { kind: 'already_terminal' }
+  | { kind: 'compatibility_wake_queued' }
+  | { kind: 'not_found_or_fence_rejected' };
+
+/**
+ * One detached runner's attempt to settle a Play Run. The queue name belongs
+ * here because a compatibility wake is a terminal delivery obligation, not a
+ * caller-side scheduler detail.
+ */
+export type PlayRunTerminalInput = {
+  queueName: string;
+  runId: string;
+  attempt: number;
+  authority?: PlayRunTerminalAuthority | null;
+  result: unknown;
+};
+
+export type PlayRunTerminalJob = {
+  accept(input: PlayRunTerminalInput): Promise<PlayRunTerminalOutcome>;
+};

package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/index.ts

@@ -0,0 +1,16 @@
+export { resolvePlayRunTerminalAuthority } from '../../play-runs/terminal/contracts';
+export { createPlayRunTerminalJob } from '../../play-runs/terminal/job';
+export { createPostgresPlayRunTerminalPorts } from '../../play-runs/terminal/postgres-adapter';
+export type {
+  PlayRunTerminalAuthority,
+  PlayRunTerminalInput,
+  PlayRunTerminalJob,
+  PlayRunTerminalOutcome,
+} from '../../play-runs/terminal/contracts';
+export type {
+  CompatibilityTerminalRecorder,
+  DirectTerminalCommitOutcome,
+  DirectTerminalCommitter,
+  PlayRunTerminalPorts,
+  RunnerTerminalWakePublisher,
+} from '../../play-runs/terminal/ports';