deepline 0.1.290 → 0.1.292

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts

@@ -54,7 +54,7 @@ const DAYTONA_UPLOAD_ATTEMPT_DEADLINE_MS = 90_000;
 const RUNTIME_POSTGRES_CONNECT_RETRY_PATTERN =
   /\bRuntime Postgres\b.*\b(connection timed out|connect timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|Connection terminated|Connection ended unexpectedly)\b/i;
 const RUNTIME_RECEIPT_GATEWAY_TRANSPORT_RETRY_PATTERN =
-  /AppRuntimeApiTransportError: App runtime API transport exhausted action=(?:get|claim|mark|complete|fail|heartbeat|release|skip)_runtime_step_receipts?/i;
+  /AppRuntimeApiTransportError: (?:App runtime API|Runtime receipt gateway) transport exhausted action=(?:get|claim|mark|complete|fail|heartbeat|release|skip)_runtime_step_receipts?/i;
 const RUNTIME_API_TRANSPORT_RETRY_PATTERN =
   /\bRuntime API request to .+ failed before receiving a response:\s*(?:fetch failed|connection (?:terminated|timed out|closed|reset)|ECONNRESET|ETIMEDOUT|ECONNREFUSED)\b/i;
 const DAYTONA_INFRASTRUCTURE_RETRY_PATTERN =

package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts

@@ -78,6 +78,7 @@ import {
   sameOwnerTerminalAttemptEpochSql,
 } from './sheet-attempt-sql';
 import type { MapRowOutcome } from './durability-store';
+import { RUNTIME_CAPACITY_POLICY } from './runtime-capacity-policy';
 import {
   normalizeRuntimeMapInputIndex,
   prepareRuntimeSheetRowsForJsonTransport,
@@ -349,7 +350,8 @@ const RUNTIME_POSTGRES_RECEIPT_ADMISSION_MAX_QUEUED = 2_000;
 // memory independently from lightweight receipt requests; overflow is a
 // retryable 503 and waits outside this long-lived gateway process.
 const RUNTIME_POSTGRES_SHEET_ADMISSION_MAX_QUEUED = 4;
-const RUNTIME_POSTGRES_RECEIPT_ADMISSION_TIMEOUT_MS = 10_000;
+const RUNTIME_POSTGRES_RECEIPT_ADMISSION_TIMEOUT_MS =
+  RUNTIME_CAPACITY_POLICY.receiptGateway.admissionTimeoutMs;
 const RECEIPT_STATUS_QUEUED = RECEIPT_STATUS_CODE.queued;
 const RECEIPT_STATUS_PENDING = RECEIPT_STATUS_CODE.pending;
 const RECEIPT_STATUS_RUNNING = RECEIPT_STATUS_CODE.running;
@@ -4173,6 +4175,9 @@ export async function claimRuntimeWorkReceipt(
         input.leaseTtlMs,
         PLAY_RUNTIME_WORK_RECEIPT_LEASE_TTL_MS,
       );
+      // A successful lease-bearing claim is the execution transition. Keep the
+      // stored queued status for mixed-version readers; the runner does not add
+      // a second queued -> running write on the provider hot path.
       const claimStatus =
         input.leaseAware === true
           ? RECEIPT_STATUS_QUEUED

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -0,0 +1,124 @@
+/**
+ * One capacity policy for the Absurd runtime.
+ *
+ * Postgres owns backlog. The active lane adds workers from durable claimable
+ * demand and oldest-claimable age; historical lanes stay cold unless they own
+ * runnable work. Keep these values together so the worker, scaler, gateway,
+ * and tests cannot independently invent concurrency or timeout budgets.
+ */
+export const RUNTIME_CAPACITY_POLICY = {
+  absurd: {
+    /** Warm floor for availability and deploy rollovers. */
+    activeLaneMachines: 2,
+    /** 32 Machines x 8 claim slots permits 256 concurrent launch/resume legs. */
+    maxActiveLaneMachines: 32,
+    workerSlotsPerMachine: 8,
+    perOrgConcurrency: 4,
+    /** Queue age is an escape hatch for small-but-stuck backlogs. */
+    scaleUpQueueAgeMs: 10_000,
+    /** Bound one Fly reconciliation without turning backlog into a stampede. */
+    maxScaleUpMachinesPerReconcile: 4,
+    /** Do not tear down burst capacity on the first empty observation. */
+    scaleDownQuietMs: 120_000,
+  },
+  receiptGateway: {
+    admissionTimeoutMs: 10_000,
+    requestTimeoutMs: 30_000,
+  },
+} as const;
+
+export const ABSURD_GLOBAL_RUN_CONCURRENCY =
+  RUNTIME_CAPACITY_POLICY.absurd.maxActiveLaneMachines *
+  RUNTIME_CAPACITY_POLICY.absurd.workerSlotsPerMachine;
+
+/**
+ * Active releases use pg-boss-style burst semantics: claimable count sets the
+ * target, an aged claimable queue forces continued growth, and growth per
+ * observation is bounded. A short/empty observation ends burst growth; the
+ * caller supplies idleForMs to hold excess capacity through a quiet window.
+ *
+ * Historical releases stay cold until they have immediately runnable or
+ * currently claimed work.
+ */
+export function desiredAbsurdLaneMachines(input: {
+  active: boolean;
+  claimableRuns: number;
+  runningRuns?: number;
+  currentMachines?: number;
+  oldestClaimableAgeMs?: number;
+  idleForMs?: number;
+}): number {
+  const claimableRuns = nonNegativeInteger(input.claimableRuns);
+  const runningRuns = nonNegativeInteger(input.runningRuns ?? 0);
+  if (!input.active) return claimableRuns > 0 || runningRuns > 0 ? 1 : 0;
+
+  const policy = RUNTIME_CAPACITY_POLICY.absurd;
+  const currentMachines = Math.max(
+    policy.activeLaneMachines,
+    nonNegativeInteger(input.currentMachines ?? policy.activeLaneMachines),
+  );
+  const demandedMachines = Math.ceil(
+    (claimableRuns + runningRuns) / policy.workerSlotsPerMachine,
+  );
+  let desired = Math.max(policy.activeLaneMachines, demandedMachines);
+
+  if (
+    claimableRuns > 0 &&
+    nonNegativeNumber(input.oldestClaimableAgeMs ?? 0) >=
+      policy.scaleUpQueueAgeMs
+  ) {
+    desired = Math.max(desired, currentMachines + 1);
+  }
+
+  desired = Math.min(
+    desired,
+    currentMachines + policy.maxScaleUpMachinesPerReconcile,
+    policy.maxActiveLaneMachines,
+  );
+
+  if (
+    claimableRuns === 0 &&
+    runningRuns === 0 &&
+    nonNegativeNumber(input.idleForMs ?? 0) < policy.scaleDownQuietMs
+  ) {
+    desired = Math.max(desired, currentMachines);
+  }
+  return desired;
+}
+
+function nonNegativeInteger(value: number): number {
+  return Math.max(0, Math.floor(nonNegativeNumber(value)));
+}
+
+function nonNegativeNumber(value: number): number {
+  return Number.isFinite(value) ? Math.max(0, value) : 0;
+}
+
+export function assertRuntimeCapacityPolicy(): void {
+  if (
+    RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs <=
+    RUNTIME_CAPACITY_POLICY.receiptGateway.admissionTimeoutMs
+  ) {
+    throw new Error(
+      'Receipt gateway request timeout must exceed its admission timeout.',
+    );
+  }
+  if (
+    RUNTIME_CAPACITY_POLICY.absurd.perOrgConcurrency >
+    ABSURD_GLOBAL_RUN_CONCURRENCY
+  ) {
+    throw new Error(
+      'Per-org runtime concurrency cannot exceed global runtime concurrency.',
+    );
+  }
+  if (
+    RUNTIME_CAPACITY_POLICY.absurd.activeLaneMachines >
+    RUNTIME_CAPACITY_POLICY.absurd.maxActiveLaneMachines
+  ) {
+    throw new Error(
+      'Active-lane minimum Machines cannot exceed the active-lane maximum.',
+    );
+  }
+}
+
+assertRuntimeCapacityPolicy();

package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts

@@ -0,0 +1,410 @@
+import type {
+  AcquireRuntimeReceiptExecutionLockInput,
+  ClaimRuntimeStepReceiptInput,
+  CompleteRuntimeStepReceiptInput,
+  ContextOptions,
+  FailRuntimeStepReceiptInput,
+  ReleaseRuntimeReceiptExecutionLockInput,
+  ReleaseRuntimeStepReceiptInput,
+  RuntimeStepReceipt,
+  SkipRuntimeStepReceiptInput,
+} from './ctx-types';
+import {
+  acquireRuntimeReceiptExecutionLockViaAppRuntime,
+  AppRuntimeApiResponseError,
+  AppRuntimeApiTransportError,
+  claimRuntimeStepReceiptsViaAppRuntime,
+  completeRuntimeStepReceiptsViaAppRuntime,
+  failRuntimeStepReceiptsViaAppRuntime,
+  getRuntimeStepReceiptsViaAppRuntime,
+  heartbeatRuntimeStepReceiptsViaAppRuntime,
+  isAppRuntimeApiCapacityError,
+  releaseRuntimeReceiptExecutionLockViaAppRuntime,
+  releaseRuntimeStepReceiptViaAppRuntime,
+  skipRuntimeStepReceiptViaAppRuntime,
+  type WorkerRuntimeApiContext,
+} from './app-runtime-api';
+import { RuntimeReceiptWriter } from './runtime-receipt-writer';
+
+type ReceiptOutput =
+  | RuntimeStepReceipt
+  | null
+  | boolean
+  | { ownerExecutionId: string; expiresAt: string };
+
+type ReceiptCommand =
+  | {
+      kind: 'get';
+      input: { key: string; runId: string };
+    }
+  | {
+      kind: 'claim';
+      input: ClaimRuntimeStepReceiptInput;
+    }
+  | {
+      kind: 'settle';
+      outcome: 'complete';
+      input: CompleteRuntimeStepReceiptInput;
+    }
+  | {
+      kind: 'settle';
+      outcome: 'fail';
+      input: FailRuntimeStepReceiptInput;
+    }
+  | {
+      kind: 'heartbeat';
+      input: {
+        key: string;
+        runId: string;
+        runAttempt?: number | null;
+        leaseId: string;
+      };
+    }
+  | {
+      kind: 'settle';
+      outcome: 'release';
+      input: ReleaseRuntimeStepReceiptInput;
+      serial: number;
+    }
+  | {
+      kind: 'settle';
+      outcome: 'skip';
+      input: SkipRuntimeStepReceiptInput;
+      serial: number;
+    }
+  | {
+      kind: 'acquire_execution_lock';
+      input: AcquireRuntimeReceiptExecutionLockInput;
+      serial: number;
+    }
+  | {
+      kind: 'release_execution_lock';
+      input: ReleaseRuntimeReceiptExecutionLockInput;
+      serial: number;
+    };
+
+type ReceiptHandlerName =
+  | 'acquireRuntimeReceiptExecutionLock'
+  | 'releaseRuntimeReceiptExecutionLock'
+  | 'getRuntimeStepReceipt'
+  | 'getRuntimeStepReceipts'
+  | 'claimRuntimeStepReceipt'
+  | 'claimRuntimeStepReceipts'
+  | 'completeRuntimeStepReceipt'
+  | 'completeRuntimeStepReceipts'
+  | 'releaseRuntimeStepReceipt'
+  | 'failRuntimeStepReceipt'
+  | 'failRuntimeStepReceipts'
+  | 'heartbeatRuntimeStepReceipts'
+  | 'skipRuntimeStepReceipt';
+
+export type RuntimeReceiptStoreHandlers = Required<
+  Pick<ContextOptions, ReceiptHandlerName>
+>;
+
+/**
+ * The runner's single Work Receipt Adapter.
+ *
+ * Receipt-domain batching and gateway transport stay behind this seam. Every
+ * command shares one writer, so heterogeneous operations cannot produce
+ * concurrent gateway requests. The low-level runtime client is explicitly
+ * one-attempt; this Adapter is the sole retry owner.
+ */
+export class RuntimeReceiptStoreAdapter {
+  readonly handlers: RuntimeReceiptStoreHandlers;
+  readonly #writer: RuntimeReceiptWriter<ReceiptCommand, ReceiptOutput>;
+  #serial = 0;
+
+  constructor(options: {
+    context: WorkerRuntimeApiContext;
+    playName: string;
+    runId: string;
+    maxBatchSize?: number;
+    maxFlushMs?: number;
+    maxBufferedBytes?: number;
+  }) {
+    const context: WorkerRuntimeApiContext = {
+      ...options.context,
+      retryPolicy: 'none',
+    };
+    this.#writer = new RuntimeReceiptWriter({
+      batchKey: receiptCommandBatchKey,
+      maxBatchSize: options.maxBatchSize,
+      maxFlushMs: options.maxFlushMs,
+      maxBufferedBytes: options.maxBufferedBytes,
+      classifyRetryableError: classifyReceiptRetry,
+      send: async (commands, { signal }) =>
+        await sendReceiptCommands({
+          context: { ...context, signal },
+          playName: options.playName,
+          commands,
+        }),
+    });
+
+    const write = <Output extends ReceiptOutput>(
+      command: ReceiptCommand,
+    ): Promise<Output> =>
+      this.#writer.write(command, {
+        signal: options.context.signal,
+      }) as Promise<Output>;
+    const writeMany = <Output extends ReceiptOutput>(
+      commands: readonly ReceiptCommand[],
+    ): Promise<Output[]> =>
+      Promise.all(commands.map((command) => write<Output>(command)));
+
+    this.handlers = {
+      acquireRuntimeReceiptExecutionLock: (input) =>
+        write({
+          kind: 'acquire_execution_lock',
+          input,
+          serial: this.#nextSerial(),
+        }),
+      releaseRuntimeReceiptExecutionLock: (input) =>
+        write({
+          kind: 'release_execution_lock',
+          input,
+          serial: this.#nextSerial(),
+        }),
+      getRuntimeStepReceipt: ({ key }) =>
+        write({
+          kind: 'get',
+          input: { key, runId: options.runId },
+        }),
+      getRuntimeStepReceipts: ({ keys }) =>
+        writeMany(
+          keys.map((key) => ({
+            kind: 'get' as const,
+            input: { key, runId: options.runId },
+          })),
+        ),
+      claimRuntimeStepReceipt: (input) => write({ kind: 'claim', input }),
+      claimRuntimeStepReceipts: (input) =>
+        writeMany(
+          input.keys.map((key, index) => ({
+            kind: 'claim' as const,
+            input: {
+              key,
+              runId: input.runId,
+              runAttempt: input.runAttempt,
+              leaseAware: input.leaseAware,
+              reclaimRunning: input.reclaimRunning,
+              forceRefresh: input.forceRefresh,
+              forceFailedRefresh: input.forceFailedRefresh,
+              ...(input.leaseIds ? { leaseId: input.leaseIds[index] } : {}),
+            },
+          })),
+        ),
+      completeRuntimeStepReceipt: (input) =>
+        write({ kind: 'settle', outcome: 'complete', input }),
+      completeRuntimeStepReceipts: ({ receipts }) =>
+        writeMany(
+          receipts.map((input) => ({
+            kind: 'settle' as const,
+            outcome: 'complete' as const,
+            input,
+          })),
+        ),
+      releaseRuntimeStepReceipt: (input) =>
+        write({
+          kind: 'settle',
+          outcome: 'release',
+          input,
+          serial: this.#nextSerial(),
+        }),
+      failRuntimeStepReceipt: (input) =>
+        write({ kind: 'settle', outcome: 'fail', input }),
+      failRuntimeStepReceipts: ({ receipts }) =>
+        writeMany(
+          receipts.map((input) => ({
+            kind: 'settle' as const,
+            outcome: 'fail' as const,
+            input,
+          })),
+        ),
+      heartbeatRuntimeStepReceipts: (input) =>
+        writeMany(
+          input.keys.map((key) => ({
+            kind: 'heartbeat' as const,
+            input: {
+              key,
+              runId: input.runId,
+              runAttempt: input.runAttempt,
+              leaseId: input.leaseId,
+            },
+          })),
+        ),
+      skipRuntimeStepReceipt: (input) =>
+        write({
+          kind: 'settle',
+          outcome: 'skip',
+          input,
+          serial: this.#nextSerial(),
+        }),
+    };
+  }
+
+  flush(): Promise<void> {
+    return this.#writer.flush();
+  }
+
+  close(reason?: unknown): Promise<void> {
+    return this.#writer.close(reason);
+  }
+
+  #nextSerial(): number {
+    return ++this.#serial;
+  }
+}
+
+function receiptCommandBatchKey(command: ReceiptCommand): string {
+  switch (command.kind) {
+    case 'get':
+      return JSON.stringify([command.kind, command.input.runId]);
+    case 'claim':
+      return JSON.stringify([
+        command.kind,
+        command.input.runId,
+        command.input.runAttempt ?? null,
+        command.input.leaseAware === true,
+        command.input.reclaimRunning === true,
+        command.input.forceRefresh === true,
+        command.input.forceFailedRefresh === true,
+        command.input.leaseId !== undefined,
+      ]);
+    case 'settle':
+      return command.outcome === 'complete' || command.outcome === 'fail'
+        ? JSON.stringify([command.kind, command.outcome, command.input.runId])
+        : `${command.kind}:${command.outcome}:${command.serial}`;
+    case 'heartbeat':
+      return JSON.stringify([
+        command.kind,
+        command.input.runId,
+        command.input.runAttempt ?? null,
+        command.input.leaseId,
+      ]);
+    case 'acquire_execution_lock':
+    case 'release_execution_lock':
+      return `${command.kind}:${command.serial}`;
+  }
+}
+
+function classifyReceiptRetry(error: unknown): { retryAfterMs: number } | null {
+  if (isAppRuntimeApiCapacityError(error)) {
+    return { retryAfterMs: error.retryAfterMs };
+  }
+  if (error instanceof AppRuntimeApiTransportError) {
+    return { retryAfterMs: 250 };
+  }
+  if (error instanceof AppRuntimeApiResponseError && error.retryable) {
+    return { retryAfterMs: 250 };
+  }
+  return null;
+}
+
+async function sendReceiptCommands(input: {
+  context: WorkerRuntimeApiContext;
+  playName: string;
+  commands: readonly ReceiptCommand[];
+}): Promise<readonly ReceiptOutput[]> {
+  const first = input.commands[0];
+  if (!first) return [];
+  switch (first.kind) {
+    case 'get': {
+      const commands = input.commands as readonly Extract<
+        ReceiptCommand,
+        { kind: 'get' }
+      >[];
+      return await getRuntimeStepReceiptsViaAppRuntime(input.context, {
+        playName: input.playName,
+        runId: first.input.runId,
+        keys: commands.map((command) => command.input.key),
+      });
+    }
+    case 'claim': {
+      const commands = input.commands as readonly Extract<
+        ReceiptCommand,
+        { kind: 'claim' }
+      >[];
+      return await claimRuntimeStepReceiptsViaAppRuntime(input.context, {
+        playName: input.playName,
+        runId: first.input.runId,
+        runAttempt: first.input.runAttempt,
+        keys: commands.map((command) => command.input.key),
+        leaseIds:
+          first.input.leaseId === undefined
+            ? undefined
+            : commands.map((command) => command.input.leaseId!),
+        leaseAware: first.input.leaseAware,
+        reclaimRunning: first.input.reclaimRunning,
+        forceRefresh: first.input.forceRefresh,
+        forceFailedRefresh: first.input.forceFailedRefresh,
+      });
+    }
+    case 'settle':
+      switch (first.outcome) {
+        case 'complete': {
+          const commands = input.commands as readonly Extract<
+            ReceiptCommand,
+            { kind: 'settle'; outcome: 'complete' }
+          >[];
+          return await completeRuntimeStepReceiptsViaAppRuntime(input.context, {
+            playName: input.playName,
+            runId: first.input.runId,
+            receipts: commands.map((command) => command.input),
+          });
+        }
+        case 'fail': {
+          const commands = input.commands as readonly Extract<
+            ReceiptCommand,
+            { kind: 'settle'; outcome: 'fail' }
+          >[];
+          return await failRuntimeStepReceiptsViaAppRuntime(input.context, {
+            playName: input.playName,
+            runId: first.input.runId,
+            receipts: commands.map((command) => command.input),
+          });
+        }
+        case 'release':
+          return [
+            await releaseRuntimeStepReceiptViaAppRuntime(input.context, {
+              playName: input.playName,
+              ...first.input,
+            }),
+          ];
+        case 'skip':
+          return [
+            await skipRuntimeStepReceiptViaAppRuntime(input.context, {
+              playName: input.playName,
+              ...first.input,
+            }),
+          ];
+      }
+    case 'heartbeat': {
+      const commands = input.commands as readonly Extract<
+        ReceiptCommand,
+        { kind: 'heartbeat' }
+      >[];
+      return await heartbeatRuntimeStepReceiptsViaAppRuntime(input.context, {
+        playName: input.playName,
+        runId: first.input.runId,
+        runAttempt: first.input.runAttempt,
+        leaseId: first.input.leaseId,
+        keys: commands.map((command) => command.input.key),
+      });
+    }
+    case 'acquire_execution_lock':
+      return [
+        await acquireRuntimeReceiptExecutionLockViaAppRuntime(input.context, {
+          playName: input.playName,
+          ...first.input,
+        }),
+      ];
+    case 'release_execution_lock':
+      return [
+        await releaseRuntimeReceiptExecutionLockViaAppRuntime(input.context, {
+          playName: input.playName,
+          ...first.input,
+        }),
+      ];
+  }
+}