deepline 0.3.128 → 0.3.129

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -38,7 +38,10 @@ import type {
 } from './governor/governor';
 import type { BudgetStateBackend } from './governor/budget-state-backend';
 import type { DocflowObservationUpsert } from './docflow-observation';
-import type { RuntimeMapMemoryLimits } from './map-memory-limits';
+import type {
+  RuntimeMapHeapSnapshot,
+  RuntimeMapMemoryLimits,
+} from './map-memory-limits';
 import type { AnyBatchOperationStrategy } from './batching-types';
 import type { ToolResultMetadataInput } from './tool-result';
 import type { PreviousCell } from '../plays/cell-staleness';
@@ -55,6 +58,10 @@ import type {
 import type { FixtureBehavior } from './fixture-behavior';
 import type { ToolResponseContract } from '../plays/tool-response-contract';
 import type { PlayArtifactRuntimeContract } from './play-artifact-runtime-contract';
+import type {
+  RuntimeReceiptValueInlineFields,
+  RuntimeReceiptValueReference,
+} from './runtime-receipt-value';
 
 export interface RowState {
   results: Map<string, unknown>;
@@ -70,6 +77,8 @@ export interface ToolCallRequest {
   cacheKey: string;
   /** Stable external-operation identity, independent of receipt serialization. */
   providerIdempotencyKeyBase?: string | null;
+  /** Action compatibility version captured when this request was enqueued. */
+  providerActionVersion?: string | null;
   cacheable?: boolean;
   receiptKey?: string | null;
   /** Pre-cutover result-cache keys to read before executing a canonical miss. */
@@ -91,6 +100,35 @@ export interface ToolCallRequest {
   tableNamespace?: string;
   rowKey?: string | null;
   description?: string;
+  /**
+   * Dispatcher-clock reading when this request entered the tool call queue.
+   * Measurement only: the dispatcher subtracts it at group launch to record
+   * `dispatchQueueWait`. Never read by scheduling decisions.
+   */
+  dispatchEnqueuedAtMs?: number;
+  /**
+   * Runner-clock reading when the provider response body carrying this
+   * request's result was fully read and parsed. Stamped by the dispatcher at
+   * the parse site and consumed by the row's delivery site, which subtracts it
+   * to record `resultDelivery`. Absent on a request served from a cache or
+   * delivered as a follower: neither read a body of its own. Measurement only;
+   * never read by scheduling decisions.
+   */
+  providerBodyCompletedAtMs?: number | null;
+  /**
+   * Runner-clock reading when this request's wrapped tool result existed —
+   * the end of `deliveryWrap` and the start of `deliveryPublishWait`. Stamped
+   * by the module dispatcher only for a request that read a provider body of
+   * its own, so every delivery sub-phase is recorded for exactly the calls
+   * `resultDelivery` is. Measurement only.
+   */
+  deliveryWrappedAtMs?: number | null;
+  /**
+   * Runner-clock reading when the cohort publication carrying this request's
+   * receipt saw its `store.complete` settle — the start of `deliveryResolve`,
+   * closed at the row's delivery site. Measurement only.
+   */
+  deliveryCompletionSettledAtMs?: number | null;
 }
 
 export interface ToolBatchResult {
@@ -100,8 +138,23 @@ export interface ToolBatchResult {
 
 export interface RuntimeStepReceipt {
   key: string;
+  /** Completion acknowledgement only; absent on older gateways. */
+  completionDisposition?: 'persisted' | 'existing';
   status: 'queued' | 'pending' | 'running' | 'completed' | 'failed' | 'skipped';
   output?: unknown;
+  /**
+   * Canonical serialized output size when a metadata-only receipt read omitted
+   * the body. It is an advisory admission price, never a cache hit by itself:
+   * callers still redeem and validate the terminal receipt before reuse.
+   */
+  outputByteLength?: number;
+  /** Bounded selected fields available without redeeming an R2 output body. */
+  inlineFields?: RuntimeReceiptValueInlineFields;
+  /**
+   * Gateway-internal representation. The gateway verifies and hydrates it
+   * before a receipt crosses back to a runner.
+   */
+  outputRef?: RuntimeReceiptValueReference | null;
   error?: string;
   errorPayload?: ToolExecutionFailureV1 | null;
   failureKind?: WorkReceiptFailureKind | null;
@@ -128,10 +181,14 @@ export type ReleaseRuntimeReceiptExecutionLockInput = {
 
 export interface GetRuntimeStepReceiptInput {
   key: string;
+  /** Read the bounded receipt manifest, not its output body. */
+  hydrateOutput?: boolean;
 }
 
 export interface ClaimRuntimeStepReceiptInput {
   key: string;
+  /** Claim state without materializing a terminal output body. */
+  hydrateOutput?: boolean;
   leaseId?: string;
   runId: string;
   runAttempt?: number | null;
@@ -154,6 +211,7 @@ export interface CompleteRuntimeStepReceiptInput {
   runAttempt?: number | null;
   leaseId?: string | null;
   output: unknown | null;
+  inlineFields?: RuntimeReceiptValueInlineFields;
 }
 
 export interface ReleaseRuntimeStepReceiptInput {
@@ -183,10 +241,14 @@ export interface SkipRuntimeStepReceiptInput {
 
 export interface GetRuntimeStepReceiptsInput {
   keys: string[];
+  /** Read bounded manifests first; hydrate terminal outputs in a later turn. */
+  hydrateOutput?: boolean;
 }
 
 export interface ClaimRuntimeStepReceiptsInput {
   keys: string[];
+  /** Claim state without materializing terminal output bodies. */
+  hydrateOutput?: boolean;
   leaseIds?: string[];
   runId: string;
   runAttempt?: number | null;
@@ -221,6 +283,22 @@ export interface CompleteRuntimeStepReceiptsInput {
     runAttempt?: number | null;
     leaseId?: string | null;
     output: unknown | null;
+    inlineFields?: RuntimeReceiptValueInlineFields;
+  }>;
+}
+
+/**
+ * A complete ToolExecuteResult that the trusted execution relay already put
+ * into the receipt-owned R2 namespace. This is a separate named runtime API
+ * operation; it is never an optional reinterpretation of a legacy completion.
+ */
+export interface CompleteRuntimeStepReceiptsByReferenceInput {
+  receipts: Array<{
+    key: string;
+    runId: string;
+    runAttempt?: number | null;
+    leaseId?: string | null;
+    outputRef: RuntimeReceiptValueReference;
   }>;
 }
 
@@ -682,6 +760,8 @@ export interface ContextOptions {
   requestDurableInvocationFence?: boolean;
   verifyDurableInvocationFence?: (signal?: AbortSignal) => Promise<boolean>;
   runtimeTestFaultHeader?: string | null;
+  /** Runner-owned graceful deadline: stop new rows without aborting commits. */
+  rowAdmissionSignal?: AbortSignal;
   /**
    * Runtime-sheet transport selected by the runner. Daytona sandboxes use the
    * execution gateway and must never fall back to minting direct DB sessions.
@@ -811,6 +891,15 @@ export interface ContextOptions {
      * idempotent settlements, not output that belongs to a newer run.
      */
     recoveredStaleKeys?: string[];
+    /**
+     * Stale keys that prove this attempt no longer owns the sheet: the rows
+     * carry a different write version than the one this attempt admitted them
+     * under. A handler that returns any (instead of throwing
+     * `RuntimeSheetAttemptSupersededError` itself) makes the Context throw it:
+     * the map fails and the persistence latch trips, so no further provider
+     * call is spent on rows that can never be persisted.
+     */
+    supersededKeys?: string[];
   }>;
   /**
    * Persists coalesced, non-terminal row patches before a map suspends.
@@ -854,6 +943,12 @@ export interface ContextOptions {
    * pass a smaller/larger budget to make the OOM boundary deterministic.
    */
   runtimeMapMemoryLimits?: Partial<RuntimeMapMemoryLimits>;
+  /**
+   * Heap reading used by map row admission. Omit to read the live process;
+   * tests pass a fixed snapshot so admission width does not depend on the
+   * host's memory state.
+   */
+  runtimeMapHeapProbe?: () => RuntimeMapHeapSnapshot | null;
   /**
    * Return map PlayDataset handles backed by Runtime Sheet reads after a map
    * has persisted. Runtime adapters with real sheet APIs should enable this;
@@ -1009,6 +1104,11 @@ export interface ContextOptions {
   ) =>
     | Promise<Array<RuntimeStepReceipt | null>>
     | Array<RuntimeStepReceipt | null>;
+  completeRuntimeStepReceiptsByReference?: (
+    input: CompleteRuntimeStepReceiptsByReferenceInput,
+  ) =>
+    | Promise<Array<RuntimeStepReceipt | null>>
+    | Array<RuntimeStepReceipt | null>;
   failRuntimeStepReceipt?: (
     input: FailRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
@@ -1025,6 +1125,12 @@ export interface ContextOptions {
   skipRuntimeStepReceipt?: (
     input: SkipRuntimeStepReceiptInput,
   ) => Promise<RuntimeStepReceipt | null> | RuntimeStepReceipt | null;
+  /**
+   * Gateway request id of the most recent receipt completion request, so a
+   * completion answered with no stored receipt can be traced to the gateway
+   * record that produced it. Diagnostics only; never a control input.
+   */
+  lastRuntimeStepReceiptCompletionRequestId?: () => string | null;
   /** Inline composition scope used for recursion and shared-resource governance. */
   governance?: GovernanceSnapshot;
   /** Internal in-process child view sharing the root run's resource governor. */

package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts

@@ -24,6 +24,7 @@ import {
 import {
   gatewayMachineConfig,
   productionGatewayTopology,
+  RUNTIME_GATEWAY_AUTOSCALED_METADATA_KEY,
   RUNTIME_TOPOLOGY,
 } from '../../deployment/runtime-topology';
 
@@ -360,6 +361,12 @@ async function prepare(sha: string): Promise<void> {
             [EXECUTION_RELAY_ROLLOUT_METADATA]: sha,
             [EXECUTION_RELAY_SOURCE_METADATA]: source.id!,
             fly_process_group: topology.processGroup,
+            ...(source.config?.metadata?.[
+              RUNTIME_GATEWAY_AUTOSCALED_METADATA_KEY
+            ] === '1' ||
+            source.name?.startsWith('execution-relay-autoscale-') === true
+              ? { [RUNTIME_GATEWAY_AUTOSCALED_METADATA_KEY]: '1' }
+              : {}),
           },
         }),
       });

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -51,6 +51,13 @@ export const ABSURD_WORKER_CONFIG =
 export const RELEASE_METADATA_KEY = 'release_id';
 /** Content-addressed Vercel ↔ Fly runtime boundary, generated at build time. */
 export const RUNTIME_SURFACE_METADATA_KEY = 'runtime_surface_sha256';
+/**
+ * Preview release-gate deploy-input identity for the worker component.
+ * `scripts/lib/runtime-component-input.ts` owns the hash; the lane only
+ * stamps it so the next gate can prove the running fleet is that exact
+ * deployment instead of replacing it blindly.
+ */
+export const COMPONENT_INPUT_METADATA_KEY = 'deepline_component_input';
 export const ROLLBACK_HOLD_UNTIL_METADATA_KEY = 'rollback_hold_until_ms';
 export const RUNTIME_DEPLOY_VERSION_ENV = 'DEEPLINE_RUNTIME_DEPLOY_VERSION';
 export const DRAIN_EXIT_ENV = 'DEEPLINE_ABSURD_DRAIN_EXIT';
@@ -67,6 +74,42 @@ export const ABSURD_RUNTIME_WORKER_EXEC = [
 /** Default health/boot poll ceilings. Loud timeout, never a silent give-up. */
 export const DEFAULT_BOOT_TIMEOUT_MS = 300_000;
 export const DEFAULT_BOOT_POLL_INTERVAL_MS = 5_000;
+/** Fail early after repeated exited-state polls and include each machine log. */
+export const DEFAULT_BOOT_EXITED_POLL_LIMIT = 3;
+const EXITED_MACHINE_STATES = new Set(['stopped', 'destroyed', 'failed']);
+/**
+ * A worker receives SIGTERM, first releases its direct scheduler LISTEN
+ * session, then waits for accepted work to settle. This must match the
+ * worker's Fly kill timeout: destroying the Machine skips that cleanup and
+ * can leave the shared scheduler at its physical-session ceiling during a
+ * lane replacement.
+ */
+export const DEFAULT_LANE_REPLACEMENT_STOP_SIGNAL = 'SIGTERM';
+export const DEFAULT_LANE_REPLACEMENT_STOP_TIMEOUT = '300s';
+export const DEFAULT_LANE_REPLACEMENT_STOP_WAIT_MS = 6 * 60_000;
+export const DEFAULT_LANE_REPLACEMENT_STOP_POLL_INTERVAL_MS = 2_000;
+const STOPPED_MACHINE_STATES = new Set([
+  'stopped',
+  'suspended',
+  'destroyed',
+  'failed',
+]);
+
+export function readMachineLogTail(app: string, machineId: string): string {
+  const result = spawnSync(
+    'flyctl',
+    ['logs', '-a', app, '-i', machineId, '--no-tail'],
+    { encoding: 'utf8', env: process.env },
+  );
+  if (result.error) {
+    return `flyctl logs failed to spawn: ${result.error.message}`;
+  }
+  return `${result.stdout ?? ''}${result.stderr ?? ''}`
+    .split('\n')
+    .filter((line) => line.trim().length > 0)
+    .slice(-60)
+    .join('\n');
+}
 /**
  * A Machines config update is accepted before a later list necessarily shows
  * the new version.  Drain is a retirement precondition, so callers must not
@@ -320,6 +363,7 @@ export function buildBootMachineConfig(input: {
   releaseId: string;
   sha: string;
   runtimeSurfaceHash?: string;
+  componentInputHash?: string;
   region?: string;
   rollbackHoldUntilMs?: number;
 }): Record<string, unknown> {
@@ -332,6 +376,9 @@ export function buildBootMachineConfig(input: {
         ...(input.runtimeSurfaceHash
           ? { [RUNTIME_SURFACE_METADATA_KEY]: input.runtimeSurfaceHash }
           : {}),
+        ...(input.componentInputHash
+          ? { [COMPONENT_INPUT_METADATA_KEY]: input.componentInputHash }
+          : {}),
         ...(input.rollbackHoldUntilMs
           ? {
               [ROLLBACK_HOLD_UNTIL_METADATA_KEY]: String(
@@ -962,6 +1009,7 @@ export async function bootLane(input: {
   releaseId: string;
   sha: string;
   runtimeSurfaceHash?: string;
+  componentInputHash?: string;
   count: number;
   region?: string;
   timeoutMs?: number;
@@ -969,6 +1017,8 @@ export async function bootLane(input: {
   imagePropagationRetryDelaysMs?: readonly number[];
   now?: () => number;
   wait?: (ms: number) => Promise<void>;
+  machineLogs?: (machineId: string) => string;
+  exitedPollLimit?: number;
   reuseExisting?: boolean;
   rollbackHoldUntilMs?: number;
   rollbackHoldScalerQuiescenceMs?: number;
@@ -1004,6 +1054,9 @@ export async function bootLane(input: {
             (!input.runtimeSurfaceHash ||
               machine.config?.metadata?.[RUNTIME_SURFACE_METADATA_KEY] ===
                 input.runtimeSurfaceHash) &&
+            (!input.componentInputHash ||
+              machine.config?.metadata?.[COMPONENT_INPUT_METADATA_KEY] ===
+                input.componentInputHash) &&
             (input.rollbackHoldUntilMs !== undefined ||
               machine.config.env?.[DRAIN_EXIT_ENV] !== '1') &&
             reusableStates.has(machine.state ?? ''),
@@ -1044,6 +1097,7 @@ export async function bootLane(input: {
         releaseId: input.releaseId,
         sha: input.sha,
         runtimeSurfaceHash: input.runtimeSurfaceHash,
+        componentInputHash: input.componentInputHash,
         region: input.region,
         rollbackHoldUntilMs: input.rollbackHoldUntilMs,
       }),
@@ -1058,6 +1112,24 @@ export async function bootLane(input: {
 
   const deadline = now() + timeoutMs;
   const ids = new Set([...reused, ...created].map((machine) => machine.id));
+  const machineLogs =
+    input.machineLogs ?? ((id: string) => readMachineLogTail(input.app, id));
+  const exitedPollLimit =
+    input.exitedPollLimit ?? DEFAULT_BOOT_EXITED_POLL_LIMIT;
+  let exitedPolls = 0;
+  const failBoot = (cause: string, laneMachines: FlyMachine[]): never => {
+    const states = laneMachines
+      .map((machine) => `${machine.id}=${machine.state ?? 'unknown'}`)
+      .join(', ');
+    for (const machine of laneMachines) {
+      console.error(
+        `[fly-absurd-lane] boot log for machine ${machine.id} (${machine.state ?? 'unknown'}):\n${machineLogs(machine.id)}`,
+      );
+    }
+    throw new Error(
+      `Lane ${input.releaseId} did not reach ${input.count} healthy machines ${cause} (last: ${states || 'none'}).`,
+    );
+  };
   for (;;) {
     const laneMachines = (
       await input.client.machinesInLane(input.releaseId)
@@ -1078,20 +1150,124 @@ export async function bootLane(input: {
       );
       return laneMachines;
     }
+    const allExited =
+      laneMachines.length > 0 &&
+      laneMachines.every((machine) =>
+        EXITED_MACHINE_STATES.has(machine.state ?? ''),
+      );
+    exitedPolls = allExited ? exitedPolls + 1 : 0;
+    if (exitedPolls >= exitedPollLimit) {
+      failBoot(
+        `because every lane machine exited (${exitedPolls} consecutive polls)`,
+        laneMachines,
+      );
+    }
+    if (now() >= deadline) failBoot(`within ${timeoutMs}ms`, laneMachines);
+    console.info(
+      `[fly-absurd-lane] waiting for lane ${input.releaseId}: healthy=${healthy.length}/${input.count}`,
+    );
+    await wait(pollIntervalMs);
+  }
+}
+
+/**
+ * Replace the Machines that serve one already-active lane without bypassing
+ * their shutdown contract. The lane id stays stable: durable state remains in
+ * the same queue, while the previous processes first release their direct
+ * wake listeners and settle work. Only after the new fleet is healthy do we
+ * delete the now-stopped predecessor Machines.
+ *
+ * This is intentionally a lane operation rather than a `flyctl destroy` loop.
+ * A force destroy terminates the process before its SIGTERM handler can
+ * unsubscribe the wake hook, which leaves direct Postgres sessions to expire
+ * on the provider's schedule and turns a routine Preview update into a shared
+ * scheduler capacity incident.
+ */
+export async function replaceLane(input: {
+  client: FlyMachinesClient;
+  app: string;
+  releaseId: string;
+  sha: string;
+  runtimeSurfaceHash?: string;
+  componentInputHash?: string;
+  count: number;
+  region?: string;
+  stopTimeoutMs?: number;
+  stopPollIntervalMs?: number;
+  now?: () => number;
+  wait?: (ms: number) => Promise<void>;
+  machineLogs?: (machineId: string) => string;
+}): Promise<FlyMachine[]> {
+  const now = input.now ?? (() => Date.now());
+  const wait = input.wait ?? sleep;
+  const stopTimeoutMs =
+    input.stopTimeoutMs ?? DEFAULT_LANE_REPLACEMENT_STOP_WAIT_MS;
+  const stopPollIntervalMs =
+    input.stopPollIntervalMs ?? DEFAULT_LANE_REPLACEMENT_STOP_POLL_INTERVAL_MS;
+  const predecessors = await input.client.machinesInLane(input.releaseId);
+  const predecessorIds = new Set(predecessors.map((machine) => machine.id));
+
+  for (const machine of predecessors) {
+    if (STOPPED_MACHINE_STATES.has(machine.state ?? '')) continue;
+    await input.client.stopMachine(machine.id, {
+      signal: DEFAULT_LANE_REPLACEMENT_STOP_SIGNAL,
+      timeout: DEFAULT_LANE_REPLACEMENT_STOP_TIMEOUT,
+    });
+    console.info(
+      `[fly-absurd-lane] requested graceful stop for machine ${machine.id} ` +
+        `(release ${input.releaseId}).`,
+    );
+  }
+
+  const deadline = now() + stopTimeoutMs;
+  for (;;) {
+    const observed = await input.client.machinesInLane(input.releaseId);
+    const byId = new Map(observed.map((machine) => [machine.id, machine]));
+    const live = [...predecessorIds].filter((id) => {
+      const machine = byId.get(id);
+      // A concurrently removed predecessor cannot retain a process or a
+      // database session, so it is equivalent to an observed stopped state.
+      return machine && !STOPPED_MACHINE_STATES.has(machine.state ?? '');
+    });
+    if (live.length === 0) break;
     if (now() >= deadline) {
-      const states = laneMachines
-        .map((machine) => `${machine.id}=${machine.state ?? 'unknown'}`)
+      const states = live
+        .map((id) => `${id}=${byId.get(id)?.state ?? 'missing'}`)
         .join(', ');
       throw new Error(
-        `Lane ${input.releaseId} did not reach ${input.count} healthy machines within ` +
-          `${timeoutMs}ms (last: ${states || 'none'}).`,
+        `Lane ${input.releaseId} did not finish graceful shutdown within ` +
+          `${stopTimeoutMs}ms; still active: ${states}. Refusing to force-destroy ` +
+          'a worker that may retain shared scheduler connections.',
       );
     }
+    await wait(stopPollIntervalMs);
+  }
+
+  const replacements = await bootLane({
+    client: input.client,
+    app: input.app,
+    releaseId: input.releaseId,
+    sha: input.sha,
+    runtimeSurfaceHash: input.runtimeSurfaceHash,
+    componentInputHash: input.componentInputHash,
+    count: input.count,
+    region: input.region,
+    now,
+    wait,
+    machineLogs: input.machineLogs,
+    // Replacements must use the candidate image. A stopped predecessor can
+    // carry the same release id but must never be silently resurrected.
+    reuseExisting: false,
+  });
+
+  for (const machineId of predecessorIds) {
+    await input.client.deleteMachine(machineId, false);
     console.info(
-      `[fly-absurd-lane] waiting for lane ${input.releaseId}: healthy=${healthy.length}/${input.count}`,
+      `[fly-absurd-lane] removed gracefully stopped predecessor ${machineId} ` +
+        `(release ${input.releaseId}).`,
     );
-    await wait(pollIntervalMs);
   }
+  return replacements;
 }
 
 async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
@@ -1110,6 +1286,15 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
       '--runtime-surface-hash must be a lowercase SHA-256 hex digest.',
     );
   }
+  const componentInputHash =
+    typeof flags['component-input-hash'] === 'string'
+      ? flags['component-input-hash']
+      : undefined;
+  if (componentInputHash && !/^[a-f0-9]{64}$/.test(componentInputHash)) {
+    throw new Error(
+      '--component-input-hash must be a lowercase SHA-256 hex digest.',
+    );
+  }
   const rawRollbackHoldUntilMs = flags['rollback-hold-until-ms'];
   const rollbackHoldUntilMs =
     typeof rawRollbackHoldUntilMs === 'string'
@@ -1135,6 +1320,7 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
     releaseId,
     sha,
     runtimeSurfaceHash,
+    componentInputHash,
     count,
     region,
     reuseExisting,
@@ -1142,6 +1328,39 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
   });
 }
 
+async function cmdReplace(
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const app = resolveApp(flags);
+  const releaseId = requireReleaseId(flags);
+  const sha = requireSha(flags);
+  const count = requireCount(flags);
+  const region = typeof flags.region === 'string' ? flags.region : undefined;
+  const componentInputHash =
+    typeof flags['component-input-hash'] === 'string'
+      ? flags['component-input-hash']
+      : undefined;
+  if (componentInputHash && !/^[a-f0-9]{64}$/.test(componentInputHash)) {
+    throw new Error(
+      '--component-input-hash must be a lowercase SHA-256 hex digest.',
+    );
+  }
+  const client = new FlyMachinesClient(
+    app,
+    requireFlyToken(),
+    globalThis.fetch as FetchLike,
+  );
+  await replaceLane({
+    client,
+    app,
+    releaseId,
+    sha,
+    componentInputHash,
+    count,
+    region,
+  });
+}
+
 export function formatMachineTable(machines: readonly FlyMachine[]): string {
   if (machines.length === 0) return '(no machines)';
   const header = ['ID', 'RELEASE', 'STATE', 'REGION', 'CHECKS', 'DRAIN'];
@@ -1791,7 +2010,8 @@ const USAGE = `fly-absurd-lane <command> [flags]
 Commands:
   push-image --sha <sha> --release <id> [--app <app>]
   plan-image --release <id> [--image-ref <ref>] [--app <app>]
-  boot       --release <id> --sha <sha> --count <K> [--reuse-existing] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
+  boot       --release <id> --sha <sha> --count <K> [--reuse-existing] [--component-input-hash <sha256>] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
+  replace    --release <id> --sha <sha> --count <K> --environment preview [--component-input-hash <sha256>] [--app <app>] [--region <r>]
   list       [--release <id>] [--json] [--app <app>]
   drain          --release <id> [--if-present] [--app <app>]
   stop-claiming  --release <id> [--environment production|preview]
@@ -1822,6 +2042,8 @@ export async function main(argv: readonly string[]): Promise<void> {
       return cmdPlanImage(flags);
     case 'boot':
       return cmdBoot(flags);
+    case 'replace':
+      return cmdReplace(flags);
     case 'list':
       return cmdList(flags);
     case 'drain':

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts

@@ -13,9 +13,30 @@ export function runtimeImageManifestExists(image: string): boolean {
   return run('docker', ['manifest', 'inspect', image], true);
 }
 
-/** Authenticate the local docker client against the Fly registry. */
-export function authenticateFlyRegistry(): void {
-  if (!run('flyctl', ['auth', 'docker'])) {
-    throw new Error('Fly registry authentication failed.');
+export const FLY_REGISTRY_AUTH_ATTEMPTS = 3;
+const FLY_REGISTRY_AUTH_RETRY_DELAY_MS = 1_500;
+
+/**
+ * Authenticate the local docker client against the Fly registry. Concurrent
+ * release tracks each run this; on macOS two simultaneous credential-store
+ * writes race on the keychain item ("already exists", -25299) and one of
+ * them fails even though the credentials are in place, so a failed attempt
+ * is retried after a short pause instead of failing the track.
+ */
+export function authenticateFlyRegistry(
+  sleep: (ms: number) => void = sleepSync,
+): void {
+  for (let attempt = 1; attempt <= FLY_REGISTRY_AUTH_ATTEMPTS; attempt += 1) {
+    if (run('flyctl', ['auth', 'docker'])) return;
+    if (attempt < FLY_REGISTRY_AUTH_ATTEMPTS) {
+      sleep(FLY_REGISTRY_AUTH_RETRY_DELAY_MS * attempt);
+    }
   }
+  throw new Error(
+    `Fly registry authentication failed after ${FLY_REGISTRY_AUTH_ATTEMPTS} attempts.`,
+  );
+}
+
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts

@@ -0,0 +1,71 @@
+/**
+ * Exact-machine capacity observations for the runtime gateway fleet.
+ *
+ * The public gateway metrics endpoint is useful for dashboards, but it cannot
+ * identify which Fly Machine answered a request. Autoscaling therefore probes
+ * the private metrics port through the Machine's 6PN hostname and requires the
+ * caller to validate the exact Machine identity header before using a sample.
+ */
+import type { FlyMachine } from './fly-machine-lifecycle';
+
+export const GATEWAY_UPSTREAM_IN_FLIGHT_METRIC =
+  'deepline_runtime_gateway_upstream_admission_in_flight';
+export const GATEWAY_UPSTREAM_WAITING_METRIC =
+  'deepline_runtime_gateway_upstream_admission_waiting';
+
+export type RuntimeGatewayCapacity = {
+  inFlight: number;
+  waiting: number;
+};
+
+export type RuntimeGatewayCapacitySnapshot = RuntimeGatewayCapacity & {
+  machineId: string;
+};
+
+function parseGauge(metrics: string, metricName: string): number {
+  const match = metrics.match(
+    new RegExp(
+      `^${metricName}(?:\\{[^\\n]*\\})?\\s+([0-9]+(?:\\.[0-9]+)?)\\s*$`,
+      'm',
+    ),
+  );
+  if (!match) {
+    throw new Error(
+      `Gateway capacity proof is missing ${metricName}; refusing to scale from an incomplete Machine sample.`,
+    );
+  }
+  const value = Number(match[1]);
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error(
+      `Gateway capacity proof reported an invalid ${metricName} value.`,
+    );
+  }
+  return value;
+}
+
+export function parseGatewayCapacityMetrics(
+  metrics: string,
+): RuntimeGatewayCapacity {
+  return {
+    inFlight: parseGauge(metrics, GATEWAY_UPSTREAM_IN_FLIGHT_METRIC),
+    waiting: parseGauge(metrics, GATEWAY_UPSTREAM_WAITING_METRIC),
+  };
+}
+
+export function aggregateGatewayCapacity(
+  observations: readonly RuntimeGatewayCapacitySnapshot[],
+): RuntimeGatewayCapacity {
+  return observations.reduce(
+    (total, observation) => ({
+      inFlight: total.inFlight + observation.inFlight,
+      waiting: total.waiting + observation.waiting,
+    }),
+    { inFlight: 0, waiting: 0 },
+  );
+}
+
+export function machineIsServingRuntimeGateway(
+  machine: Pick<FlyMachine, 'state' | 'cordoned'>,
+): boolean {
+  return machine.state === 'started' && machine.cordoned === false;
+}