deepline 0.3.461 → 0.3.463

package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts

@@ -525,6 +525,8 @@ export interface MapExecutionFrame {
     /** Digest of the authored input before Runtime Sheet outputs were merged. */
     authoredInputDigest?: string;
   }>;
+  /** A sealed, bounded array source is readable by page after a wake. */
+  arrayInput?: { hasSourceIdentities: boolean };
   /** Successful inputs, including reused inputs with no distinct stored key. */
   completedRowsCount?: number;
   pendingRowsCount?: number;
@@ -982,6 +984,10 @@ export interface ContextOptions {
       columnTypes?: PlayDatasetColumnTypes;
     },
   ) => Promise<MapStartResult>;
+  /** Internal transport pages; each yielded value is one actual admission response. */
+  onMapStartPages?: (
+    ...args: Parameters<NonNullable<ContextOptions['onMapStart']>>
+  ) => AsyncIterable<MapStartResult>;
   /**
    * Called as ctx.dataset() rows complete — persists executed rows (data +
    * cell meta) into the tenant runtime sheet so the sheet, not an in-memory
@@ -1082,6 +1088,10 @@ export interface ContextOptions {
    * in-memory callback tests can omit it and keep local result resolvers.
    */
   runtimeSheetBackedMapDatasets?: boolean;
+  /** Runner-owned durable source pages for arrays that cannot fit a second resident copy. */
+  runtimeArrayInputStore?: import('./array-input-store').RuntimeArrayInputStore;
+  /** Internal rollout switch for new array snapshots; existing snapshots always replay. */
+  runtimeArrayInputStagingEnabled?: boolean;
   resolvePlay?: (playRef: string) => Promise<ResolvedPlayExecution | null>;
   /** Admit an independently scheduled scalar/HITL Play Run. */
   launchPlayAsync?: (
@@ -1284,6 +1294,13 @@ export interface ProviderRetryCursor {
 }
 
 export interface PlayCheckpoint {
+  /** Scheduler silence proof for one exact detached-runner heartbeat. Older
+   * checkpoints acquire a fresh proof through the existing durable park. */
+  detachedRunnerSilenceConfirmation?: {
+    boundaryId: string;
+    runnerAttempt: number;
+    heartbeatIdentity: string | null;
+  };
   /** Last completed runner invocation within the current scheduler attempt. */
   workCounterInvocationAttempt?: number;
   workCounterInvocationSeq?: number;

package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts

@@ -1,3 +1,4 @@
+import { createRateLimitedRuntimeLogger } from '../observability/runtime-console';
 import { withRuntimeSchedulerClient } from './/scheduler-backends/postgres-pool';
 import { RUNTIME_CAPACITY_POLICY } from './/runtime-capacity-policy';
 import { isIsolatedRuntimeSchedulerSchema } from './/runtime-scheduler-topology';
@@ -6,6 +7,12 @@ import { isIsolatedRuntimeSchedulerSchema } from './/runtime-scheduler-topology'
  * their own attempts; they do not each enumerate the shared Daytona account. */
 export const DAYTONA_JANITOR_INTERVAL_MS = 5 * 60 * 1_000;
 
+/**
+ * Spacing of the `[daytona-janitor.stale_list]` line. The page does not depend
+ * on it: the monitor reads every pass summary and holds one open incident.
+ */
+export const DAYTONA_STALE_LIST_ALERT_INTERVAL_MS = 30 * 60 * 1_000;
+
 export type DaytonaSandboxJanitorOptions = {
   url: string;
   schema: string;
@@ -36,6 +43,11 @@ export function startDaytonaSandboxJanitor(
   }
 
   let running = false;
+  // Spans passes, so it cannot live in the per-pass reaper.
+  const reportStaleList = createRateLimitedRuntimeLogger({
+    log: (...args) => console.warn(...args),
+    intervalMs: DAYTONA_STALE_LIST_ALERT_INTERVAL_MS,
+  });
   const runPass = async (): Promise<void> => {
     if (running) {
       console.info('[daytona-janitor.controller] skip (previous pass running)');
@@ -80,6 +92,7 @@ export function startDaytonaSandboxJanitor(
           },
           schema: options.schema,
         }),
+        options: { reportStaleList },
       });
     } catch (error) {
       // This is a best-effort resource backstop. A failed sweep must never

package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts

@@ -38,6 +38,12 @@
  * authorization failure is logged and the sweep continues — one wedged sandbox
  * must never abort the fleet-wide pass. The only hard failure is an inability
  * to enumerate at all.
+ *
+ * A successful list can still be wrong. From Oct 29 23:00 to Oct 30 06:00 UTC it
+ * returned about 75 sandboxes from other lanes and none of the 1 to 3 live
+ * production ones, so `scanned` never reached zero and every pass reported
+ * errors=0. Each pass therefore compares the list with the durable ledger. A
+ * mismatch only alerts: absence from a list never proves a sandbox is gone.
  */
 import { PLAY_RUNNER_TERMINAL_GRACE_SECONDS } from './/runtime-constants';
 import { DAYTONA_PLAY_RUNNER_LABEL_SOURCE } from './/runner-backends/backends/daytona-labels';
@@ -52,6 +58,15 @@ export { DAYTONA_PLAY_RUNNER_LABEL_SOURCE };
  */
 export const REAP_AGE_GRACE_MS = 5 * 60 * 1000;
 
+/**
+ * A sandbox must be this old when the list begins before the list missing it
+ * means anything, which leaves room for clock skew and Daytona's own lag.
+ */
+export const DAYTONA_DURABLE_LIVE_SETTLE_MS = 60 * 1000;
+/** A live sandbox whose runner has been silent this long may be lost, so one
+ * such sandbox missing from the list is not evidence of a blind list. */
+export const DAYTONA_DURABLE_LIVE_HEARTBEAT_STALE_MS = 5 * 60 * 1000;
+
 /**
  * A single sandbox as seen by the janitor. Structurally a subset of the Daytona
  * SDK `Sandbox`, so a real `Sandbox` satisfies it directly.
@@ -133,6 +148,11 @@ export type DaytonaJanitorOwnershipPort = {
     runId: string;
     attempt: number;
   }): Promise<DaytonaJanitorCleanupAuthorization>;
+  /** One statement per pass; its cost never scales with the provider list. */
+  countDurableLive(input: {
+    createdBeforeMs: number;
+    heartbeatStaleMs: number;
+  }): Promise<{ live: number; heartbeating: number }>;
 };
 
 export type ReapOptions = {
@@ -142,6 +162,8 @@ export type ReapOptions = {
   dryRun?: boolean;
   /** Structured logger; defaults to console. */
   logger?: Pick<Console, 'info' | 'warn'>;
+  /** Rate limited by the long-lived controller; defaults to `logger.warn`. */
+  reportStaleList?: (message: string, detail: Record<string, unknown>) => void;
 };
 
 export type ReapSummary = {
@@ -152,6 +174,14 @@ export type ReapSummary = {
   spared: number;
   alerts: number;
   errors: number;
+  /** Sandboxes the ledger held live throughout this list; null if unreadable. */
+  durableLive: number | null;
+  /** Of `durableLive`, those whose runner heartbeat is recent or not started. */
+  durableLiveHeartbeating: number | null;
+  /** Listed sandboxes that the ledger holds live. */
+  listedDurableLive: number;
+  /** The list succeeded but showed none of `durableLive`. */
+  listStale: boolean;
 };
 
 type ReapDecision =
@@ -232,8 +262,14 @@ export async function reapOrphanDaytonaSandboxes(input: {
     spared: 0,
     alerts: 0,
     errors: 0,
+    durableLive: null,
+    durableLiveHeartbeating: null,
+    listedDurableLive: 0,
+    listStale: false,
   };
+  let ownershipReadFailed = false;
 
+  const listStartedAtMs = now();
   const iterator = input.daytona.list({
     labels: { source: DAYTONA_PLAY_RUNNER_LABEL_SOURCE },
   });
@@ -253,6 +289,7 @@ export async function reapOrphanDaytonaSandboxes(input: {
         routingDomain,
       });
     } catch (error) {
+      ownershipReadFailed = true;
       summary.errors += 1;
       summary.spared += 1;
       logger.warn('[daytona-janitor.ownership_read_failed]', {
@@ -279,6 +316,8 @@ export async function reapOrphanDaytonaSandboxes(input: {
       continue;
     }
 
+    if (ownership.operationState === 'live') summary.listedDurableLive += 1;
+
     if (labelRunId && labelRunId !== ownership.runId) {
       logger.info('[daytona-janitor.label_correlation_mismatch]', {
         sandboxId: sandbox.id,
@@ -388,6 +427,42 @@ export async function reapOrphanDaytonaSandboxes(input: {
     }
   }
 
+  // Read after the list ends: a row still live now, and created before the list
+  // began, was live for the whole list, so a miss cannot be a race.
+  try {
+    const durable = await input.ownership.countDurableLive({
+      createdBeforeMs: listStartedAtMs - DAYTONA_DURABLE_LIVE_SETTLE_MS,
+      heartbeatStaleMs: DAYTONA_DURABLE_LIVE_HEARTBEAT_STALE_MS,
+    });
+    summary.durableLive = durable.live;
+    summary.durableLiveHeartbeating = durable.heartbeating;
+  } catch (error) {
+    summary.errors += 1;
+    logger.warn('[daytona-janitor.durable_live_read_failed]', {
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+  // One missing sandbox whose runner went silent is likely a loss whose run
+  // has not settled. A missing runner that still heartbeats, or several
+  // missing at once (a gateway outage silences them all), is a blind list.
+  summary.listStale =
+    summary.durableLive !== null &&
+    summary.durableLiveHeartbeating !== null &&
+    summary.listedDurableLive === 0 &&
+    (summary.durableLiveHeartbeating > 0 || summary.durableLive >= 2) &&
+    !ownershipReadFailed;
+  if (summary.listStale) {
+    const reportStaleList =
+      input.options?.reportStaleList ??
+      ((message, detail) => logger.warn(message, detail));
+    reportStaleList('[daytona-janitor.stale_list]', {
+      provider: 'daytona',
+      durableLive: summary.durableLive,
+      scanned: summary.scanned,
+      dryRun,
+    });
+  }
+
   logger.info('[daytona-janitor.summary]', { ...summary, dryRun });
   return summary;
 }
@@ -453,6 +528,7 @@ export function createPostgresDaytonaJanitorOwnershipPort(input: {
   schema: string;
 }): DaytonaJanitorOwnershipPort {
   const runs = `${quoteIdent(input.schema)}.${quoteIdent('runs')}`;
+  const attempts = `${quoteIdent(input.schema)}.${quoteIdent('attempts')}`;
   const operations = `${quoteIdent(input.schema)}.${quoteIdent(
     'runtime_sandbox_operations',
   )}`;
@@ -652,6 +728,40 @@ export function createPostgresDaytonaJanitorOwnershipPort(input: {
       }
       return row.inserted ? 'scheduled' : 'already_authorized';
     },
+    async countDurableLive({ createdBeforeMs, heartbeatStaleMs }) {
+      // The schema already scopes this to one lane. Listing both environments
+      // keeps the reconcile index usable. A finished run's sandbox is cleanup's
+      // to settle, so it says nothing about whether the list is current.
+      const result = await input.client.query<{
+        live: string;
+        heartbeating: string;
+      }>(
+        `
+          SELECT count(*)::text AS live,
+                 count(*) FILTER (
+                   WHERE attempt.heartbeat_at IS NULL
+                      OR attempt.heartbeat_at > clock_timestamp()
+                        - make_interval(secs => $2::double precision / 1000.0)
+                 )::text AS heartbeating
+          FROM ${operations} AS operation
+          JOIN ${runs} AS run ON run.run_id = operation.run_id
+          LEFT JOIN ${attempts} AS attempt
+            ON attempt.run_id = operation.run_id
+           AND attempt.attempt = operation.attempt
+          WHERE operation.provider = 'daytona'
+            AND operation.environment IN ('production', 'preview')
+            AND operation.state = 'live'
+            AND operation.provider_created_at
+              <= to_timestamp($1::double precision / 1000.0)
+            AND run.status IN ('queued', 'running', 'waiting')
+        `,
+        [createdBeforeMs, heartbeatStaleMs],
+      );
+      return {
+        live: Number(result.rows[0]?.live ?? 0),
+        heartbeating: Number(result.rows[0]?.heartbeating ?? 0),
+      };
+    },
   };
 }
 

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -27,6 +27,7 @@
  * secret and correctly lives in the environment.
  */
 import { spawnSync } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
 import { appendFileSync } from 'node:fs';
 import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
 import { ABSURD_RELEASE_ID_RE } from '../runtime-scheduler-topology';
@@ -228,6 +229,13 @@ export type FlyMachine = {
    * routing event as an explicit legacy-drain fence. */
   events?: Array<{ type?: string; timestamp?: number }>;
   region?: string;
+  /** Native resolved identity; config.image can omit its immutable digest. */
+  image_ref?: {
+    registry?: string;
+    repository?: string;
+    tag?: string;
+    digest?: string;
+  } | null;
   config?: {
     image?: string;
     metadata?: Record<string, string>;
@@ -279,6 +287,12 @@ export function parseArgs(argv: readonly string[]): ParsedArgs {
       index += 1;
     }
   }
+  if (
+    'image-ref' in flags &&
+    !['boot', 'replace', 'plan-image'].includes(command)
+  ) {
+    throw new Error(`--image-ref is unsupported for ${command}.`);
+  }
   return { command, flags };
 }
 
@@ -346,6 +360,58 @@ export function laneImageRef(app: string, sha: string): string {
   return `${FLY_REGISTRY_HOST}/${app}:git-${sha}`;
 }
 
+/** Preserve the registered image identity instead of deriving a tag in the target app.
+ * Shared Preview images must retain both their exact source tag and native digest.
+ */
+export function resolveWorkerLaneImage(input: {
+  app: string;
+  sha?: string;
+  imageRef: string;
+}): { sha: string; imageRef: string } {
+  const prefix = `${FLY_REGISTRY_HOST}/${input.app}:git-`;
+  const legacySha = input.imageRef.startsWith(prefix)
+    ? input.imageRef.slice(prefix.length)
+    : null;
+  const sharedSha =
+    /^registry\.fly\.io\/deepline-absurd-worker-preview-cache:git-([a-f0-9]{40})-worker-[a-f0-9]{64}@sha256:[a-f0-9]{64}$/.exec(
+      input.imageRef,
+    )?.[1];
+  const sha =
+    sharedSha ?? (legacySha && GIT_SHA_RE.test(legacySha) ? legacySha : null);
+  if (!sha)
+    throw new Error(`Invalid worker image reference: ${input.imageRef}`);
+  if (input.sha !== undefined && sha !== input.sha) {
+    throw new Error(
+      `Worker image source ${sha} disagrees with requested source ${input.sha}.`,
+    );
+  }
+  return { sha, imageRef: input.imageRef };
+}
+
+/** Exact native immutable identity, shared semantically with the release gate's
+ * jq predicates. Config equality alone proves a requested tag, not its digest. */
+export function flyMachineMatchesImage(
+  machine: FlyMachine,
+  image: string,
+): boolean {
+  if (!image.includes('@')) return machine.config?.image === image;
+  const requested =
+    /^([A-Za-z0-9.-]+(?::[0-9]+)?)\/([A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*):([A-Za-z0-9_][A-Za-z0-9_.-]*)@(sha256:[a-f0-9]{64})$/.exec(
+      image,
+    );
+  const resolved = machine.image_ref;
+  if (!requested || !resolved || typeof resolved !== 'object') return false;
+  const [, registry, repository, imageTag, digest] = requested;
+  const tag = `${registry}/${repository}:${imageTag}`;
+  return (
+    (machine.config?.image === image || machine.config?.image === tag) &&
+    resolved.registry === registry &&
+    resolved.repository === repository &&
+    resolved.tag === imageTag &&
+    resolved.digest === digest
+  );
+}
+
 export function laneQueueName(releaseId: string): string {
   return `runs_${releaseId}`;
 }
@@ -366,16 +432,32 @@ export function buildBootMachineConfig(input: {
   app: string;
   releaseId: string;
   sha: string;
+  imageRef?: string;
   runtimeSurfaceHash?: string;
   componentInputHash?: string;
   region?: string;
   rollbackHoldUntilMs?: number;
   draining?: boolean;
+  /** Provider secrets propagation fence; never a runtime schema or wire floor. */
+  secretsPropagationFence?: number;
 }): Record<string, unknown> {
+  if (input.secretsPropagationFence !== undefined) {
+    assertMinSecretsVersion(input.secretsPropagationFence);
+  }
   return {
     region: input.region ?? DEFAULT_ABSURD_FLY_REGION,
+    ...(input.secretsPropagationFence === undefined
+      ? {}
+      : { min_secrets_version: input.secretsPropagationFence }),
     config: {
-      image: laneImageRef(input.app, input.sha),
+      image:
+        input.imageRef === undefined
+          ? laneImageRef(input.app, input.sha)
+          : resolveWorkerLaneImage({
+              app: input.app,
+              sha: input.sha,
+              imageRef: input.imageRef,
+            }).imageRef,
       metadata: {
         [RELEASE_METADATA_KEY]: input.releaseId,
         ...(input.runtimeSurfaceHash
@@ -569,6 +651,12 @@ export function requireFlyToken(
   return token;
 }
 
+function assertMinSecretsVersion(value: unknown): asserts value is number {
+  if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) {
+    throw new Error('Fly secrets version must be a positive safe integer.');
+  }
+}
+
 export class FlyMachinesClient {
   constructor(
     private readonly app: string,
@@ -681,6 +769,22 @@ export class FlyMachinesClient {
     return (await this.request('POST', '/machines', config)) as FlyMachine;
   }
 
+  /**
+   * Match native `flyctl secrets sync` without consulting its local config.
+   * The native protocol unsets a random nonexistent name to obtain a version;
+   * GET /secrets returns metadata only and cannot establish this fence.
+   * https://github.com/superfly/flyctl/blob/b3263736a487b5c713fc7733633cfbb2e2976dd6/internal/appsecrets/secrets.go#L48-L57
+   */
+  async syncAppSecretsVersion(): Promise<number> {
+    const name = `BogusDummySecret_${randomBytes(8).toString('hex')}`;
+    const result = await this.request('POST', '/secrets', {
+      values: { [name]: null },
+    });
+    const version = (result as { version?: unknown } | null)?.version;
+    assertMinSecretsVersion(version);
+    return version;
+  }
+
   async updateMachine(
     id: string,
     config: Record<string, unknown>,
@@ -1063,8 +1167,11 @@ export async function bootLane(input: {
   app: string;
   releaseId: string;
   sha: string;
+  imageRef?: string;
   runtimeSurfaceHash?: string;
   componentInputHash?: string;
+  /** Provider secrets propagation fence; never a runtime schema or wire floor. */
+  secretsPropagationFence?: number;
   count: number;
   region?: string;
   timeoutMs?: number;
@@ -1078,6 +1185,9 @@ export async function bootLane(input: {
   rollbackHoldUntilMs?: number;
   rollbackHoldScalerQuiescenceMs?: number;
 }): Promise<FlyMachine[]> {
+  if (input.secretsPropagationFence !== undefined) {
+    assertMinSecretsVersion(input.secretsPropagationFence);
+  }
   const now = input.now ?? (() => Date.now());
   const wait = input.wait ?? sleep;
   const timeoutMs = input.timeoutMs ?? DEFAULT_BOOT_TIMEOUT_MS;
@@ -1086,7 +1196,14 @@ export async function bootLane(input: {
     input.imagePropagationRetryDelaysMs ??
     DEFAULT_IMAGE_PROPAGATION_RETRY_DELAYS_MS;
 
-  const expectedImage = laneImageRef(input.app, input.sha);
+  const expectedImage =
+    input.imageRef === undefined
+      ? laneImageRef(input.app, input.sha)
+      : resolveWorkerLaneImage({
+          app: input.app,
+          sha: input.sha,
+          imageRef: input.imageRef,
+        }).imageRef;
   const reusableStates = new Set([
     'started',
     'created',
@@ -1105,7 +1222,7 @@ export async function bootLane(input: {
     ? (await input.client.machinesInLane(input.releaseId))
         .filter(
           (machine) =>
-            machine.config?.image === expectedImage &&
+            flyMachineMatchesImage(machine, expectedImage) &&
             (!input.runtimeSurfaceHash ||
               machine.config?.metadata?.[RUNTIME_SURFACE_METADATA_KEY] ===
                 input.runtimeSurfaceHash) &&
@@ -1113,7 +1230,7 @@ export async function bootLane(input: {
               machine.config?.metadata?.[COMPONENT_INPUT_METADATA_KEY] ===
                 input.componentInputHash) &&
             (input.rollbackHoldUntilMs !== undefined ||
-              machine.config.env?.[DRAIN_EXIT_ENV] !== '1') &&
+              machine.config?.env?.[DRAIN_EXIT_ENV] !== '1') &&
             reusableStates.has(machine.state ?? ''),
         )
         .sort((left, right) => statePriority(left) - statePriority(right))
@@ -1151,8 +1268,10 @@ export async function bootLane(input: {
         app: input.app,
         releaseId: input.releaseId,
         sha: input.sha,
+        imageRef: input.imageRef,
         runtimeSurfaceHash: input.runtimeSurfaceHash,
         componentInputHash: input.componentInputHash,
+        secretsPropagationFence: input.secretsPropagationFence,
         region: input.region,
         rollbackHoldUntilMs: input.rollbackHoldUntilMs,
       }),
@@ -1243,8 +1362,11 @@ export async function replaceLane(input: {
   app: string;
   releaseId: string;
   sha: string;
+  imageRef?: string;
   runtimeSurfaceHash?: string;
   componentInputHash?: string;
+  /** Provider secrets propagation fence; never a runtime schema or wire floor. */
+  secretsPropagationFence?: number;
   count: number;
   region?: string;
   stopTimeoutMs?: number;
@@ -1253,12 +1375,21 @@ export async function replaceLane(input: {
   wait?: (ms: number) => Promise<void>;
   machineLogs?: (machineId: string) => string;
 }): Promise<FlyMachine[]> {
+  if (input.secretsPropagationFence !== undefined) {
+    assertMinSecretsVersion(input.secretsPropagationFence);
+  }
   const now = input.now ?? (() => Date.now());
   const wait = input.wait ?? sleep;
   const stopTimeoutMs =
     input.stopTimeoutMs ?? DEFAULT_LANE_REPLACEMENT_STOP_WAIT_MS;
   const stopPollIntervalMs =
     input.stopPollIntervalMs ?? DEFAULT_LANE_REPLACEMENT_STOP_POLL_INTERVAL_MS;
+  if (input.imageRef !== undefined)
+    resolveWorkerLaneImage({
+      app: input.app,
+      sha: input.sha,
+      imageRef: input.imageRef,
+    });
   const predecessors = await input.client.machinesInLane(input.releaseId);
   const predecessorIds = new Set(predecessors.map((machine) => machine.id));
 
@@ -1303,8 +1434,10 @@ export async function replaceLane(input: {
     app: input.app,
     releaseId: input.releaseId,
     sha: input.sha,
+    imageRef: input.imageRef,
     runtimeSurfaceHash: input.runtimeSurfaceHash,
     componentInputHash: input.componentInputHash,
+    secretsPropagationFence: input.secretsPropagationFence,
     count: input.count,
     region: input.region,
     now,
@@ -1329,6 +1462,10 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
   const app = resolveApp(flags);
   const releaseId = requireReleaseId(flags);
   const sha = requireSha(flags);
+  const imageRef =
+    flags['image-ref'] === undefined
+      ? undefined
+      : requireStringFlag(flags, 'image-ref');
   const count = requireCount(flags);
   const region = typeof flags.region === 'string' ? flags.region : undefined;
   const reuseExisting = flags['reuse-existing'] === true;
@@ -1374,12 +1511,16 @@ async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
     app,
     releaseId,
     sha,
+    imageRef,
     runtimeSurfaceHash,
     componentInputHash,
     count,
     region,
     reuseExisting,
     rollbackHoldUntilMs,
+    ...(flags.environment === 'preview'
+      ? { secretsPropagationFence: await client.syncAppSecretsVersion() }
+      : {}),
   });
 }
 
@@ -1389,6 +1530,10 @@ async function cmdReplace(
   const app = resolveApp(flags);
   const releaseId = requireReleaseId(flags);
   const sha = requireSha(flags);
+  const imageRef =
+    flags['image-ref'] === undefined
+      ? undefined
+      : requireStringFlag(flags, 'image-ref');
   const count = requireCount(flags);
   const region = typeof flags.region === 'string' ? flags.region : undefined;
   const componentInputHash =
@@ -1410,9 +1555,13 @@ async function cmdReplace(
     app,
     releaseId,
     sha,
+    imageRef,
     componentInputHash,
     count,
     region,
+    ...(flags.environment === 'preview'
+      ? { secretsPropagationFence: await client.syncAppSecretsVersion() }
+      : {}),
   });
 }
 
@@ -2065,8 +2214,8 @@ const USAGE = `fly-absurd-lane <command> [flags]
 Commands:
   push-image --sha <sha> --release <id> [--app <app>]
   plan-image --release <id> [--image-ref <ref>] [--app <app>]
-  boot       --release <id> --sha <sha> --count <K> [--reuse-existing] [--component-input-hash <sha256>] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
-  replace    --release <id> --sha <sha> --count <K> --environment preview [--component-input-hash <sha256>] [--app <app>] [--region <r>]
+  boot       --release <id> --sha <sha> --count <K> [--image-ref <ref>] [--reuse-existing] [--component-input-hash <sha256>] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
+  replace    --release <id> --sha <sha> --count <K> [--image-ref <ref>] --environment preview [--component-input-hash <sha256>] [--app <app>] [--region <r>]
   list       [--release <id>] [--json] [--app <app>]
   drain          --release <id> [--if-present] [--app <app>]
   stop-claiming  --release <id> [--environment production|preview]