@molecule/api-code-sandbox-flyio 1.2.7 → 1.2.9

package/dist/api.d.ts

@@ -125,7 +125,12 @@ export declare class FlyApiClient {
     private readonly baseUrl;
     private readonly timeoutMs;
     private readonly fetchImpl;
-    private readonly sleep;
+    /**
+     * Injectable delay, shared with the provider: a retry that a CALLER drives
+     * (the provider's exec-precondition backoff) waits on the same injected clock
+     * the client's own retries use, so tests collapse every wait in one place.
+     */
+    readonly sleep: (ms: number) => Promise<void>;
     /**
      * Creates a Fly Machines API client.
      * @param options - Token resolver, base URL, and injectable fetch/sleep.

package/dist/api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAgBH;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,KAAK;IACpC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB;;;;;;;OAOG;gBAED,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAgBhC;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0DAA0D;IAC1D,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAQhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAoBjE;AAED,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAClC,gFAAgF;IAChF,KAAK,EAAE,MAAM,MAAM,GAAG,SAAS,CAAA;IAC/B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAgDD,kFAAkF;AAClF,wBAAgB,qBAAqB,IAAI,IAAI,CAG5C;AAED;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IAErD;;;OAGG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;;;;;;;;;OAUG;IACG,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAiFnF"}
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAgBH;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,KAAK;IACpC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB;;;;;;;OAOG;gBAED,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAgBhC;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0DAA0D;IAC1D,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAQhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAoBjE;AAED,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAClC,gFAAgF;IAChF,KAAK,EAAE,MAAM,MAAM,GAAG,SAAS,CAAA;IAC/B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAgDD,kFAAkF;AAClF,wBAAgB,qBAAqB,IAAI,IAAI,CAG5C;AAED;;;;GAIG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC;;;;OAIG;IACH,QAAQ,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAE7C;;;OAGG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;;;;;;;;;OAUG;IACG,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAiFnF"}

package/dist/api.js

@@ -176,6 +176,11 @@ export class FlyApiClient {
     baseUrl;
     timeoutMs;
     fetchImpl;
+    /**
+     * Injectable delay, shared with the provider: a retry that a CALLER drives
+     * (the provider's exec-precondition backoff) waits on the same injected clock
+     * the client's own retries use, so tests collapse every wait in one place.
+     */
     sleep;
     /**
      * Creates a Fly Machines API client.

package/dist/index.d.ts

@@ -315,12 +315,26 @@
  * short-term burst limit up to 3 req/s, per action", scoped "per-action,
  * per-machine … That might be Machine ID or App ID, depending on the type of
  * request", with Get Machine at 5 req/s and app deletions capped at 100 per
- * minute. `429` and `5xx` are retried with `Retry-After`-aware backoff; other
- * `4xx` responses are real answers and are never retried. The consequence to
- * plan for is that EVERY file operation is an exec API call, so one sandbox's
- * reads and writes serialize at roughly one per second and a multi-megabyte
- * `writeFile` (chunked at ~45 KB decoded) takes tens of seconds. An agent that
- * edits files in a tight loop will feel this; batch where you can.
+ * minute. `429` and `5xx` are retried with `Retry-After`-aware backoff, plus the
+ * `404` an app-propagation lag answers the Machine create with; every other
+ * `4xx` is a real answer and is never retried. The consequence to plan for is
+ * that EVERY file operation is an exec API call, so one sandbox's reads and
+ * writes serialize at roughly one per second and a multi-megabyte `writeFile`
+ * (chunked at ~45 KB decoded) takes tens of seconds. An agent that edits files
+ * in a tight loop will feel this; batch where you can.
+ *
+ * **A refused exec that dropped the connection is retried; a refused exec that
+ * names a Machine state is not.** Fly answers `412 failed_precondition: exec
+ * request failed: EOF` when the request reached a `started` Machine's exec agent
+ * and the connection died before the command ran — a Machine seconds past boot,
+ * or Fly's own exec burst. `exec` retries that one at the transport (up to four
+ * attempts, ~11 s of backoff, each preceded by a Machine-state read and a wait
+ * for `started` when it has not got there), because every caller's alternative
+ * is to throw away a whole boot or deploy and start over. The OTHER `412`,
+ * `failed_precondition: machine not running`, is a real state and is not
+ * repeated: `start()`/`wake()` wait it out before the first exec. A 412 that is
+ * not retried comes back with the Machine's current state appended to its
+ * message, and a 412 the retries could not clear carries the state it ended in.
  *
  * **`create()` launches the Machine, and `start()`/`wake()` WAIT for it.** Unlike
  * Docker's create-then-start, the Machines API starts a Machine as part of

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqVG;AAEH,cAAc,UAAU,CAAA;AACxB,cAAc,oBAAoB,CAAA;AAClC,cAAc,aAAa,CAAA;AAC3B,cAAc,WAAW,CAAA;AACzB,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,eAAe,CAAA;AAC7B,cAAc,cAAc,CAAA;AAC5B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,YAAY,CAAA;AAC1B,cAAc,gBAAgB,CAAA;AAE9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAMjE;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,eAUrB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmWG;AAEH,cAAc,UAAU,CAAA;AACxB,cAAc,oBAAoB,CAAA;AAClC,cAAc,aAAa,CAAA;AAC3B,cAAc,WAAW,CAAA;AACzB,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,eAAe,CAAA;AAC7B,cAAc,cAAc,CAAA;AAC5B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,YAAY,CAAA;AAC1B,cAAc,gBAAgB,CAAA;AAE9B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAMjE;;GAEG;AACH,eAAO,MAAM,QAAQ,EAAE,eAUrB,CAAA"}

package/dist/index.js

@@ -315,12 +315,26 @@
  * short-term burst limit up to 3 req/s, per action", scoped "per-action,
  * per-machine … That might be Machine ID or App ID, depending on the type of
  * request", with Get Machine at 5 req/s and app deletions capped at 100 per
- * minute. `429` and `5xx` are retried with `Retry-After`-aware backoff; other
- * `4xx` responses are real answers and are never retried. The consequence to
- * plan for is that EVERY file operation is an exec API call, so one sandbox's
- * reads and writes serialize at roughly one per second and a multi-megabyte
- * `writeFile` (chunked at ~45 KB decoded) takes tens of seconds. An agent that
- * edits files in a tight loop will feel this; batch where you can.
+ * minute. `429` and `5xx` are retried with `Retry-After`-aware backoff, plus the
+ * `404` an app-propagation lag answers the Machine create with; every other
+ * `4xx` is a real answer and is never retried. The consequence to plan for is
+ * that EVERY file operation is an exec API call, so one sandbox's reads and
+ * writes serialize at roughly one per second and a multi-megabyte `writeFile`
+ * (chunked at ~45 KB decoded) takes tens of seconds. An agent that edits files
+ * in a tight loop will feel this; batch where you can.
+ *
+ * **A refused exec that dropped the connection is retried; a refused exec that
+ * names a Machine state is not.** Fly answers `412 failed_precondition: exec
+ * request failed: EOF` when the request reached a `started` Machine's exec agent
+ * and the connection died before the command ran — a Machine seconds past boot,
+ * or Fly's own exec burst. `exec` retries that one at the transport (up to four
+ * attempts, ~11 s of backoff, each preceded by a Machine-state read and a wait
+ * for `started` when it has not got there), because every caller's alternative
+ * is to throw away a whole boot or deploy and start over. The OTHER `412`,
+ * `failed_precondition: machine not running`, is a real state and is not
+ * repeated: `start()`/`wake()` wait it out before the first exec. A 412 that is
+ * not retried comes back with the Machine's current state appended to its
+ * message, and a 412 the retries could not clear carries the state it ended in.
  *
  * **`create()` launches the Machine, and `start()`/`wake()` WAIT for it.** Unlike
  * Docker's create-then-start, the Machines API starts a Machine as part of

package/dist/provider.d.ts

@@ -10,6 +10,23 @@ import type { SandboxProvider } from '@molecule/api-code-sandbox';
 import { FlyApiClient } from './api.js';
 import { type ObjectStore } from './storage.js';
 import type { FlyioConfig } from './types.js';
+/**
+ * Whether a failed exec is Fly's TRANSIENT precondition — the request reached
+ * the Machine's exec agent and the connection was dropped before the command
+ * ran (`412 failed_precondition: exec request failed: EOF`) — rather than the
+ * Machine being in a state an exec cannot run in.
+ *
+ * The distinction decides what retrying can achieve. A dropped connection is
+ * worth repeating: the Machine is `started` and its agent recovers in seconds.
+ * `failed_precondition: machine not running` is NOT — the Machine really is not
+ * running, and that is what {@link FlyioSandboxProvider.startMachine} waits out
+ * before the first exec. Repeating the exec there would only burn the caller's
+ * budget against a condition the wait endpoint already answers.
+ *
+ * @param error - The failure an exec attempt raised.
+ * @returns `true` when repeating the exec is the right recovery.
+ */
+export declare function isTransientExecPrecondition(error: unknown): boolean;
 /**
  * Creates a Fly.io Machines sandbox provider.
  * @param config - Optional Fly configuration: API token/URL, org slug, app

package/dist/provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAWV,eAAe,EAGhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAwBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAIV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAigFnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAWV,eAAe,EAGhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAwBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAIV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAkBnB;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAGnE;AA8oFD;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}

package/dist/provider.js

@@ -31,6 +31,27 @@ function describeDestroyFailure(step, id, error) {
     const reason = error instanceof Error ? error.message : String(error);
     return `Fly sandbox destroy failed at the ${step} for ${id} (${status}): ${reason}`;
 }
+/**
+ * Whether a failed exec is Fly's TRANSIENT precondition — the request reached
+ * the Machine's exec agent and the connection was dropped before the command
+ * ran (`412 failed_precondition: exec request failed: EOF`) — rather than the
+ * Machine being in a state an exec cannot run in.
+ *
+ * The distinction decides what retrying can achieve. A dropped connection is
+ * worth repeating: the Machine is `started` and its agent recovers in seconds.
+ * `failed_precondition: machine not running` is NOT — the Machine really is not
+ * running, and that is what {@link FlyioSandboxProvider.startMachine} waits out
+ * before the first exec. Repeating the exec there would only burn the caller's
+ * budget against a condition the wait endpoint already answers.
+ *
+ * @param error - The failure an exec attempt raised.
+ * @returns `true` when repeating the exec is the right recovery.
+ */
+export function isTransientExecPrecondition(error) {
+    if (!(error instanceof FlyApiError) || error.status !== 412)
+        return false;
+    return EXEC_BODY_TRANSIENT.test(error.body) && !EXEC_BODY_MACHINE_NOT_RUNNING.test(error.body);
+}
 /**
  * Default sandbox image. Fly pulls images itself, so this MUST be a reference
  * Fly can resolve — a tag in the org's `registry.fly.io` repository or a public
@@ -149,10 +170,59 @@ const MACHINE_CREATE_ATTEMPTS = 6;
 /**
  * Retrying a `POST /machines` on 404 is SAFE precisely because "app not found"
  * means no Machine was created — unlike a 5xx/timeout, where one might have
- * been. So only 404 is added here, on top of the client's default retryable
- * set; the retry cannot double-create.
+ * been. 412 is safe for the same reason: "volume already claimed by machine
+ * <id>" refuses the create. It is what the create answers in the seconds after
+ * {@link FlyioSandboxProvider.destroyStaleMachines} removed the Machine that
+ * held the volume — Fly releases the claim asynchronously, and an immediate
+ * create lost a production deploy to it (2026-10-14; the identical retry a
+ * minute later succeeded). Only these two are added on top of the client's
+ * default retryable set; neither retry can double-create.
+ */
+const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
+/**
+ * What Fly answers when it reached the exec endpoint but the connection to the
+ * Machine's exec agent died before the command ran: `412 {"error":
+ * "failed_precondition: exec request failed: EOF"}`. The Machine is `started`;
+ * the agent inside it dropped the request (a Machine seconds past boot, or
+ * Fly's own exec burst). The SAME status carries the precondition this provider
+ * already handles — `failed_precondition: machine not running` — which is a
+ * real state, not a dropped connection, and is reconciled by `startMachine`
+ * BEFORE the first exec instead of by repeating it (see {@link
+ * isTransientExecPrecondition}).
+ */
+const EXEC_BODY_TRANSIENT = /exec request failed/i;
+/** The real-precondition 412, which retrying an exec cannot fix. */
+const EXEC_BODY_MACHINE_NOT_RUNNING = /machine not running/i;
+/**
+ * Attempts for one exec when Fly answers the transient 412 of
+ * {@link EXEC_BODY_TRANSIENT}. Four attempts over the three backoffs below is
+ * under 12 s of waiting, against the ~75 s each fresh production deploy costs
+ * when this is left to a caller: X1 run flash-m85 (2026-10-16) lost three
+ * consecutive deploys, and the run, to this one condition.
+ */
+const EXEC_PRECONDITION_ATTEMPTS = 4;
+/**
+ * Backoff before each exec retry (ms), indexed by the failed attempt number.
+ * Short on purpose: the Machine is up, and the agent recovers in seconds —
+ * nothing here is waiting on a boot.
+ */
+const EXEC_PRECONDITION_BACKOFF_MS = [1_000, 3_000, 7_000];
+/**
+ * Wall-clock ceiling on {@link EXEC_PRECONDITION_ATTEMPTS}, so a Machine whose
+ * exec agent keeps dropping requests cannot hold a caller for the whole budget
+ * while the rest of the backoff ladder runs. Measured from the first attempt.
  */
-const MACHINE_CREATE_RETRY_STATUSES = [404];
+const EXEC_PRECONDITION_BUDGET_MS = 60_000;
+/**
+ * The backoff before an exec retry, clamped to the last rung so an attempt
+ * count raised past the ladder's length cannot read off the end of it.
+ * @param attempt - The 1-based number of the attempt that just failed.
+ * @returns The delay in milliseconds.
+ */
+function execPreconditionBackoffMs(attempt) {
+    const last = EXEC_PRECONDITION_BACKOFF_MS.length - 1;
+    return EXEC_PRECONDITION_BACKOFF_MS[Math.min(Math.max(attempt - 1, 0), last)];
+}
 /**
  * Base64 chunk size for {@link Sandbox.importFiles}'s temp-file write. Bounded by
  * the same Fly exec payload limit as `WRITE_CHUNK_BASE64` (~15 KB, silently
@@ -1769,26 +1839,126 @@ class FlyioSandboxProvider {
     // ---------------------------------------------------------------------------
     /**
      * Issues one Fly exec call against a Machine.
+     *
+     * A transient 412 (see {@link isTransientExecPrecondition}) is retried here, at
+     * the transport, because everywhere it surfaces — a boot, the production
+     * artifact copy, the boot script, a health probe — the caller's alternative is
+     * to throw the whole operation away and start over, which costs tens of seconds
+     * or minutes. Before the retry the Machine's state is re-read (and waited on
+     * when it is not yet `started`), so a Machine that really is still coming up is
+     * not hammered.
+     *
+     * A 412 that is NOT that condition is never retried, and its message carries
+     * the Machine's current state — the caller is told which state the exec was
+     * refused in rather than a bare `failed_precondition`.
+     *
      * @param app - The Fly app name.
      * @param machineId - The Fly Machine id.
      * @returns A transport callback for {@link execCommand}.
      */
     rawExec(app, machineId) {
         return async (command, timeoutSeconds) => {
-            const response = await this.client.request(`/apps/${app}/machines/${machineId}/exec`, {
-                method: 'POST',
-                body: { command, timeout: timeoutSeconds },
-                // The API's own timeout bounds the command; allow the HTTP request a
-                // little longer so the transport never fires first.
-                timeoutMs: (timeoutSeconds + 15) * 1000,
-                // A 404 on a Machine we created and are actively driving (e.g. the
-                // exec burst importFiles issues) is Fly API inconsistency under load,
-                // not a real absence — retry it alongside the default 429/5xx.
-                retryStatuses: [404],
-            });
-            return response ?? {};
+            const deadline = Date.now() + EXEC_PRECONDITION_BUDGET_MS;
+            for (let attempt = 1;; attempt++) {
+                try {
+                    const response = await this.client.request(`/apps/${app}/machines/${machineId}/exec`, {
+                        method: 'POST',
+                        body: { command, timeout: timeoutSeconds },
+                        // The API's own timeout bounds the command; allow the HTTP request a
+                        // little longer so the transport never fires first.
+                        timeoutMs: (timeoutSeconds + 15) * 1000,
+                        // A 404 on a Machine we created and are actively driving (e.g. the
+                        // exec burst importFiles issues) is Fly API inconsistency under load,
+                        // not a real absence — retry it alongside the default 429/5xx.
+                        retryStatuses: [404],
+                    });
+                    return response ?? {};
+                }
+                catch (error) {
+                    if (!isTransientExecPrecondition(error)) {
+                        throw await this.withMachineState(error, app, machineId);
+                    }
+                    if (attempt >= EXEC_PRECONDITION_ATTEMPTS || Date.now() >= deadline) {
+                        logger.warn('Fly kept dropping the exec connection; giving up on this exec', {
+                            app,
+                            machineId,
+                            attempt,
+                        });
+                        throw await this.withMachineState(error, app, machineId);
+                    }
+                    logger.warn('Fly dropped the exec connection (412 exec request failed); retrying', {
+                        app,
+                        machineId,
+                        attempt,
+                        attempts: EXEC_PRECONDITION_ATTEMPTS,
+                    });
+                    // The Machine is up (that is what makes this 412 the transient one), but
+                    // re-read it anyway: `stop`/`suspend` can land between attempts, and
+                    // retrying an exec against a Machine that is not running would repeat a
+                    // 412 that waiting — not repeating — is what answers.
+                    await this.waitForExecReady(app, machineId);
+                    await this.client.sleep(execPreconditionBackoffMs(attempt));
+                }
+            }
         };
     }
+    /**
+     * Confirms the Machine is in a state an exec can run in, waiting for `started`
+     * when it is still on its way up.
+     *
+     * A wait that does not reach `started` is logged and the exec is retried
+     * anyway: the retry is the recovery this path exists for, and turning a
+     * recoverable dropped connection into a start-timeout error would hide the
+     * condition the caller's error message is supposed to name.
+     *
+     * @param app - The Fly app name.
+     * @param machineId - The Fly Machine id.
+     */
+    async waitForExecReady(app, machineId) {
+        const machine = await this.client
+            .request(`/apps/${app}/machines/${machineId}`, { nullOn: [404] })
+            .catch(() => null);
+        if (machine?.state === 'started')
+            return;
+        try {
+            await this.waitForStarted(app, machineId);
+        }
+        catch (error) {
+            logger.warn('The Machine did not report started after a refused exec; retrying the exec', {
+                app,
+                machineId,
+                state: machine?.state ?? 'unknown',
+                error,
+            });
+        }
+    }
+    /**
+     * Names the Machine's state on an exec 412, so the caller is told WHICH state
+     * the exec was refused in. Fly's own body says only `failed_precondition`.
+     *
+     * A state that cannot be read is reported as such rather than guessed, and any
+     * non-412 error (or an error that already carries the state) is returned
+     * untouched — this runs on the failure path, where a diagnostic must never
+     * replace the failure it is describing.
+     *
+     * @param error - The failure to annotate.
+     * @param app - The Fly app name.
+     * @param machineId - The Fly Machine id.
+     * @returns The original error, or a 412 carrying the Machine's state.
+     */
+    async withMachineState(error, app, machineId) {
+        if (!(error instanceof Error))
+            return new Error(String(error));
+        if (!(error instanceof FlyApiError) || error.status !== 412)
+            return error;
+        if (/machine state:/i.test(error.message))
+            return error;
+        const machine = await this.client
+            .request(`/apps/${app}/machines/${machineId}`, { nullOn: [404] })
+            .catch(() => null);
+        const state = machine?.state ?? 'unreadable';
+        return new FlyApiError(error.method, error.path, error.status, `${error.body} (machine ${machineId} state: ${state})`, { cause: error });
+    }
     /**
      * Starts (or resumes) a Machine, tolerating the case where it is already up.
      *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@molecule/api-code-sandbox-flyio",
-  "version": "1.2.7",
+  "version": "1.2.9",
   "description": "Fly.io Machines (Firecracker microVM) code sandbox provider",
   "type": "module",
   "main": "dist/index.js",