@molecule/api-code-sandbox-flyio 1.0.12 → 1.0.14

package/README.md

@@ -3,7 +3,7 @@ AUTO-GENERATED — DO NOT EDIT THIS FILE.
 Generated by `mlcl sync-docs` from the package's src/index.ts JSDoc + mlcl/registry.json.
 Edits here are overwritten on the next commit (molecule's pre-commit hook regenerates).
 To change this document, edit the module-level JSDoc in src/index.ts.
-Generated: 2026-08-05T16:24:32.555Z
+Generated: 2026-08-08T23:10:20.743Z
 -->
 
 # @molecule/api-code-sandbox-flyio
@@ -609,6 +609,13 @@ interface FlyRequestOptions {
    * Used for idempotent deletes and existence checks (`[404]`).
    */
   nullOn?: number[]
+  /**
+   * Extra HTTP statuses to treat as transient (retry) beyond the default
+   * `429`/`5xx`. The exec endpoint passes `[404]`: a machine-not-found for a
+   * Machine we just created and are actively driving is Fly API inconsistency
+   * under exec-burst load, not a real absence, so it is worth repeating.
+   */
+  retryStatuses?: number[]
 }
 ```
 
@@ -1125,6 +1132,10 @@ function encodePrivateRoutes(routes: Record<string, string>): string
 Executes a command on a Machine, choosing the direct or detached strategy
 based on the caller's time budget.
 
+A command too large to pass inline (over {@link FLY_EXEC_MAX_ARG_BYTES}) is
+spilled to a file via chunked writes and run from there, so there is no size
+ceiling the caller must respect.
+
 ```typescript
 function execCommand(
   rawExec: RawExec,
@@ -1751,6 +1762,20 @@ Exit status the capture script uses when `tar` failed outright (as opposed to wa
 const EXIT_TAR_FAILED: 92
 ```
 
+#### `FLY_EXEC_MAX_ARG_BYTES`
+
+Hard cap on the `sh -c <script>` argument SENT to Fly's exec endpoint. This is
+NOT the OS `MAX_ARG_STRLEN` (128 KB) — the Fly Machines exec API rejects a
+command well below that, and does so INVISIBLY: it answers `200` with
+`exit_code: 0` and an `Unhandled rejection: Rejection([PayloadTooLarge…])`
+body, so the command never runs yet reads as success. Measured against the
+live API: ~15.8 KB scripts run, ~23.7 KB are rejected. 15 KB is the safe
+ceiling; callers that move bulk data (writeFile, importFiles) chunk beneath it.
+
+```typescript
+const FLY_EXEC_MAX_ARG_BYTES: 15000
+```
+
 #### `FLY_EXEC_MAX_TIMEOUT_SECONDS`
 
 Fly's hard ceiling on the exec endpoint's `timeout` field, in seconds.

package/dist/api.d.ts

@@ -113,13 +113,8 @@ export interface FlyApiClientOptions {
     /** Injectable sleep, for tests. Defaults to `setTimeout`. */
     sleep?: (ms: number) => Promise<void>;
 }
-/**
- * Thin, retrying JSON client for the Fly Machines API.
- *
- * Deliberately not a generated SDK: this provider touches ~12 endpoints, and a
- * generated client would drag in the whole Machines + Postgres + tokens surface
- * for no benefit.
- */
+/** Test-only: reset the shared pacer so timing tests start from a clean slate. */
+export declare function resetFlyPacerForTests(): void;
 export declare class FlyApiClient {
     private readonly token;
     private readonly baseUrl;

package/dist/api.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAgBH;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,KAAK;IACpC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB;;;;;;;OAOG;gBAED,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAgBhC;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0DAA0D;IAC1D,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAQhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAoBjE;AAED,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAClC,gFAAgF;IAChF,KAAK,EAAE,MAAM,MAAM,GAAG,SAAS,CAAA;IAC/B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAED;;;;;;GAMG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IAErD;;;OAGG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;;;;;;;;;OAUG;IACG,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CA8EnF"}
+{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAgBH;;;;GAIG;AACH,qBAAa,WAAY,SAAQ,KAAK;IACpC,kFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IAErB;;;;;;;OAOG;gBAED,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAgBhC;AAED,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,sCAAsC;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0DAA0D;IAC1D,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAQhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAoBjE;AAED,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAClC,gFAAgF;IAChF,KAAK,EAAE,MAAM,MAAM,GAAG,SAAS,CAAA;IAC/B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,6DAA6D;IAC7D,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAyCD,kFAAkF;AAClF,wBAAgB,qBAAqB,IAAI,IAAI,CAG5C;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA+B;IAErD;;;OAGG;gBACS,OAAO,EAAE,mBAAmB;IASxC;;;;;;;;;;OAUG;IACG,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAiFnF"}

package/dist/api.js

@@ -124,6 +124,42 @@ export function normalizeApiUrl(value) {
  * generated client would drag in the whole Machines + Postgres + tokens surface
  * for no benefit.
  */
+/**
+ * Minimum gap between Fly Machines API requests, PROCESS-WIDE. Fly rate-limits
+ * per account (most tightly the exec endpoint, ~a few req/s), and one molecule
+ * process makes every call with one token — so without pacing, a sandbox boot's
+ * burst of exec calls trips `429`, and retrying under the cap sustains a storm
+ * that LIVELOCKS the boot (observed live 2026-08-09: a workspace stuck
+ * "creating" with continuous per-second 429s that never drained). ~220 ms ≈
+ * 4.5 req/s keeps us under the cap. Tunable via `FLY_API_MIN_REQUEST_GAP_MS`.
+ */
+function minRequestGapMs() {
+    const v = Number(process.env.FLY_API_MIN_REQUEST_GAP_MS);
+    return Number.isFinite(v) && v >= 0 ? v : 220;
+}
+/**
+ * Serializes every request through a shared chain so no two fire closer than
+ * {@link MIN_REQUEST_GAP_MS}. Module-level (not per-client) so concurrent
+ * sandbox boots share ONE pacer and cannot collectively exceed the account cap.
+ * Errors in the chain are swallowed so one failed wait never wedges the queue.
+ */
+let flyPacerTail = Promise.resolve();
+let flyLastRequestAt = 0;
+function paceFlyRequest(sleep) {
+    const mine = flyPacerTail.then(async () => {
+        const wait = Math.max(0, flyLastRequestAt + minRequestGapMs() - Date.now());
+        if (wait > 0)
+            await sleep(wait);
+        flyLastRequestAt = Date.now();
+    });
+    flyPacerTail = mine.catch(() => { });
+    return mine;
+}
+/** Test-only: reset the shared pacer so timing tests start from a clean slate. */
+export function resetFlyPacerForTests() {
+    flyPacerTail = Promise.resolve();
+    flyLastRequestAt = 0;
+}
 export class FlyApiClient {
     token;
     baseUrl;
@@ -164,6 +200,9 @@ export class FlyApiClient {
         }
         let lastError;
         for (let attempt = 1; attempt <= attempts; attempt++) {
+            // Pace EVERY network call (including retries) through the process-wide
+            // limiter so a boot's burst of exec calls never trips Fly's account 429.
+            await paceFlyRequest(this.sleep);
             const controller = new AbortController();
             const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? this.timeoutMs);
             // Initialized to 0 ("no answer received"), which is the value that

package/dist/provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAUV,eAAe,EAEhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAsBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAEV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAs2DnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAUV,eAAe,EAEhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAsBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAEV,WAAW,EAUZ,MAAM,YAAY,CAAA;AA83DnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}

package/dist/provider.js

@@ -82,6 +82,23 @@ const DEFAULT_PROBE_TIMEOUT_MS = 3000;
  * like `npm install`, not for un-tarring.
  */
 const IMPORT_EXTRACT_TIMEOUT_MS = 55_000;
+/**
+ * Attempts for the create's `POST /machines`, retrying the app-propagation 404
+ * (see {@link MACHINE_CREATE_RETRY_STATUSES}). Fly propagates a new app to its
+ * sub-APIs asynchronously, so `POST /apps` can return before
+ * `POST /apps/{app}/machines` sees the app — which 404s "app not found" for a
+ * few seconds. Observed live during boot on 2026-08-08 (surfaced to the user as
+ * "Failed to start sandbox"). Six attempts span the client's 1+2+4+8+16 s
+ * backoff ≈ 31 s, comfortably past the observed window.
+ */
+const MACHINE_CREATE_ATTEMPTS = 6;
+/**
+ * Retrying a `POST /machines` on 404 is SAFE precisely because "app not found"
+ * means no Machine was created — unlike a 5xx/timeout, where one might have
+ * been. So only 404 is added here, on top of the client's default retryable
+ * set; the retry cannot double-create.
+ */
+const MACHINE_CREATE_RETRY_STATUSES = [404];
 /**
  * Base64 chunk size for {@link Sandbox.importFiles}'s temp-file write. Bounded by
  * the same Fly exec payload limit as `WRITE_CHUNK_BASE64` (~15 KB, silently
@@ -397,7 +414,7 @@ class FlyioSandboxProvider {
      */
     async destroyStaleMachines(app) {
         const managedKey = `${this.metadataPrefix}.managed`;
-        let machines = null;
+        let machines;
         try {
             machines = await this.client.request(`/apps/${app}/machines`, {
                 nullOn: [404],
@@ -797,6 +814,11 @@ class FlyioSandboxProvider {
         const machine = await this.client.request(`/apps/${app}/machines`, {
             method: 'POST',
             body: { region: this.region(), config: machineConfig },
+            // A just-created app propagates to the Machines API asynchronously, so this
+            // POST can 404 "app not found" for a few seconds. Retry the 404 (safe: no
+            // Machine is created when the app is not found) across the client's backoff.
+            attempts: MACHINE_CREATE_ATTEMPTS,
+            retryStatuses: MACHINE_CREATE_RETRY_STATUSES,
         });
         if (!machine?.id) {
             throw new Error(t('codeSandbox.flyio.error.createFailed', { app }, { defaultValue: `Fly Machine creation in app "${app}" returned no Machine id.` }));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@molecule/api-code-sandbox-flyio",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "Fly.io Machines (Firecracker microVM) code sandbox provider",
   "type": "module",
   "main": "dist/index.js",