@molecule/api-code-sandbox-flyio 1.2.6 → 1.2.8

package/dist/provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAWV,eAAe,EAGhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAwBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAIV,WAAW,EAUZ,MAAM,YAAY,CAAA;AA+2EnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAWV,eAAe,EAGhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAwBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAIV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAsgFnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}

package/dist/provider.js

@@ -141,17 +141,23 @@ const IMPORT_EXTRACT_TIMEOUT_MS = 55_000;
  * sub-APIs asynchronously, so `POST /apps` can return before
  * `POST /apps/{app}/machines` sees the app — which 404s "app not found" for a
  * few seconds. Observed live during boot on 2026-08-08 (surfaced to the user as
- * "Failed to start sandbox"). Six attempts span the client's 1+2+4+8+16 s
- * backoff ≈ 31 s, comfortably past the observed window.
+ * "Failed to start sandbox"). Six attempts span the client's 1+2+4+8+10 s
+ * backoff ≈ 25 s (the fifth step is clamped at the client's 10 s cap),
+ * comfortably past the observed window.
  */
 const MACHINE_CREATE_ATTEMPTS = 6;
 /**
  * Retrying a `POST /machines` on 404 is SAFE precisely because "app not found"
  * means no Machine was created — unlike a 5xx/timeout, where one might have
- * been. So only 404 is added here, on top of the client's default retryable
- * set; the retry cannot double-create.
+ * been. 412 is safe for the same reason: "volume already claimed by machine
+ * <id>" refuses the create. It is what the create answers in the seconds after
+ * {@link FlyioSandboxProvider.destroyStaleMachines} removed the Machine that
+ * held the volume — Fly releases the claim asynchronously, and an immediate
+ * create lost a production deploy to it (2026-10-14; the identical retry a
+ * minute later succeeded). Only these two are added on top of the client's
+ * default retryable set; neither retry can double-create.
  */
-const MACHINE_CREATE_RETRY_STATUSES = [404];
+const MACHINE_CREATE_RETRY_STATUSES = [404, 412];
 /**
  * Base64 chunk size for {@link Sandbox.importFiles}'s temp-file write. Bounded by
  * the same Fly exec payload limit as `WRITE_CHUNK_BASE64` (~15 KB, silently
@@ -365,8 +371,12 @@ class FlyioSandboxProvider {
      *   is what the throwaway `verifyEgress()` app wants: it never connects to
      *   anything but the probe targets, and allocating for an app that is deleted
      *   seconds later would leak one address per probe on every target app.
-     * @returns Target app name → allocated private address, for recording in the
-     *   Machine's metadata. Empty when no private services are declared.
+     * @returns `routes`: target app name → allocated private address, for
+     *   recording in the Machine's metadata (empty when no private services are
+     *   declared); `allocated`: the subset of `routes` this call created, which
+     *   the caller releases when the boot fails and the app is NOT discarded
+     *   (an app that pre-existed is kept, and with it the addresses it already
+     *   held — only this call's own allocations are safe to take back).
      */
     async ensureApp(app, options = {}) {
         const existing = await this.client.request(`/apps/${app}`, { nullOn: [404] });
@@ -399,9 +409,9 @@ class FlyioSandboxProvider {
                 await this.assignSharedIpv4(app);
             await this.applyEgressPolicy(app);
             if (options.privateRoutes === false)
-                return { routes: {}, created: createdHere };
+                return { routes: {}, allocated: {}, created: createdHere };
             return {
-                routes: await this.ensurePrivateRoutes(app, Boolean(existing)),
+                ...(await this.ensurePrivateRoutes(app, Boolean(existing))),
                 created: createdHere,
             };
         }
@@ -450,25 +460,52 @@ class FlyioSandboxProvider {
      * allocated. Fly's own `IPAssignment` carries no network, so a listing on the
      * target app cannot answer "does this project already have a route" — the
      * Machine metadata this provider wrote is the only record.
+     *
+     * A failure part-way down the list RELEASES the addresses this call had
+     * already allocated before rethrowing: the caller's own failure path discards
+     * the app, which is a different resource on a different app than the addresses
+     * (see the allocation loop).
      * @param app - The Fly app name (which is also its 6PN network name).
      * @param appExisted - Whether the app was already there, in which case its
      *   Machines may carry addresses allocated by an earlier boot.
-     * @returns Target app name → private address.
+     * @returns `routes`: target app name → private address (what the Machine's
+     *   metadata records); `allocated`: the subset of `routes` THIS call created —
+     *   the only addresses safe to release when the boot fails after allocation,
+     *   since the recorded ones belong to Machines an earlier boot may still run.
      */
     async ensurePrivateRoutes(app, appExisted) {
         const services = this.privateServices();
         const network = this.networkFor(app);
         if (!services?.length || !network)
-            return {};
+            return { routes: {}, allocated: {} };
         const routes = appExisted ? await this.recordedPrivateRoutes(app) : {};
-        for (const service of services) {
-            if (routes[service.app])
-                continue;
-            const ip = await this.allocatePrivateAddress(service, network);
-            if (ip)
-                routes[service.app] = ip;
+        // The addresses THIS call allocated, tracked apart from `routes` so a
+        // failure part-way down the next list can release exactly them. The caller's
+        // failure path can only discard the APP (`discardCreatedApp(app, {})`), and
+        // these addresses live on OTHER apps — so without this, an address already
+        // allocated for an earlier service stays behind on its target app, inside a
+        // 6PN whose app is gone, and Fly's listing does not report which network an
+        // address belongs to: nothing could ever find it there to release it.
+        const allocated = {};
+        try {
+            for (const service of services) {
+                if (routes[service.app])
+                    continue;
+                const ip = await this.allocatePrivateAddress(service, network);
+                if (ip) {
+                    routes[service.app] = ip;
+                    allocated[service.app] = ip;
+                }
+            }
+        }
+        catch (error) {
+            // Only what this call allocated. An address recorded by an EARLIER boot
+            // (`appExisted`) is left alone: the app survived this failure and may
+            // still hold a Machine reaching through that route.
+            await this.releasePrivateRoutes(allocated);
+            throw error;
         }
-        return routes;
+        return { routes, allocated };
     }
     /**
      * Reads the private addresses an earlier boot recorded on this app's Machines.
@@ -927,13 +964,26 @@ class FlyioSandboxProvider {
         // leave in a Machine config.
         assertPrivateRoutesForEnv({ ...config.env, ...config.selfDeliveredEnv }, this.privateServices());
         const app = this.resolveApp(config.projectId);
-        const { routes: privateRoutes, created } = await this.ensureApp(app);
+        const { routes: privateRoutes, allocated, created } = await this.ensureApp(app);
         try {
             return await this.createInApp(app, privateRoutes, config, manifest);
         }
         catch (error) {
-            if (created && this.appPerProject())
+            if (created && this.appPerProject()) {
+                // The whole app goes, addresses included — discardCreatedApp releases
+                // every route it was handed (a created app holds only this call's).
                 await this.discardCreatedApp(app, privateRoutes);
+            }
+            else {
+                // The app SURVIVES this failure (it pre-existed, or it is the shared
+                // app), so the addresses this call allocated would otherwise stay
+                // behind on their target apps with nothing recording them: the
+                // Machine that would have carried the metadata was never created, and
+                // Fly's listing cannot say which network an address belongs to. Take
+                // back exactly this call's own allocations — never the routes an
+                // earlier boot recorded, whose Machines may still be running.
+                await this.releasePrivateRoutes(allocated);
+            }
             throw error;
         }
     }
@@ -1038,7 +1088,35 @@ class FlyioSandboxProvider {
         // `412 failed_precondition: machine not running` until the Machine is up.
         // Wait for `started` here so `create` always hands back a live sandbox, as
         // the JSDoc promises — the template path already does this via restore.
-        await this.waitForStarted(app, machine.id);
+        try {
+            await this.waitForStarted(app, machine.id);
+        }
+        catch (error) {
+            // The boot failed after the Machine existed. This call created it and
+            // handed it to NOBODY — the caller got a rejection, not a handle — so
+            // leaving it behind strands state nothing else ever cleans up: a Machine
+            // that reached `started` bills compute until it is reaped, and one that
+            // did not keeps the app's single-attach volume claimed (the next
+            // create's stale-Machine sweep only runs if this project is retried).
+            // The created-app failure path below cannot do this either: it deletes
+            // the APP, and `discardCreatedApp` deliberately has no Machine-delete
+            // step — an app delete that runs under a live Machine either fails or
+            // hides the Machine. Destroy the Machine first (the same order
+            // `restoreTemplate`'s failure path and `destroy()` use), then rethrow
+            // the boot failure. Best-effort: the boot failure is what the caller
+            // must see; a failed cleanup is logged like every other cleanup here.
+            try {
+                await this.client.request(`/apps/${app}/machines/${machine.id}?force=true`, {
+                    method: 'DELETE',
+                    nullOn: [404],
+                });
+            }
+            catch (cleanupError) {
+                logger.warn('Failed to destroy the Fly Machine whose boot did not reach started — it will keep ' +
+                    'billing (and holding its volume) until it is reaped', { app, machineId: machine.id, error: cleanupError });
+            }
+            throw error;
+        }
         return this.buildSandbox(app, machine.id, 'running');
     }
     /**
@@ -1097,6 +1175,14 @@ class FlyioSandboxProvider {
      * In per-project mode the Machines live in many apps, so this pages through
      * `GET /v1/orgs/{org}/machines` and keeps the ones carrying this provider's
      * `managed` metadata. In shared-app mode it lists that app directly.
+     *
+     * Only the APP-scoped 404 is an absence (`nullOn: [404]` on
+     * `/apps/{app}/machines` — the app is gone, so its Machines are too). A 404
+     * from the ORG-scoped listing is not: an org with no Machines answers 200
+     * with an empty list, so a 404 there means the org is unknown to the token —
+     * a failed query. Swallowing it as `[]` would tell a reaper the fleet is
+     * clean when nothing had been looked at, the same conflation
+     * `appsToEnumerate()` refuses for the org's app listing.
      * @param _userId - Reserved for future per-user filtering; currently ignored,
      *   matching the Docker provider.
      * @returns Every managed sandbox.
@@ -1118,7 +1204,7 @@ class FlyioSandboxProvider {
             const query = new URLSearchParams({ limit: String(LIST_PAGE_SIZE) });
             if (cursor)
                 query.set('cursor', cursor);
-            const response = await this.client.request(`/orgs/${this.orgSlug()}/machines?${query.toString()}`, { nullOn: [404] });
+            const response = await this.client.request(`/orgs/${this.orgSlug()}/machines?${query.toString()}`);
             for (const machine of response?.machines ?? []) {
                 if (machine.config?.metadata?.[managedKey] !== 'true')
                     continue;
@@ -1964,8 +2050,17 @@ class FlyioSandboxProvider {
                 const store = provider.templateContext().store;
                 if (store) {
                     const key = `${store.prefix}/.imports/${machineId}-${Date.now().toString(36)}${Math.floor(Math.random() * 1e9).toString(36)}.tar.gz`;
+                    // Whether the uploaded object may still exist. Every exit from this
+                    // block removes it — not just the success one: the upload is the
+                    // SECOND write of this import, and a throw after it (a failing
+                    // `presignGet`, an exec that times out or loses the connection) used to
+                    // fall straight into the chunked fallback below with a multi-MB object
+                    // left in the bucket. The presigned URL expires on its own; the OBJECT
+                    // does not, so "it expires anyway" was only ever true of the URL.
+                    let uploaded = false;
                     try {
                         const putUrl = await store.presignPut(key, IMPORT_PRESIGN_EXPIRY_SECONDS);
+                        uploaded = true;
                         const putRes = await fetch(putUrl, { method: 'PUT', body: gz });
                         // The PUT's body is never read (a presigned store answers an empty
                         // 2xx or a short error document): release it on EVERY path, or the
@@ -1983,25 +2078,47 @@ class FlyioSandboxProvider {
                             'set -e',
                             `mkdir -p ${quotedPath}`,
                             'archive=$(mktemp /tmp/mol-import-XXXXXX.tar.gz)',
-                            `curl --fail-with-body -sS --retry 3 --retry-connrefused -o "$archive" ${shellQuote(getUrl)}`,
-                            // The archive is removed whether the extract SUCCEEDS or fails:
-                            // `&& rm` after tar (under `set -e`) left every failed extract's
-                            // download behind in the sandbox's /tmp until the Machine died.
-                            // The `|| { }` guard keeps `set -e` from aborting before the
-                            // cleanup and re-raises the real exit code.
+                            // The archive is removed whether the DOWNLOAD or the extract
+                            // SUCCEEDS or fails: under `set -e`, a bare `curl`/`tar` line
+                            // aborted the script before any cleanup — a failed curl (an
+                            // expired presign, a reset mid-body) left its partial download
+                            // in the sandbox's /tmp until the Machine died, exactly like
+                            // the failed extracts `&& rm` used to leave. The `|| { }` guard
+                            // keeps `set -e` from aborting before the cleanup and re-raises
+                            // the real exit code.
+                            `curl --fail-with-body -sS --retry 3 --retry-connrefused -o "$archive" ${shellQuote(getUrl)} || { rc=$?; rm -f "$archive"; exit $rc; }`,
                             `tar -C ${quotedPath} --no-same-owner --no-same-permissions -xzf "$archive" || { rc=$?; rm -f "$archive"; exit $rc; }`,
                             'rm -f "$archive"',
                         ].join('\n'), { timeout: IMPORT_EXTRACT_TIMEOUT_MS });
-                        // Best-effort: the temp object expires anyway, but delete it promptly.
+                        // Best-effort, but a failure is not silent: only the presigned URL
+                        // expires on its own — the OBJECT does not (see the comment above
+                        // the `uploaded` flag), so a failed removal here strands the
+                        // archive in the bucket exactly like the failure path below does,
+                        // and is logged at the same level so it is findable. Cleared
+                        // first: past this point the removal is under way, so the catch
+                        // below must not queue a second one for the same key (a non-zero
+                        // extract exit still throws from `failIfError`).
+                        uploaded = false;
                         store.remove([key]).catch((error) => {
-                            logger.debug('importFiles temp object cleanup failed', { key, error });
+                            logger.warn('importFiles temp object cleanup failed — it stays in the object store', {
+                                key,
+                                error,
+                            });
                         });
                         failIfError(result, 'download');
                         return;
                     }
                     catch (error) {
                         // The store path is unavailable or failed — fall through to the
-                        // chunked-exec transport rather than failing the whole import.
+                        // chunked-exec transport rather than failing the whole import. Any
+                        // object this attempt uploaded goes first: the fallback re-imports
+                        // the same archive under a DIFFERENT key on the workspace volume, so
+                        // nothing else would ever remove this one.
+                        if (uploaded) {
+                            await store.remove([key]).catch((cleanupError) => {
+                                logger.warn('Could not remove the uploaded import archive after a failed import — it stays in the object store', { key, error: cleanupError });
+                            });
+                        }
                         logger.warn('importFiles via object store failed; falling back to chunked exec', {
                             error,
                         });
@@ -2018,9 +2135,29 @@ class FlyioSandboxProvider {
                 const base64 = gz.toString('base64');
                 const tmp = `${WORKSPACE_PATH}/.mol-import-${machineId}-${Date.now().toString(36)}${Math.floor(Math.random() * 1e9).toString(36)}.tgz`;
                 const quotedTmp = shellQuote(tmp);
-                failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(0, IMPORT_WRITE_CHUNK_BASE64))} | base64 -d > ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');
-                for (let i = IMPORT_WRITE_CHUNK_BASE64; i < base64.length; i += IMPORT_WRITE_CHUNK_BASE64) {
-                    failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(i, i + IMPORT_WRITE_CHUNK_BASE64))} | base64 -d >> ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');
+                // A failed WRITE step leaves a partial archive behind on the durable
+                // workspace volume — the chunked transport is the fallback for a
+                // multi-MB archive, so a mid-write failure (a 429, a full disk) used to
+                // strand every byte written so far until the Machine died. Removed
+                // best-effort on any write failure; the extract step below cleans up
+                // after ITSELF (inside the sandbox), so it needs no outer cleanup.
+                const discardTempArchive = async () => {
+                    try {
+                        await this.exec(`rm -f ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS });
+                    }
+                    catch (cleanupError) {
+                        logger.warn('Could not remove the temp import archive after a failed import — it stays on the workspace volume', { tmp, error: cleanupError });
+                    }
+                };
+                try {
+                    failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(0, IMPORT_WRITE_CHUNK_BASE64))} | base64 -d > ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');
+                    for (let i = IMPORT_WRITE_CHUNK_BASE64; i < base64.length; i += IMPORT_WRITE_CHUNK_BASE64) {
+                        failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(i, i + IMPORT_WRITE_CHUNK_BASE64))} | base64 -d >> ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');
+                    }
+                }
+                catch (error) {
+                    await discardTempArchive();
+                    throw error;
                 }
                 failIfError(await this.exec(
                 // The temp archive is removed whether the extract SUCCEEDS or fails:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@molecule/api-code-sandbox-flyio",
-  "version": "1.2.6",
+  "version": "1.2.8",
   "description": "Fly.io Machines (Firecracker microVM) code sandbox provider",
   "type": "module",
   "main": "dist/index.js",