@molecule/api-code-sandbox-flyio 1.0.10 → 1.0.12

package/dist/exec.d.ts

@@ -83,13 +83,15 @@ export declare function buildScript(command: string, opts: ExecOptions | undefin
  * Executes a command on a Machine, choosing the direct or detached strategy
  * based on the caller's time budget.
  *
+ * A command too large to pass inline (over {@link FLY_EXEC_MAX_ARG_BYTES}) is
+ * spilled to a file via chunked writes and run from there, so there is no size
+ * ceiling the caller must respect.
  * @param rawExec - Transport callback issuing one Fly exec call.
  * @param command - The shell command to run.
  * @param opts - Core exec options (cwd, env, timeout in ms).
  * @param defaultCwd - Working directory used when `opts.cwd` is unset.
  * @returns The command's stdout, stderr and exit code.
- * @throws {Error} When the command is too large to launch (see
- *   {@link MAX_SCRIPT_BASE64_BYTES}), or when the transport itself fails.
+ * @throws {Error} When staging a large command or the transport itself fails.
  */
 export declare function execCommand(rawExec: RawExec, command: string, opts: ExecOptions | undefined, defaultCwd: string): Promise<ExecResult>;
 //# sourceMappingURL=exec.d.ts.map

package/dist/exec.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../src/exec.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAGzE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAKjD;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,KAAK,CAAA;AAE9C;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,KAAK,CAAA;AAW5C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,QAAkB,CAAA;AAQxD;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,QAAS,CAAA;AAY5C;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAA;AAEzC,0FAA0F;AAC1F,MAAM,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,cAAc,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,CAAC,CAAA;AAE7F;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,CAiBlE;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,UAAU,EAAE,MAAM,GACjB,MAAM,CAGR;AAaD;;;;;;;;;;;GAWG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,UAAU,CAAC,CAqBrB"}
+{"version":3,"file":"exec.d.ts","sourceRoot":"","sources":["../src/exec.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAGzE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAKjD;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,KAAK,CAAA;AAE9C;;;;GAIG;AACH,eAAO,MAAM,0BAA0B,KAAK,CAAA;AAW5C;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,QAAkB,CAAA;AAQxD;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,QAAS,CAAA;AAY5C;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAA;AAEzC,0FAA0F;AAC1F,MAAM,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,cAAc,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,CAAC,CAAA;AAE7F;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,CAiBlE;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,UAAU,EAAE,MAAM,GACjB,MAAM,CAGR;AAmDD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,OAAO,EAChB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,WAAW,GAAG,SAAS,EAC7B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,UAAU,CAAC,CAqBrB"}

package/dist/exec.js

@@ -50,10 +50,10 @@ const DEFAULT_TIMEOUT_MS = 600_000;
  */
 export const MAX_CAPTURED_OUTPUT_BYTES = 5 * 1024 * 1024;
 /**
- * Cap on the base64-encoded launcher script. `sh -c` receives it as a single
- * argument, which Linux limits to MAX_ARG_STRLEN (128 KB).
+ * Base64 chunk size for spilling a large script to a file, comfortably under
+ * {@link FLY_EXEC_MAX_ARG_BYTES} once the `printf … | base64 -d` wrapper is added.
  */
-const MAX_SCRIPT_BASE64_BYTES = 60_000;
+const SCRIPT_SPILL_CHUNK_BASE64 = 12_000;
 /**
  * Hard cap on the `sh -c <script>` argument SENT to Fly's exec endpoint. This is
  * NOT the OS `MAX_ARG_STRLEN` (128 KB) — the Fly Machines exec API rejects a
@@ -128,31 +128,62 @@ function newRunId() {
         id += Math.floor(Math.random() * 0xffffffff).toString(16);
     return id.slice(0, 16);
 }
+/**
+ * Writes `script` to `path` on the Machine via chunked base64 execs, each under
+ * {@link FLY_EXEC_MAX_ARG_BYTES}.
+ *
+ * Fly's exec API silently drops a single over-limit `printf %s <base64>`, so a
+ * script too large to pass inline (a big `node -e`, a bulk file write) must be
+ * delivered in pieces. This is the transport-level equivalent of `docker cp` for
+ * a command body — callers never see the limit.
+ * @param rawExec - Transport callback issuing one Fly exec call.
+ * @param path - Absolute destination path on the Machine.
+ * @param script - The script body to stage.
+ * @throws {Error} When the work dir or any chunk write fails.
+ */
+async function writeScriptChunked(rawExec, path, script) {
+    const base64 = Buffer.from(script, 'utf8').toString('base64');
+    const ensure = toExecResult(await rawExec(['sh', '-c', `mkdir -p ${WORK_DIR}`], CONTROL_EXEC_TIMEOUT_SECONDS));
+    if (ensure.exitCode !== 0) {
+        throw new Error(`Failed to create the Fly exec work dir: ${ensure.stderr || ensure.stdout}`);
+    }
+    for (let i = 0; i < base64.length; i += SCRIPT_SPILL_CHUNK_BASE64) {
+        const op = i === 0 ? '>' : '>>';
+        const chunk = base64.slice(i, i + SCRIPT_SPILL_CHUNK_BASE64);
+        const written = toExecResult(await rawExec(['sh', '-c', `printf %s ${shellQuote(chunk)} | base64 -d ${op} ${path}`], CONTROL_EXEC_TIMEOUT_SECONDS));
+        if (written.exitCode !== 0) {
+            throw new Error(`Failed to stage a Fly exec script chunk: ${written.stderr || written.stdout}`);
+        }
+    }
+}
 /**
  * Executes a command on a Machine, choosing the direct or detached strategy
  * based on the caller's time budget.
  *
+ * A command too large to pass inline (over {@link FLY_EXEC_MAX_ARG_BYTES}) is
+ * spilled to a file via chunked writes and run from there, so there is no size
+ * ceiling the caller must respect.
  * @param rawExec - Transport callback issuing one Fly exec call.
  * @param command - The shell command to run.
  * @param opts - Core exec options (cwd, env, timeout in ms).
  * @param defaultCwd - Working directory used when `opts.cwd` is unset.
  * @returns The command's stdout, stderr and exit code.
- * @throws {Error} When the command is too large to launch (see
- *   {@link MAX_SCRIPT_BASE64_BYTES}), or when the transport itself fails.
+ * @throws {Error} When staging a large command or the transport itself fails.
  */
 export async function execCommand(rawExec, command, opts, defaultCwd) {
     const budgetMs = opts?.timeout ?? DEFAULT_TIMEOUT_MS;
     const script = buildScript(command, opts, defaultCwd);
-    // A direct command travels as the `sh -c <script>` argument. Fly rejects one
-    // over FLY_EXEC_MAX_ARG_BYTES with a silent `exit_code: 0` (see the constant),
-    // so refuse it here with a message that names the real limit rather than
-    // letting the caller's data vanish. Bulk callers chunk beneath this.
-    if (script.length > FLY_EXEC_MAX_ARG_BYTES) {
-        throw new Error(`Fly exec command is ${script.length} bytes, over the ${FLY_EXEC_MAX_ARG_BYTES}-byte ` +
-            `limit the API silently rejects — chunk the payload smaller.`);
-    }
     if (budgetMs <= DIRECT_EXEC_BUDGET_SECONDS * 1000) {
         const seconds = Math.max(1, Math.min(Math.ceil(budgetMs / 1000), FLY_EXEC_MAX_TIMEOUT_SECONDS));
+        // Fly's exec API silently drops a command over FLY_EXEC_MAX_ARG_BYTES. A
+        // script that big cannot go inline, so spill it to a file (chunked writes,
+        // each under the limit) and run that — matching docker exec, which has no
+        // such ceiling. The caller stays unaware of the transport's limit.
+        if (script.length > FLY_EXEC_MAX_ARG_BYTES) {
+            const path = `${WORK_DIR}/direct-${newRunId()}.sh`;
+            await writeScriptChunked(rawExec, path, script);
+            return toExecResult(await rawExec(['sh', '-c', `sh ${path}; ec=$?; rm -f ${path}; exit $ec`], seconds));
+        }
         return toExecResult(await rawExec(['sh', '-c', script], seconds));
     }
     return await execDetached(rawExec, script, budgetMs);
@@ -172,23 +203,18 @@ export async function execCommand(rawExec, command, opts, defaultCwd) {
  */
 async function execDetached(rawExec, script, budgetMs) {
     const runId = newRunId();
-    const base64 = Buffer.from(script, 'utf8').toString('base64');
-    if (base64.length > MAX_SCRIPT_BASE64_BYTES) {
-        throw new Error(t('codeSandbox.flyio.error.commandTooLarge', { limit: String(MAX_SCRIPT_BASE64_BYTES) }, {
-            defaultValue: `Command is too large to launch on a Fly Machine (encoded ${base64.length} bytes, limit ${MAX_SCRIPT_BASE64_BYTES}). ` +
-                'Write it to a file with writeFile() and exec that file instead.',
-        }));
-    }
     const scriptPath = `${WORK_DIR}/${runId}.sh`;
     const outPath = `${WORK_DIR}/${runId}.out`;
     const errPath = `${WORK_DIR}/${runId}.err`;
     const rcPath = `${WORK_DIR}/${runId}.rc`;
+    // Stage the script via chunked writes (each under the Fly exec arg limit),
+    // then launch it. A single `printf %s <base64>` would exceed the limit for a
+    // large script and Fly would drop it silently. writeScriptChunked also creates
+    // WORK_DIR, so the launcher below only has to background the run.
+    await writeScriptChunked(rawExec, scriptPath, script);
     // The `&` must apply to the nohup line ALONE. In POSIX sh, `a && b &`
-    // backgrounds the whole AND-list, which would race the script write against
-    // the launch, so these are separate statements rather than one chain.
+    // backgrounds the whole AND-list, so these are separate statements.
     const launcher = [
-        `mkdir -p ${WORK_DIR} || exit 1`,
-        `printf %s ${shellQuote(base64)} | base64 -d > ${scriptPath} || exit 1`,
         `nohup sh -c 'sh ${scriptPath} > ${outPath} 2> ${errPath}; printf %s "$?" > ${rcPath}' > /dev/null 2>&1 < /dev/null &`,
         `printf %s ${runId}`,
     ].join('\n');

package/dist/provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAUV,eAAe,EAEhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAsBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAEV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAmzDnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAUV,eAAe,EAEhB,MAAM,4BAA4B,CAAA;AAGnC,OAAO,EAAE,YAAY,EAAgC,MAAM,UAAU,CAAA;AAsBrE,OAAO,EAAmD,KAAK,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG,OAAO,KAAK,EAEV,WAAW,EAUZ,MAAM,YAAY,CAAA;AAs2DnB;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,YAAY,EACrB,KAAK,CAAC,EAAE,WAAW,GAAG,IAAI,GACzB,eAAe,CAEjB"}

package/dist/provider.js

@@ -90,6 +90,12 @@ const IMPORT_EXTRACT_TIMEOUT_MS = 55_000;
  * the gzip pass before transfer is what keeps that count reasonable.
  */
 const IMPORT_WRITE_CHUNK_BASE64 = 12_000;
+/**
+ * Expiry for the presigned URLs {@link Sandbox.importFiles} uses to hand an
+ * archive to the object store and back to the Machine. Short — the upload and
+ * the single download happen back-to-back.
+ */
+const IMPORT_PRESIGN_EXPIRY_SECONDS = 3600;
 /** Extra seconds allowed for the probe exec beyond its connection budget. */
 const PROBE_EXEC_HEADROOM_SECONDS = 10;
 /**
@@ -1495,25 +1501,63 @@ class FlyioSandboxProvider {
                 const parts = [];
                 for await (const chunk of archive)
                     parts.push(Buffer.from(chunk));
-                const base64 = gzipSync(Buffer.concat(parts)).toString('base64');
-                // The temp tar lives on the WORKSPACE VOLUME, not /tmp. The chunked
-                // write and the extract are separate exec calls, and a Machine with
-                // autostop=suspend can cold-start between them — which recreates the
-                // rootfs from the image (wiping /tmp) while the ext4 volume at
-                // /workspace persists. A /tmp temp file vanished mid-import; the volume
-                // one survives.
-                const tmp = `${WORKSPACE_PATH}/.mol-import-${machineId}.tgz`;
-                const quotedTmp = shellQuote(tmp);
+                const gz = gzipSync(Buffer.concat(parts));
                 const quotedPath = shellQuote(path);
                 const failIfError = (result, step) => {
                     if (result.exitCode !== 0) {
                         throw new Error(t('codeSandbox.flyio.error.importFailed', { path, step, error: result.stderr }, {
-                            defaultValue: `Failed to import files into ${path} (${step}): ${result.stderr}`,
+                            defaultValue: `Failed to import files into ${path} (${step}): ${result.stderr || result.stdout}`,
                         }));
                     }
                 };
-                // First chunk truncates the temp file; the rest append. Each chunk is a
-                // multiple of 4 base64 chars so it decodes standalone (see writeFile).
+                // PREFERRED: upload the archive to the object store and have the Machine
+                // PULL it with a single curl. Fly's exec endpoint is rate-limited (~5/s)
+                // and caps each command near 15 KB, so streaming a multi-MB tar through
+                // it as base64 chunks triggers a 429 storm that never finishes. A
+                // presigned download is ONE request — the same pull pattern templates use
+                // (see buildRestoreCommand). Falls back to chunked exec when no store is
+                // configured (e.g. unit tests).
+                const store = provider.templateContext().store;
+                if (store) {
+                    const key = `${store.prefix}/.imports/${machineId}-${Date.now().toString(36)}${Math.floor(Math.random() * 1e9).toString(36)}.tar.gz`;
+                    try {
+                        const putUrl = await store.presignPut(key, IMPORT_PRESIGN_EXPIRY_SECONDS);
+                        const putRes = await fetch(putUrl, { method: 'PUT', body: gz });
+                        if (!putRes.ok) {
+                            throw new Error(`presigned PUT to the object store failed: ${putRes.status}`);
+                        }
+                        const getUrl = await store.presignGet(key, IMPORT_PRESIGN_EXPIRY_SECONDS);
+                        const result = await this.exec([
+                            'set -e',
+                            `mkdir -p ${quotedPath}`,
+                            'archive=$(mktemp /tmp/mol-import-XXXXXX.tar.gz)',
+                            `curl --fail-with-body -sS --retry 3 --retry-connrefused -o "$archive" ${shellQuote(getUrl)}`,
+                            `tar -C ${quotedPath} --no-same-owner --no-same-permissions -xzf "$archive"`,
+                            'rm -f "$archive"',
+                        ].join('\n'), { timeout: IMPORT_EXTRACT_TIMEOUT_MS });
+                        // Best-effort: the temp object expires anyway, but delete it promptly.
+                        store.remove([key]).catch((error) => {
+                            logger.debug('importFiles temp object cleanup failed', { key, error });
+                        });
+                        failIfError(result, 'download');
+                        return;
+                    }
+                    catch (error) {
+                        // The store path is unavailable or failed — fall through to the
+                        // chunked-exec transport rather than failing the whole import.
+                        logger.warn('importFiles via object store failed; falling back to chunked exec', {
+                            error,
+                        });
+                    }
+                }
+                // FALLBACK: chunked base64 over exec. Only used when no object store is
+                // configured. The temp tar lives on the WORKSPACE VOLUME, not /tmp — the
+                // chunked write and the extract are separate exec calls and a Machine
+                // with autostop can cold-start between them, wiping /tmp (rootfs) while
+                // the ext4 volume at /workspace persists.
+                const base64 = gz.toString('base64');
+                const tmp = `${WORKSPACE_PATH}/.mol-import-${machineId}.tgz`;
+                const quotedTmp = shellQuote(tmp);
                 failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(0, IMPORT_WRITE_CHUNK_BASE64))} | base64 -d > ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');
                 for (let i = IMPORT_WRITE_CHUNK_BASE64; i < base64.length; i += IMPORT_WRITE_CHUNK_BASE64) {
                     failIfError(await this.exec(`printf %s ${shellQuote(base64.slice(i, i + IMPORT_WRITE_CHUNK_BASE64))} | base64 -d >> ${quotedTmp}`, { timeout: FILE_OP_TIMEOUT_MS }), 'write');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@molecule/api-code-sandbox-flyio",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "description": "Fly.io Machines (Firecracker microVM) code sandbox provider",
   "type": "module",
   "main": "dist/index.js",