humanish 0.144.0 → 0.146.0

package/dist/remote/launch.js

@@ -15,14 +15,17 @@ import { pointLatestRunAt } from "../run/run.js";
 import { appendSandboxOwner, SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
 import { loadE2BBaseModule } from "../substrates/e2b/base-module.js";
 import { e2bConnection } from "../substrates/e2b/connection.js";
+import { E2B_SYSTEM_CA_BUNDLE, holdKeysInEgress } from "../substrates/e2b/egress-keys.js";
 import { acquireE2BShellSandbox } from "../substrates/e2b/sandbox.js";
 import { loadModalShellModule } from "../substrates/modal/sandbox.js";
 import { serveObserverProxy } from "./observer-proxy.js";
-import { installHumanish, listRemoteRun, listSettledEvidence, readRemoteBytes, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
-/** How often the launcher reads the runner's run while it follows it. */
-const POLL_MS = 5_000;
-/** Reads in a row the runner may leave unanswered before the launcher gives it up. */
-const UNANSWERED_LIMIT = 24;
+import { listRemoteRun, listSettledEvidence, readProject, readRemoteBytes, readRemoteFiles, RUNNER_OBSERVER_PORT, runnerOf, setUpRunner, startRemoteRun, stopRemoteRun, waitForRemoteRun, } from "./runner.js";
+/** The longest one read of the runner's run waits there for a change. */
+const WAIT_MS = 5_000;
+/** How long the launcher waits after a read the runner did not answer before it reads again. */
+const RETRY_MS = 5_000;
+/** How long the runner may leave reads unanswered before the launcher gives it up. */
+const UNANSWERED_LIMIT_MS = 120_000;
 /** How long the run has to write its status record after it is started. */
 const START_DEADLINE_MS = 3 * 60_000;
 /** How often the launcher copies the run's settled evidence while it goes on. */
@@ -51,21 +54,33 @@ export async function launchRemoteRun(launch) {
     // Installed before the first sandbox exists, so a signal never ends this process while one is
     // being created without a cleanup.
     const signals = watchSignals();
-    let tarballDir;
     let proxy;
+    let tarball;
     try {
-        const tarball = await packThisHumanish();
-        tarballDir = tarball.dir;
+        // Read and checked before any sandbox exists, so a refused file costs nothing.
+        const project = await readProject(cwd, remote.uploads, remote.uploadDirectories);
+        // Packed while the runner is created and Node installs in it.
+        tarball = packThisHumanish();
+        tarball.catch(() => undefined);
         if (signals.requested !== undefined)
             return { runId, exitCode: signals.exitCode, error: "stopped before the runner was created" };
         const where = remote.provider === "modal" ? "a Modal sandbox" : "an E2B sandbox";
         say(`Running this study on ${where} as run ${runId}; creating the runner.`);
-        const runner = runnerOf((await acquireRunner(launch, paths)).sandbox);
+        const keys = runnerKeys(remote, launch.env);
+        const runner = runnerOf((await acquireRunner(launch, paths, keys.network)).sandbox);
         if (signals.requested !== undefined)
             return await endRun(launch, runId, {
                 exitCode: signals.exitCode,
                 error: "stopped before the run started",
             });
+        // Setting up can take minutes; a signal stops waiting for it, and the reclaim below stops the
+        // runner, and the setup with it. A setup left behind that way fails once its runner is gone.
+        const setup = setUpRunner(runner, {
+            tarball,
+            dependencies: [...(await ownDependencies()), await peerSpec(remote.provider)],
+            project,
+        });
+        setup.catch(() => undefined);
         await writeRemoteRecord(paths, {
             runId,
             provider: remote.provider,
@@ -73,29 +88,18 @@ export async function launchRemoteRun(launch) {
         });
         // Bound before the run starts, so a taken --port fails before any participant exists.
         proxy = await serveObserverProxy(await runner.sandbox.portAccess(RUNNER_OBSERVER_PORT), launch.port);
-        // Installing can take minutes; a signal stops waiting for it, and the reclaim below stops the
-        // runner, and the install with it.
-        const installing = installHumanish(runner, tarball.bytes, [await peerSpec(remote.provider)]);
-        // An install left behind by a signal fails once its runner is gone; that is expected.
-        installing.catch(() => undefined);
-        const version = await Promise.race([installing, signals.received.then(() => "")]);
-        if (signals.requested !== undefined)
-            return await endRun(launch, runId, {
-                exitCode: signals.exitCode,
-                error: "stopped before the run started",
-            });
-        say(`Installed humanish ${version} in the runner; uploading the study.`);
-        await uploadProject(runner, cwd, remote.uploads, remote.uploadDirectories);
+        const version = await Promise.race([setup, signals.received.then(() => "")]);
         if (signals.requested !== undefined)
             return await endRun(launch, runId, {
                 exitCode: signals.exitCode,
                 error: "stopped before the run started",
             });
+        say(`Installed humanish ${version} in the runner and uploaded the study; starting its run.`);
         await startRemoteRun(runner, {
             study: launch.study,
             runId,
             ownerKey,
-            env: forwardedEnv(remote, launch.env),
+            env: keys.env,
         });
         return await followRemoteRun({ launch, runner, paths, runId, signals, proxyUrl: proxy.url });
     }
@@ -108,11 +112,11 @@ export async function launchRemoteRun(launch) {
     finally {
         signals.dispose();
         await proxy?.close();
-        if (tarballDir !== undefined)
-            await rm(tarballDir, { recursive: true, force: true });
+        // A pack still going removes its directory when it ends; this process waits for that.
+        await tarball?.catch(() => undefined);
     }
 }
-async function acquireRunner(launch, paths) {
+async function acquireRunner(launch, paths, network) {
     const { provider, runnerMs } = launch.remote;
     const module = provider === "modal" ? await loadModalShellModule(launch.env) : await loadE2BBaseModule();
     // A Modal module takes its connection from launch.env and ignores these two.
@@ -125,7 +129,7 @@ async function acquireRunner(launch, paths) {
             ...account,
             timeoutMs: runnerMs,
             metadata: { participantId: "runner" },
-            network: { allowPublicTraffic: false },
+            network,
         },
         receipt: { root: paths, participantId: "runner" },
     });
@@ -145,11 +149,25 @@ export async function followRemoteRun(args) {
     let syncAt = Date.now() + SYNC_EVERY_MS;
     let state;
     let stopAt;
-    let unanswered = 0;
+    let answeredAt = Date.now();
+    // The run was started just before, so it is taken as alive until a read says otherwise.
+    let alive = true;
     const startBy = Date.now() + START_DEADLINE_MS;
     for (;;) {
-        const run = await readRemoteRun(runner, runId).catch(() => undefined);
-        unanswered = run === undefined ? unanswered + 1 : 0;
+        const read = await readRun(runner, runId, { state, alive }, signals);
+        // A signal ends the read, and is handled below without one.
+        const run = read === "signal" ? undefined : read;
+        if (read === undefined) {
+            if (Date.now() - answeredAt < UNANSWERED_LIMIT_MS)
+                await Promise.race([
+                    new Promise((resolve) => setTimeout(resolve, RETRY_MS)),
+                    signals.requested === undefined ? signals.received : new Promise(() => undefined),
+                ]);
+        }
+        else if (run !== undefined) {
+            answeredAt = Date.now();
+            alive = run.alive;
+        }
         if (run?.state !== undefined && state === undefined) {
             say(`Run ${runId} is running. Watch it live at ${args.proxyUrl}`);
             say("Closing this terminal leaves the study running; Ctrl-C stops it.");
@@ -195,8 +213,8 @@ export async function followRemoteRun(args) {
                 });
             await stopRemoteRun(runner).catch(() => undefined);
         }
-        const ended = unanswered >= UNANSWERED_LIMIT
-            ? `the runner sandbox stopped answering for ${(UNANSWERED_LIMIT * POLL_MS) / 1000} s`
+        const ended = Date.now() - answeredAt >= UNANSWERED_LIMIT_MS
+            ? `the runner sandbox stopped answering for ${UNANSWERED_LIMIT_MS / 1000} s`
             : run !== undefined && state !== undefined && !run.alive
                 ? stopAt === undefined
                     ? `the run ended without a result: ${run.errTail || "no output"}`
@@ -222,9 +240,18 @@ export async function followRemoteRun(args) {
             // Best effort: what this misses, the copy at the end takes.
             await copySettledEvidence(runner, paths, runId, copied).catch(() => undefined);
         }
-        await new Promise((resolve) => setTimeout(resolve, POLL_MS));
     }
 }
+/**
+ * One read of the runner's run, waiting there for a change. The first signal ends the wait here,
+ * and the read left in the runner ends on its own. Undefined when the runner did not answer.
+ */
+async function readRun(runner, runId, seen, signals) {
+    const read = waitForRemoteRun(runner, runId, seen, WAIT_MS).catch(() => undefined);
+    if (signals.requested !== undefined)
+        return await read;
+    return await Promise.race([read, signals.received.then(() => "signal")]);
+}
 /**
  * Copy every file of the runner's run directory into this machine's run directory, each checked
  * against the digest the runner listed. The sandbox journal keeps this machine's lines (the
@@ -233,24 +260,24 @@ export async function followRemoteRun(args) {
 async function copyRunBack(runner, paths, runId) {
     const files = await listRemoteRun(runner, runId);
     let alreadyHere = 0;
+    const wanted = [];
     for (const file of files) {
         if (file.path === "owner-key")
             continue;
         // Already here with the same bytes, copied while the run went on or by an earlier attempt.
         if (file.path !== SANDBOX_RECEIPTS_ARTIFACT &&
-            (await localDigest(paths, file.path)) === file.sha256) {
+            (await localDigest(paths, file.path)) === file.sha256)
             alreadyHere += 1;
-            continue;
-        }
-        const bytes = await readRemoteFile(runner, runId, file);
-        if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
-            const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
-            const mine = local.status === "read" ? Buffer.from(local.bytes).toString("utf8") : "";
-            await writeContainedOutputFile(paths, file.path, joinJournals(mine, Buffer.from(bytes).toString("utf8")), "utf8");
-            continue;
-        }
-        await writeContainedOutputFile(paths, file.path, bytes);
+        else
+            wanted.push(file);
     }
+    await readRemoteFiles(runner, runId, wanted, async (file, bytes) => {
+        if (file.path !== SANDBOX_RECEIPTS_ARTIFACT)
+            return await writeContainedOutputFile(paths, file.path, bytes);
+        const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
+        const mine = local.status === "read" ? Buffer.from(local.bytes).toString("utf8") : "";
+        await writeContainedOutputFile(paths, file.path, joinJournals(mine, Buffer.from(bytes).toString("utf8")), "utf8");
+    });
     await pointLatestRunAt(paths, runId);
     return { files: files.length, alreadyHere };
 }
@@ -350,13 +377,30 @@ export function watchSignals() {
     };
 }
 /**
- * The names and values the runner's run receives: each of the plan's names that is set here, and of
- * names the run reads only the first of, only the first one set.
+ * The keys the runner's run receives: each of the plan's names that is set here, and of names the
+ * run reads only the first of, only the first one set. On E2B, each key that travels as a header
+ * to a fixed host stays in an E2B header transform, and the run gets a placeholder and the system
+ * CA bundle that holds E2B's proxy CA, which Node does not read by default. Modal has no such
+ * transform, so a Modal runner receives every key as an environment variable.
  */
-export function forwardedEnv(remote, env) {
+export function runnerKeys(remote, env) {
     const set = (name) => (env[name]?.trim() ?? "") !== "";
     const unread = new Set(remote.alternatives.flatMap((names) => names.filter(set).slice(1)));
-    return Object.fromEntries(remote.envNames.flatMap((name) => (set(name) && !unread.has(name) ? [[name, env[name]]] : [])));
+    const forwarded = Object.fromEntries(remote.envNames.flatMap((name) => (set(name) && !unread.has(name) ? [[name, env[name]]] : [])));
+    const routing = { allowPublicTraffic: false };
+    if (remote.provider === "modal")
+        return { env: forwarded, network: routing, held: [], sent: Object.keys(forwarded).sort() };
+    const held = holdKeysInEgress(forwarded, { domain: forwarded.E2B_DOMAIN, existing: routing });
+    return {
+        env: held.held.length === 0
+            ? held.env
+            : { ...held.env, NODE_EXTRA_CA_CERTS: E2B_SYSTEM_CA_BUNDLE },
+        network: held.network,
+        held: held.held,
+        sent: Object.keys(forwarded)
+            .filter((name) => !held.held.includes(name))
+            .sort(),
+    };
 }
 /** Whether this humanish runs from a built package, which a runner can install. */
 export function runsFromPackage() {
@@ -368,15 +412,30 @@ export function runsFromPackage() {
  * This machine's humanish as an npm tarball, so the runner runs exactly the same code. A run from
  * source has no built package to pack.
  */
-async function packThisHumanish() {
+function packThisHumanish() {
     const here = fileURLToPath(import.meta.url);
     const root = path.resolve(path.dirname(here), "..", "..");
+    // Thrown here, not from the promise, so it refuses the run before any sandbox exists.
     if (!runsFromPackage())
         throw new Error("A remote run installs this humanish in the runner, and a run from source has no built package. Run `pnpm build`, then `node dist/cli.js`, or use an installed humanish.");
+    return pack(root);
+}
+async function pack(root) {
     const dir = await mkdtemp(path.join(tmpdir(), "humanish-pack-"));
-    const { stdout } = await promisify(execFile)("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", dir, root], { maxBuffer: 16 * 1024 * 1024 });
-    const [packed] = JSON.parse(stdout);
-    return { bytes: await readFile(path.join(dir, packed.filename)), dir };
+    try {
+        const { stdout } = await promisify(execFile)("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", dir, root], { maxBuffer: 16 * 1024 * 1024 });
+        const [packed] = JSON.parse(stdout);
+        return await readFile(path.join(dir, packed.filename));
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}
+/** The npm specs this humanish's package.json depends on, as name@range. */
+async function ownDependencies() {
+    const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+    const manifest = JSON.parse(await readFile(path.join(root, "package.json"), "utf8"));
+    return Object.entries(manifest.dependencies ?? {}).map(([name, range]) => `${name}@${range}`);
 }
 /** The provider SDK the runner's participants need, at the version this machine has. */
 async function peerSpec(provider) {

package/dist/remote/runner.d.ts

@@ -13,16 +13,36 @@ export interface Runner {
     readonly shell: Shell;
 }
 export declare function runnerOf(sandbox: E2BDesktopSandbox): Runner;
+/** A study file read here, for the same place in the runner's project. */
+export interface ProjectFile {
+    /** Relative to the project, with `/` separators. */
+    path: string;
+    bytes: Uint8Array;
+}
 /**
- * Install Node and the humanish package `tarball` (this machine's own humanish, packed), with the
- * provider SDKs its participants need at `peers` (name@version). Returns the installed version.
+ * Read each project-relative file in `files`, and every file under each directory in `directories`
+ * that exists here, for upload to the runner. Read before any sandbox exists, so a refusal costs
+ * nothing.
  */
-export declare function installHumanish(runner: Runner, tarball: Uint8Array, peers: readonly string[]): Promise<string>;
+export declare function readProject(cwd: string, files: readonly string[], directories: readonly string[]): Promise<ProjectFile[]>;
+/** What a runner is set up with. */
+export interface RunnerSetup {
+    /** This machine's humanish as an npm tarball; it may still be packing when setup starts. */
+    tarball: Promise<Uint8Array>;
+    /**
+     * The npm specs its package depends on, from its own package.json, and the provider SDK its
+     * participants need, as name@version.
+     */
+    dependencies: readonly string[];
+    project: readonly ProjectFile[];
+}
 /**
- * Upload each project-relative file in `files`, and every file under each directory in
- * `directories` that exists here, to the same place in the runner's project.
+ * Install Node, this humanish's dependencies and this humanish in the runner, and upload the
+ * study's files. Node's download starts at once and the dependencies install as soon as it is
+ * there, while the tarball packs and uploads beside them; the tarball is then unpacked in place,
+ * so no install waits for the pack. Returns the installed version.
  */
-export declare function uploadProject(runner: Runner, cwd: string, files: readonly string[], directories: readonly string[]): Promise<number>;
+export declare function setUpRunner(runner: Runner, setup: RunnerSetup): Promise<string>;
 /** What the runner's run is started with. */
 export interface RemoteRunStart {
     /** The study file, relative to the project. */
@@ -51,19 +71,28 @@ export interface RemoteRunState {
     /** The last lines the run wrote to stderr. */
     errTail: string;
 }
-export declare function readRemoteRun(runner: Runner, runId: string): Promise<RemoteRunState>;
-/** One file of the runner's run directory: its path in the directory and its sha256. */
+/**
+ * How the runner's run is going. The runner checks its run every 0.2 s for up to `waitMs` and
+ * answers as soon as it differs from `seen`: a new state, a result, or its process ending. One
+ * command answers each wait, so the launcher learns of a change within a check of it.
+ */
+export declare function waitForRemoteRun(runner: Runner, runId: string, seen: {
+    state: string | undefined;
+    alive: boolean;
+}, waitMs: number): Promise<RemoteRunState>;
+/** One file of the runner's run directory: its path in the directory, its size and its sha256. */
 export interface RemoteFile {
     path: string;
+    size: number;
     sha256: string;
 }
 /**
- * The regular files of the runner's run directory and their digests. Refuses a directory that holds
- * a link, which a run never writes.
+ * The regular files of the runner's run directory, with their sizes and digests. Refuses a
+ * directory that holds a link, which a run never writes.
  */
 export declare function listRemoteRun(runner: Runner, runId: string): Promise<RemoteFile[]>;
-/** sha256sum lines, `<hex>  ./<path>`, as RemoteFiles with paths relative to the directory. */
-export declare function parseDigests(output: string): RemoteFile[];
+/** sha256sum lines, `<hex>  ./<path>`, with paths relative to the directory. */
+export declare function parseDigests(output: string): Omit<RemoteFile, "size">[];
 /**
  * The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
  * with their sizes. A run writes each of them once, so one that has settled can be copied while
@@ -80,8 +109,12 @@ export declare function parseSizes(output: string): {
 }[];
 /** The bytes of one file of the runner's run directory, as they are now. */
 export declare function readRemoteBytes(runner: Runner, runId: string, file: string): Promise<Uint8Array>;
-/** The bytes of one file of the runner's run directory, checked against its listed digest. */
-export declare function readRemoteFile(runner: Runner, runId: string, file: RemoteFile): Promise<Uint8Array>;
+/**
+ * Read `files` of the runner's run directory and hand each to `take`, checked against its listed
+ * digest. The files go in batches, each joined into one runner file and read back in one request,
+ * so a batch costs two round trips whatever its number of files.
+ */
+export declare function readRemoteFiles(runner: Runner, runId: string, files: readonly RemoteFile[], take: (file: RemoteFile, bytes: Uint8Array) => Promise<void>): Promise<void>;
 /**
  * Ask the runner's run to stop as a signal would, so it reclaims its participants and ends. The
  * `[h]` keeps the pattern from matching the shell that runs it.