humanish 0.142.0 → 0.144.0

package/dist/remote/launch.js

@@ -2,7 +2,7 @@
 // participants' provider, follow it from here, and copy the finished run back. This machine only
 // plans, launches and watches, so it may sleep or lose its network while the study goes on.
 import { execFile } from "node:child_process";
-import { randomBytes } from "node:crypto";
+import { createHash, randomBytes } from "node:crypto";
 import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
@@ -18,13 +18,19 @@ import { e2bConnection } from "../substrates/e2b/connection.js";
 import { acquireE2BShellSandbox } from "../substrates/e2b/sandbox.js";
 import { loadModalShellModule } from "../substrates/modal/sandbox.js";
 import { serveObserverProxy } from "./observer-proxy.js";
-import { installHumanish, listRemoteRun, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
+import { installHumanish, listRemoteRun, listSettledEvidence, readRemoteBytes, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
 /** How often the launcher reads the runner's run while it follows it. */
 const POLL_MS = 5_000;
 /** Reads in a row the runner may leave unanswered before the launcher gives it up. */
 const UNANSWERED_LIMIT = 24;
 /** How long the run has to write its status record after it is started. */
 const START_DEADLINE_MS = 3 * 60_000;
+/** How often the launcher copies the run's settled evidence while it goes on. */
+const SYNC_EVERY_MS = 30_000;
+/** How long a screenshot or recording must be unchanged before it is copied mid-run. */
+const SETTLED_SECONDS = 10;
+/** The largest file read back to compare digests: a recording's own limit, with room. */
+const LARGEST_FILE_BYTES = 256 * 1024 * 1024;
 /** How long a stopped run has to end and write its result. */
 const STOP_DEADLINE_MS = 2 * 60_000;
 /** Launch, follow and copy back one remote run. */
@@ -50,6 +56,8 @@ export async function launchRemoteRun(launch) {
     try {
         const tarball = await packThisHumanish();
         tarballDir = tarball.dir;
+        if (signals.requested !== undefined)
+            return { runId, exitCode: signals.exitCode, error: "stopped before the runner was created" };
         const where = remote.provider === "modal" ? "a Modal sandbox" : "an E2B sandbox";
         say(`Running this study on ${where} as run ${runId}; creating the runner.`);
         const runner = runnerOf((await acquireRunner(launch, paths)).sandbox);
@@ -65,7 +73,17 @@ export async function launchRemoteRun(launch) {
         });
         // Bound before the run starts, so a taken --port fails before any participant exists.
         proxy = await serveObserverProxy(await runner.sandbox.portAccess(RUNNER_OBSERVER_PORT), launch.port);
-        const version = await installHumanish(runner, tarball.bytes, [await peerSpec(remote.provider)]);
+        // Installing can take minutes; a signal stops waiting for it, and the reclaim below stops the
+        // runner, and the install with it.
+        const installing = installHumanish(runner, tarball.bytes, [await peerSpec(remote.provider)]);
+        // An install left behind by a signal fails once its runner is gone; that is expected.
+        installing.catch(() => undefined);
+        const version = await Promise.race([installing, signals.received.then(() => "")]);
+        if (signals.requested !== undefined)
+            return await endRun(launch, runId, {
+                exitCode: signals.exitCode,
+                error: "stopped before the run started",
+            });
         say(`Installed humanish ${version} in the runner; uploading the study.`);
         await uploadProject(runner, cwd, remote.uploads, remote.uploadDirectories);
         if (signals.requested !== undefined)
@@ -77,16 +95,15 @@ export async function launchRemoteRun(launch) {
             study: launch.study,
             runId,
             ownerKey,
-            env: Object.fromEntries(remote.envNames.flatMap((name) => {
-                const value = launch.env[name];
-                return value === undefined || value === "" ? [] : [[name, value]];
-            })),
+            env: forwardedEnv(remote, launch.env),
         });
         return await followRemoteRun({ launch, runner, paths, runId, signals, proxyUrl: proxy.url });
     }
     catch (error) {
-        await reclaimRunSandboxes(cwd, runId, { env: launch.env }).catch(() => undefined);
-        throw error;
+        const reclaimed = await reclaimRunSandboxes(cwd, runId, { env: launch.env }).then((result) => result.ok, () => false);
+        if (reclaimed)
+            throw error;
+        throw new Error(`${message(error)}; its sandboxes were not confirmed stopped, so run humanish reclaim --run ${runId}`, { cause: error });
     }
     finally {
         signals.dispose();
@@ -117,7 +134,15 @@ async function acquireRunner(launch, paths) {
 export async function followRemoteRun(args) {
     const { launch, runner, paths, runId, signals } = args;
     const { say } = launch;
-    const copy = () => copyRunBack(runner, paths, runId);
+    // Evidence already copied while the run went on, by path, with its size then.
+    const copied = new Map();
+    const copy = async () => {
+        const { files, alreadyHere } = await copyRunBack(runner, paths, runId);
+        if (alreadyHere > 0)
+            say(`${alreadyHere} of its ${files} files were already here.`);
+        return files;
+    };
+    let syncAt = Date.now() + SYNC_EVERY_MS;
     let state;
     let stopAt;
     let unanswered = 0;
@@ -138,8 +163,10 @@ export async function followRemoteRun(args) {
             say("The run ended; copying its evidence back.");
             const copied = await copy().then((files) => ({ files }), (error) => ({ error }));
             // The run reclaimed its participants as it ended; only the runner is left, and it holds the
-            // only whole copy of the evidence until a copy succeeds.
-            if ("error" in copied && stopAt === undefined)
+            // only whole copy of the evidence until a copy succeeds. It is kept only then: a run that
+            // refused before its run directory existed has nothing to keep, and its own result says why.
+            // Its result is not reported as the run's own, since the evidence is not here.
+            if ("error" in copied && stopAt === undefined && run.state !== undefined)
                 return {
                     runId,
                     exitCode: 2,
@@ -151,7 +178,9 @@ export async function followRemoteRun(args) {
                 result: run.result.result,
                 ...("files" in copied
                     ? { files: copied.files }
-                    : { error: `copying the evidence back failed: ${message(copied.error)}` }),
+                    : run.state === undefined
+                        ? {}
+                        : { error: `copying the evidence back failed: ${message(copied.error)}` }),
             });
         }
         if (signals.requested === "detach")
@@ -188,6 +217,11 @@ export async function followRemoteRun(args) {
                 ...("files" in copied ? { files: copied.files } : {}),
             });
         }
+        if (state === "running" && Date.now() >= syncAt) {
+            syncAt = Date.now() + SYNC_EVERY_MS;
+            // Best effort: what this misses, the copy at the end takes.
+            await copySettledEvidence(runner, paths, runId, copied).catch(() => undefined);
+        }
         await new Promise((resolve) => setTimeout(resolve, POLL_MS));
     }
 }
@@ -198,9 +232,16 @@ export async function followRemoteRun(args) {
  */
 async function copyRunBack(runner, paths, runId) {
     const files = await listRemoteRun(runner, runId);
+    let alreadyHere = 0;
     for (const file of files) {
         if (file.path === "owner-key")
             continue;
+        // Already here with the same bytes, copied while the run went on or by an earlier attempt.
+        if (file.path !== SANDBOX_RECEIPTS_ARTIFACT &&
+            (await localDigest(paths, file.path)) === file.sha256) {
+            alreadyHere += 1;
+            continue;
+        }
         const bytes = await readRemoteFile(runner, runId, file);
         if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
             const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
@@ -211,13 +252,32 @@ async function copyRunBack(runner, paths, runId) {
         await writeContainedOutputFile(paths, file.path, bytes);
     }
     await pointLatestRunAt(paths, runId);
-    return files.length;
+    return { files: files.length, alreadyHere };
 }
-/** Two sandbox journals as one: every line of each, in order, one per line. */
+/**
+ * Copy the run's screenshots and recordings that have settled and are not here yet, while the run
+ * goes on, so the copy at its end is only what is left.
+ */
+async function copySettledEvidence(runner, paths, runId, copied) {
+    for (const file of await listSettledEvidence(runner, runId, SETTLED_SECONDS)) {
+        if (copied.get(file.path) === file.size)
+            continue;
+        await writeContainedOutputFile(paths, file.path, await readRemoteBytes(runner, runId, file.path));
+        copied.set(file.path, file.size);
+    }
+}
+async function localDigest(paths, file) {
+    const read = await readContainedRegularFile(paths, file, LARGEST_FILE_BYTES);
+    return read.status === "read" ? createHash("sha256").update(read.bytes).digest("hex") : undefined;
+}
+/**
+ * Two sandbox journals as one: every line of each, in order, once. A copy that runs again adds
+ * nothing it already added.
+ */
 export function joinJournals(...journals) {
-    const lines = journals
-        .flatMap((journal) => journal.split("\n"))
-        .filter((line) => line.trim() !== "");
+    const lines = [
+        ...new Set(journals.flatMap((journal) => journal.split("\n")).filter((line) => line.trim() !== "")),
+    ];
     return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
 }
 /**
@@ -257,17 +317,20 @@ async function writeRemoteRecord(paths, record) {
 export function watchSignals() {
     let requested;
     let exitCode = 130;
-    const interrupt = () => {
-        requested ??= "stop";
-    };
-    const terminate = () => {
-        if (requested === undefined)
-            exitCode = 143;
-        requested ??= "stop";
-    };
-    const detach = () => {
-        requested ??= "detach";
+    let signalled;
+    const received = new Promise((resolve) => {
+        signalled = resolve;
+    });
+    const first = (kind, code) => {
+        if (requested !== undefined)
+            return;
+        requested = kind;
+        exitCode = code;
+        signalled();
     };
+    const interrupt = () => first("stop", 130);
+    const terminate = () => first("stop", 143);
+    const detach = () => first("detach", 129);
     process.on("SIGINT", interrupt);
     process.on("SIGTERM", terminate);
     process.on("SIGHUP", detach);
@@ -278,6 +341,7 @@ export function watchSignals() {
         get exitCode() {
             return exitCode;
         },
+        received,
         dispose() {
             process.off("SIGINT", interrupt);
             process.off("SIGTERM", terminate);
@@ -285,6 +349,21 @@ export function watchSignals() {
         },
     };
 }
+/**
+ * The names and values the runner's run receives: each of the plan's names that is set here, and of
+ * names the run reads only the first of, only the first one set.
+ */
+export function forwardedEnv(remote, env) {
+    const set = (name) => (env[name]?.trim() ?? "") !== "";
+    const unread = new Set(remote.alternatives.flatMap((names) => names.filter(set).slice(1)));
+    return Object.fromEntries(remote.envNames.flatMap((name) => (set(name) && !unread.has(name) ? [[name, env[name]]] : [])));
+}
+/** Whether this humanish runs from a built package, which a runner can install. */
+export function runsFromPackage() {
+    const here = fileURLToPath(import.meta.url);
+    const root = path.resolve(path.dirname(here), "..", "..");
+    return path.relative(root, here).split(path.sep)[0] === "dist";
+}
 /**
  * This machine's humanish as an npm tarball, so the runner runs exactly the same code. A run from
  * source has no built package to pack.
@@ -292,7 +371,7 @@ export function watchSignals() {
 async function packThisHumanish() {
     const here = fileURLToPath(import.meta.url);
     const root = path.resolve(path.dirname(here), "..", "..");
-    if (path.relative(root, here).split(path.sep)[0] !== "dist")
+    if (!runsFromPackage())
         throw new Error("A remote run installs this humanish in the runner, and a run from source has no built package. Run `pnpm build`, then `node dist/cli.js`, or use an installed humanish.");
     const dir = await mkdtemp(path.join(tmpdir(), "humanish-pack-"));
     const { stdout } = await promisify(execFile)("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", dir, root], { maxBuffer: 16 * 1024 * 1024 });

package/dist/remote/plan.d.ts

@@ -11,6 +11,8 @@ export interface RemotePlan {
     readonly uploadDirectories: readonly string[];
     /** The environment names the runner's run may receive, and nothing else. */
     readonly envNames: readonly string[];
+    /** Names of which only the first one set is sent, as the run reads only that one. */
+    readonly alternatives: readonly (readonly string[])[];
 }
 export type RemotePlanResult = {
     ok: true;

package/dist/remote/plan.js

@@ -2,6 +2,7 @@
 // long it must live, which files it must receive and which keys it may hold. A pure function of the
 // study and its plan, so a refusal happens before anything is created.
 import path from "node:path";
+import { analysisKeyNames } from "../keys/key-resolution.js";
 import { keyNamesOf } from "../study/requirements.js";
 import { participantList } from "../study/study-fields.js";
 /** Time the runner needs before the study starts: create, Node and humanish install, uploads. */
@@ -46,6 +47,8 @@ export function planRemoteRun(args) {
         return refuse("it loads a scorer module, which does not run remotely yet.");
     if (plan.rerun !== undefined)
         return refuse("a rerun reads a run on this machine.");
+    if (config.comms !== undefined)
+        return refuse("its email setup reads files and connections on this machine.");
     const study = inside(cwd, args.studyPath);
     if (study === undefined)
         return refuse("the study file is outside the project directory.");
@@ -72,6 +75,7 @@ export function planRemoteRun(args) {
             envNames: [
                 ...new Set([...keyNamesOf(plan), ...PROVIDER_KEYS[runner.provider], ...PLAN_SETTINGS]),
             ].sort(),
+            alternatives: analystKeyNames(plan),
         },
     };
 }
@@ -112,3 +116,11 @@ function servedHere(appUrl) {
         (a === 192 && b === 168) ||
         (a === 169 && b === 254));
 }
+/** The analyst's key names, when it reads the first one set of several. */
+function analystKeyNames(plan) {
+    const analyst = plan.analysis?.config.provider;
+    if (analyst === undefined || analyst === "codex")
+        return [];
+    const names = analysisKeyNames(analyst);
+    return names.length > 1 ? [names] : [];
+}

package/dist/remote/runner.d.ts

@@ -1,5 +1,10 @@
 import type { E2BDesktopSandbox } from "../substrates/e2b/sdk.js";
 import { type Shell } from "../substrates/shell.js";
+/**
+ * Set in the runner's run, so any humanish there runs locally even when its study could run
+ * remotely: a runner that launched a runner would repeat without end.
+ */
+export declare const INSIDE_RUNNER_ENV = "HUMANISH_INSIDE_RUNNER";
 /** The port the runner's Observer listens on. */
 export declare const RUNNER_OBSERVER_PORT = 7000;
 /** A runner sandbox and the shell into it. */
@@ -59,6 +64,22 @@ export interface RemoteFile {
 export declare function listRemoteRun(runner: Runner, runId: string): Promise<RemoteFile[]>;
 /** sha256sum lines, `<hex>  ./<path>`, as RemoteFiles with paths relative to the directory. */
 export declare function parseDigests(output: string): RemoteFile[];
+/**
+ * The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
+ * with their sizes. A run writes each of them once, so one that has settled can be copied while
+ * the run goes on; files whose name starts with a dot are still being written.
+ */
+export declare function listSettledEvidence(runner: Runner, runId: string, settledSeconds: number): Promise<{
+    path: string;
+    size: number;
+}[]>;
+/** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
+export declare function parseSizes(output: string): {
+    path: string;
+    size: number;
+}[];
+/** The bytes of one file of the runner's run directory, as they are now. */
+export declare function readRemoteBytes(runner: Runner, runId: string, file: string): Promise<Uint8Array>;
 /** The bytes of one file of the runner's run directory, checked against its listed digest. */
 export declare function readRemoteFile(runner: Runner, runId: string, file: RemoteFile): Promise<Uint8Array>;
 /**

package/dist/remote/runner.js

@@ -11,6 +11,11 @@ import { tailOf } from "../substrates/shell.js";
 const RUNNER_PROJECT = "project";
 /** Where the runner keeps the launcher's own files: the result file and the run's output. */
 const RUNNER_STATE = ".humanish-remote";
+/**
+ * Set in the runner's run, so any humanish there runs locally even when its study could run
+ * remotely: a runner that launched a runner would repeat without end.
+ */
+export const INSIDE_RUNNER_ENV = "HUMANISH_INSIDE_RUNNER";
 /** The port the runner's Observer listens on. */
 export const RUNNER_OBSERVER_PORT = 7000;
 /**
@@ -115,13 +120,20 @@ export async function startRemoteRun(runner, start) {
         start.runId,
         "--owner-key",
         start.ownerKey,
+        // The runner's run is the remote run: it must not launch a runner of its own.
+        "--local",
     ];
     const line = [
         `cd "$HOME/${RUNNER_PROJECT}"`,
         `export PATH="$HOME/${RUNNER_STATE}/node/bin:$PATH"`,
         `exec ${args.map((arg) => (arg.startsWith("$HOME") ? `"${arg}"` : shellQuote(arg))).join(" ")} > "$HOME/${RUNNER_STATE}/watch.out" 2> "$HOME/${RUNNER_STATE}/watch.err"`,
     ].join("; ");
-    const env = { ...start.env, DO_NOT_TRACK: "1", HUMANISH_TELEMETRY_DISABLED: "1" };
+    const env = {
+        ...start.env,
+        DO_NOT_TRACK: "1",
+        HUMANISH_TELEMETRY_DISABLED: "1",
+        [INSIDE_RUNNER_ENV]: "1",
+    };
     const result = await runner.shell.start(`bash -c ${shellQuote(line)}`, {
         env,
         timeoutMs: 60_000,
@@ -189,16 +201,41 @@ export function parseDigests(output) {
     }
     return files;
 }
-/** The bytes of one file of the runner's run directory, checked against its listed digest. */
-export async function readRemoteFile(runner, runId, file) {
+/**
+ * The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
+ * with their sizes. A run writes each of them once, so one that has settled can be copied while
+ * the run goes on; files whose name starts with a dot are still being written.
+ */
+export async function listSettledEvidence(runner, runId, settledSeconds) {
+    const dir = `${RUNNER_PROJECT}/.humanish/runs/${runId}`;
+    const listed = await runner.shell.run(`cd ${shellQuote(dir)} 2>/dev/null && find screenshots recordings -type f ! -name '.*' ! -newermt '${settledSeconds} seconds ago' -printf '%p\\t%s\\n' 2>/dev/null`, { timeoutMs: 120_000 });
+    return parseSizes(listed.stdout);
+}
+/** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
+export function parseSizes(output) {
+    const files = [];
+    for (const line of output.split("\n")) {
+        const match = /^((?:screenshots|recordings)\/[^\t]+)\t(\d+)$/.exec(line);
+        if (match === null || match[1].split("/").includes(".."))
+            continue;
+        files.push({ path: match[1], size: Number(match[2]) });
+    }
+    return files;
+}
+/** The bytes of one file of the runner's run directory, as they are now. */
+export async function readRemoteBytes(runner, runId, file) {
     const read = runner.sandbox.files.read;
     if (read === undefined)
         throw new Error("This sandbox cannot read files back.");
-    const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file.path}`, { format: "stream", requestTimeoutMs: 300_000 });
+    const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file}`, { format: "stream", requestTimeoutMs: 300_000 });
     const parts = [];
     for await (const chunk of stream)
         parts.push(chunk);
-    const bytes = Buffer.concat(parts);
+    return Buffer.concat(parts);
+}
+/** The bytes of one file of the runner's run directory, checked against its listed digest. */
+export async function readRemoteFile(runner, runId, file) {
+    const bytes = await readRemoteBytes(runner, runId, file.path);
     const digest = createHash("sha256").update(bytes).digest("hex");
     if (digest !== file.sha256)
         throw new Error(`${file.path} changed or arrived damaged while it was copied from the runner.`);

package/dist/run/results.d.ts

@@ -31,8 +31,16 @@ export interface RunResult {
     /** The Observer rendered for this run, when `RunOptions.observer` asked for one. */
     observer?: ObserverResult;
     warnings: string[];
+    /**
+     * Set when the run executed in a runner sandbox: the provider, and the names of the keys this
+     * machine sent to the runner, in the operator's own account.
+     */
+    remote?: {
+        provider: string;
+        keys: string[];
+    };
     error?: {
-        code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_ANALYSIS_UNSUPPORTED" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_STUDY_COMMS_UNSUPPORTED" | "HUMANISH_LIVE_RUN_UNIMPLEMENTED" | "HUMANISH_INVALID_CWD" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_INVALID_PARTICIPANT_COUNT" | "HUMANISH_INVALID_PORT" | "HUMANISH_UNSUPPORTED_RERUN_FLAGS" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_RUN_OPTION_CONFLICT" | "HUMANISH_OBSERVE_OPTION_CONFLICT" | "HUMANISH_REMOTE_REFUSED" | "HUMANISH_REMOTE_FAILED" | "HUMANISH_STUDY_SCORER_BAD_REF" | "HUMANISH_STUDY_SCORER_NOT_FOUND" | "HUMANISH_STUDY_SCORER_LOAD_FAILED" | "HUMANISH_STUDY_SCORER_NO_HOOKS" | "HUMANISH_STUDY_SCORER_UNSUPPORTED_BACKEND";
+        code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_ANALYSIS_UNSUPPORTED" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_STUDY_COMMS_UNSUPPORTED" | "HUMANISH_LIVE_RUN_UNIMPLEMENTED" | "HUMANISH_INVALID_CWD" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_INVALID_PARTICIPANT_COUNT" | "HUMANISH_INVALID_PORT" | "HUMANISH_UNSUPPORTED_RERUN_FLAGS" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_RUN_OPTION_CONFLICT" | "HUMANISH_OBSERVE_OPTION_CONFLICT" | "HUMANISH_REMOTE_REFUSED" | "HUMANISH_REMOTE_FAILED" | "HUMANISH_REMOTE_DETACHED" | "HUMANISH_STUDY_SCORER_BAD_REF" | "HUMANISH_STUDY_SCORER_NOT_FOUND" | "HUMANISH_STUDY_SCORER_LOAD_FAILED" | "HUMANISH_STUDY_SCORER_NO_HOOKS" | "HUMANISH_STUDY_SCORER_UNSUPPORTED_BACKEND";
         message: string;
     };
 }

package/dist/study/keys.d.ts

@@ -174,6 +174,7 @@ export declare const ACTOR: {
 };
 export declare const EXECUTION: {
     target: true;
+    remote: true;
     runtime: {
         version: true;
     };

package/dist/study/keys.js

@@ -112,6 +112,7 @@ export const ACTOR = {
 };
 export const EXECUTION = {
     target: true,
+    remote: true,
     runtime: { version: true },
     runtimeAuth: true,
     concurrency: true,

package/dist/study/parse/execution.js

@@ -23,6 +23,11 @@ export function parseExecution(raw) {
         }
         execution.target = target;
     }
+    if (raw.remote !== undefined) {
+        if (typeof raw.remote !== "boolean")
+            return invalid("`execution.remote` must be true or false.");
+        execution.remote = raw.remote;
+    }
     const timeoutMs = posInt(raw.timeoutMs);
     if (timeoutMs !== undefined)
         execution.timeoutMs = timeoutMs;

package/dist/study/types.d.ts

@@ -581,6 +581,12 @@ export type StudyNetworkCapture = boolean | {
 };
 export interface StudyExecution {
     target?: StudyExecutionTarget;
+    /**
+     * Where the study's own process runs. Unset: in a runner sandbox on the participants' provider
+     * when the study can, so it outlives this machine; false: always on this machine; true: only
+     * remotely, refused when the study cannot. `--local` and `--remote` override it for one run.
+     */
+    remote?: boolean;
     /** Actor session wall-clock budget. Consumed on the app-url route. */
     timeoutMs?: number;
     /** Forward-declared: no route reads it yet, so a set value warns. */

package/dist/tui/launch.js

@@ -104,6 +104,9 @@ export async function launchRun(options) {
         cwd,
         "--json",
         "--no-open",
+        // The TUI follows and stops a run through its local status record, which a remote run does
+        // not have until it is copied back.
+        "--local",
         ...(options.mode === "dry-run" ? ["--dry-run"] : []),
         // `--` ends option parsing, so the handle can only ever be read as the positional argument.
         "--",

package/docs/contracts/study-analysis.md

@@ -25,7 +25,8 @@ study's input and admission estimate without credentials, a provider request,
 or a new analysis artifact. It prints the expected cost, the worst case and the
 cap; a refused dry run says why, and names the `--max-cost` that admits it.
 
-The default provider is `openai`; its default model is `gpt-6-astra`, with high reasoning effort. A request sends selected retained text and
+The default provider is `openai`; its default model is `gpt-6.1-sol`, at medium reasoning effort
+([reasoning effort](#reasoning-effort)). A request sends selected retained text and
 captures to OpenAI, without tools, redirects, provider-side response storage, or
 automatic retries. `--question` adds a reviewer question; it never changes the
 participant assignment. For OpenAI API analysis, `--max-cost` is required, including for dry-run
@@ -118,6 +119,37 @@ the current merge instructions, billed $1.92, most of its input served from the
 provider's prompt cache, and each of its 466 citations resolved. Two runs do not
 calibrate the merge request's estimate.
 
+## Reasoning effort
+
+`review.analysis.reasoningEffort`, or `analyze --reasoning-effort <level>`, sets how long an API
+analyst reasons before it answers. OpenAI receives it as `reasoning.effort`, and Gemini as
+`generationConfig.thinkingConfig.thinkingLevel` in capitals. Each model takes the levels its
+provider documents for it (OpenAI's model pages and Gemini's thinking guide, read 2026-10-21):
+
+| Model                                                                   | Levels                                          |
+| ----------------------------------------------------------------------- | ----------------------------------------------- |
+| `gpt-6-astra`, `gpt-6.1-sol`                                            | `low`, `medium`, `high`, `xhigh`, `max`         |
+| `gpt-6-luna`, `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna` | `none`, `low`, `medium`, `high`, `xhigh`, `max` |
+| `gpt-5.5`                                                               | `none`, `low`, `medium`, `high`, `xhigh`        |
+| `gemini-3.8-flash`                                                      | `low`, `medium`, `high`                         |
+
+Any other level is refused before a request, with a message naming the levels the model takes.
+The Codex account analyst runs at its fixed low effort, and a declared effort with
+`provider: codex` is refused.
+
+The default analyst, an OpenAI analysis that names no model and declares no effort, is
+`gpt-6.1-sol` at `medium`, and its resolved configuration writes `reasoningEffort: medium`. A study
+or command that names a model but no effort runs that model at `high`; only the default analyst
+runs at medium. That analysis's configuration has no `reasoningEffort` field, so its configuration
+hash, and with it reuse of an earlier analysis, is what it was before the field existed. A declared effort is stored in the analysis artifact's
+`config`, `high` included, and gives the configuration its own hash. An artifact whose config has
+no `reasoningEffort` ran at `high`.
+
+Admission expects the same output at every effort, from analyses billed at `high`. No billed
+analysis at a lower effort calibrates a smaller expectation yet, so a lower effort's expected cost
+is likely over its bill: on 24 one-participant Taskly analyses on 2026-10-22, `gpt-6.1-sol` at
+medium wrote a mean 3,747 output tokens against the 13,000 admission expects. The worst case spends the whole output allowance at any effort.
+
 ## Gemini API analysis
 
 `review.analysis.provider: gemini`, or `analyze --provider gemini`, sends the same request to
@@ -131,7 +163,8 @@ humanish analyze --run latest --provider gemini --model gemini-3.8-flash --max-c
 
 The request goes to `generativelanguage.googleapis.com/v1beta/models/<model>:generateContent`
 with the key in the `x-goog-api-key` header. It carries the same instructions, packet and
-captures, with each capture at `MEDIA_RESOLUTION_HIGH`, thinking level `HIGH`, one candidate,
+captures, with each capture at `MEDIA_RESOLUTION_HIGH`, the analysis's reasoning effort as its
+thinking level (`HIGH` unless declared), one candidate,
 the standard service tier, no tools and `store: false`. The key is `GEMINI_API_KEY` when set,
 else `GOOGLE_API_KEY`. Google's client libraries prefer `GOOGLE_API_KEY`; humanish does not, so a
 `GOOGLE_API_KEY` exported for another Google API never replaces the stored Gemini key. Key

package/docs/product/automatic-analysis.md

@@ -10,8 +10,12 @@ Every other participant, including a Claude Code `local-agent` participant, gets
 analyst. humanish never moves a study to the other analyst because a key is missing: the account
 that pays for analysis is the one the study names or its participant implies.
 
-The OpenAI default is `gpt-6-astra` with high reasoning effort, a separate $10 cap on its
-expected cost and a 600-second timeout. When no output limit is specified,
+The OpenAI default is `gpt-6.1-sol` at medium reasoning effort (`review.analysis.reasoningEffort`
+sets another), a separate $10 cap on its
+expected cost and a 600-second timeout. A study or command that names a model but no effort runs
+that model at high; only the default analyst, with no model named, runs at medium. On the same
+recordings it matched `gpt-6-astra` at high, the earlier default, at about one seventh of the cost
+([analysis models on fixed recordings](https://github.com/danielgwilson/humanish/blob/main/docs/evidence/benchmark/2026-10-22-analysis-models.md)). When no output limit is specified,
 humanish selects 32,768 tokens if the exact input's expected cost still fits that
 cap; otherwise it keeps the established 16,384-token allowance. This preserves
 previously admitted studies without increasing their spending limit. To customize it:
@@ -21,7 +25,9 @@ review:
   analysis:
     maxCostUsd: 10
     # Optional overrides. Omit maxOutputTokens for budget-aware selection.
-    model: gpt-6-astra
+    # A named model without reasoningEffort runs at high.
+    model: gpt-6.1-sol
+    reasoningEffort: medium
     timeoutMs: 600000
     # maxOutputTokens: 32768
     # question: Where did participants need to recover?
@@ -41,9 +47,12 @@ the cap admits no analysis at all, it says so instead. A study of more than 16 p
 analysed in cohorts of at most 16, one request each, and one more request merges their reports;
 the range covers every request and the analysis line names them (see
 [runs with more than 16 participants](../contracts/study-analysis.md#runs-with-more-than-16-participants)).
-The default $10 cap admits a single request at the evidence limits ($3.79 expected for one
-participant, $4.54 for 16) and the analysis of 17 to 40 participants from $4.60 to $7.05 with no
-evidence; a run that keeps more evidence can still be refused, and its skip names the
+The default $10 cap admits the default analyst up to the evidence limits: a single request at
+$0.76 expected for one participant and $0.91 for 16, and 17 to 40 participants at $0.92 to $3.24
+from no evidence to the limits. Admission expects 13,000 output tokens for one participant at any
+effort; on 24 Taskly analyses `gpt-6.1-sol` at medium wrote a mean 3,747 and cost a mean $0.06.
+`gpt-6-astra` at high expects $3.79 for one participant at the limits and $4.60 to $7.05 for 17 to
+40 with no evidence, so a run that keeps more evidence can be refused, and its skip names the
 `--max-cost` that admits it. Use `analyze --dry-run --max-cost <usd>` on retained
 evidence to inspect the expected cost, the worst case and the selected token
 allowance before deliberately choosing a larger budget. Explicit
@@ -89,7 +98,8 @@ review:
     maxCostUsd: 2
 ```
 
-Gemini analysis sends the same evidence packet and captures to Google, at high thinking level,
+Gemini analysis sends the same evidence packet and captures to Google, at the high thinking level
+unless the study declares a `reasoningEffort`,
 with each capture at high media resolution. It reads `GEMINI_API_KEY`, which `humanish keys set
 gemini` stores, and `GOOGLE_API_KEY` only when `GEMINI_API_KEY` is unset. Key discovery treats
 the two names as one key: a project's `.humanish/local/provider.env` or the user store never adds

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "humanish",
-  "version": "0.142.0",
+  "version": "0.144.0",
   "description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
   "keywords": [
     "cli",