humanish 0.141.0 → 0.142.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -0
- package/CHANGELOG.md +11 -0
- package/README.md +2 -0
- package/dist/analysis/admission.d.ts +8 -0
- package/dist/analysis/admission.js +15 -4
- package/dist/analysis/automatic-config.d.ts +5 -2
- package/dist/analysis/automatic-config.js +23 -8
- package/dist/analysis/automatic.js +13 -7
- package/dist/analysis/execute.d.ts +17 -6
- package/dist/analysis/execute.js +36 -18
- package/dist/analysis/gemini-wire.d.ts +8 -0
- package/dist/analysis/gemini-wire.js +165 -0
- package/dist/analysis/image-tokens.d.ts +4 -2
- package/dist/analysis/image-tokens.js +24 -3
- package/dist/analysis/job.d.ts +1 -0
- package/dist/analysis/job.js +1 -0
- package/dist/analysis/openai-wire.d.ts +5 -0
- package/dist/analysis/openai-wire.js +101 -0
- package/dist/analysis/provider.d.ts +15 -7
- package/dist/analysis/provider.js +21 -101
- package/dist/analysis/responses.d.ts +3 -0
- package/dist/analysis/responses.js +19 -0
- package/dist/analysis/service.js +11 -3
- package/dist/analysis/types.d.ts +8 -2
- package/dist/analysis/validation.d.ts +1 -0
- package/dist/analysis/validation.js +13 -12
- package/dist/cli/commands/analyze.js +9 -7
- package/dist/cli/commands/observe.js +26 -1
- package/dist/cli/commands/run-command.d.ts +1 -1
- package/dist/cli/commands/run-command.js +12 -0
- package/dist/cli/commands/study-remote.d.ts +13 -0
- package/dist/cli/commands/study-remote.js +125 -0
- package/dist/cli/commands/study-route-computer-use.js +26 -2
- package/dist/cli/commands/study-route-scripted.js +1 -0
- package/dist/cli/commands/study-route-shared-world.js +1 -0
- package/dist/cli/commands/study-route-terminal.js +1 -0
- package/dist/cli/commands/study-run.js +34 -0
- package/dist/cli/commands/watch.js +5 -0
- package/dist/cli/doctor.js +7 -2
- package/dist/cli/findings.d.ts +8 -1
- package/dist/cli/findings.js +38 -15
- package/dist/cli/io.d.ts +8 -0
- package/dist/cli/io.js +2 -1
- package/dist/keys/key-resolution.d.ts +10 -0
- package/dist/keys/key-resolution.js +33 -5
- package/dist/observer/listen.d.ts +7 -2
- package/dist/observer/listen.js +10 -4
- package/dist/observer/render.d.ts +2 -0
- package/dist/observer/render.js +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/remote/launch.d.ts +53 -0
- package/dist/remote/launch.js +320 -0
- package/dist/remote/observer-proxy.d.ts +11 -0
- package/dist/remote/observer-proxy.js +66 -0
- package/dist/remote/plan.d.ts +35 -0
- package/dist/remote/plan.js +114 -0
- package/dist/remote/reattach.d.ts +17 -0
- package/dist/remote/reattach.js +81 -0
- package/dist/remote/runner.d.ts +68 -0
- package/dist/remote/runner.js +216 -0
- package/dist/routes/computer-use/e2b-desktop/start.js +1 -1
- package/dist/routes/computer-use/types.d.ts +2 -0
- package/dist/routes/scripted/types.d.ts +2 -0
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/routes/terminal/types.d.ts +2 -0
- package/dist/run/costs.d.ts +2 -1
- package/dist/run/pricing.d.ts +13 -1
- package/dist/run/pricing.js +50 -4
- package/dist/run/results.d.ts +1 -1
- package/dist/run/run-cost.d.ts +2 -1
- package/dist/run/run-cost.js +6 -1
- package/dist/run/run.d.ts +13 -0
- package/dist/run/run.js +18 -7
- package/dist/run/sandbox-creates.d.ts +10 -3
- package/dist/run/sandbox-creates.js +48 -6
- package/dist/run-study.d.ts +5 -0
- package/dist/study/doctor.js +14 -1
- package/dist/study/requirements.d.ts +1 -1
- package/dist/study/requirements.js +8 -6
- package/dist/study/route-inputs.js +4 -0
- package/dist/study/run-study-events.d.ts +1 -1
- package/dist/study/run-study-events.js +9 -2
- package/dist/substrates/e2b/base-module.d.ts +3 -0
- package/dist/substrates/e2b/base-module.js +40 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.js +24 -6
- package/dist/substrates/e2b/sdk.d.ts +24 -0
- package/dist/substrates/e2b/sdk.js +5 -1
- package/dist/substrates/e2b/shell-only.d.ts +5 -0
- package/dist/substrates/e2b/shell-only.js +27 -0
- package/dist/substrates/modal/sandbox.js +27 -28
- package/dist/substrates/modal/sdk.d.ts +7 -0
- package/dist/tui-app.js +46 -46
- package/docs/contracts/study-analysis.md +51 -3
- package/docs/product/automatic-analysis.md +26 -2
- package/package.json +2 -2
- package/skills/humanish/SKILL.md +10 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
2
|
+
import type { RemotePlan } from "./plan.js";
|
|
3
|
+
import { type Runner } from "./runner.js";
|
|
4
|
+
/** What following a remote run needs from its caller: the project, its env and a progress line. */
|
|
5
|
+
export interface RemoteContext {
|
|
6
|
+
cwd: string;
|
|
7
|
+
env: Readonly<Record<string, string | undefined>>;
|
|
8
|
+
/** Progress lines for a person, on stderr. */
|
|
9
|
+
say: (line: string) => void;
|
|
10
|
+
}
|
|
11
|
+
export interface RemoteLaunch extends RemoteContext {
|
|
12
|
+
/** The study file, relative to cwd. */
|
|
13
|
+
study: string;
|
|
14
|
+
remote: RemotePlan;
|
|
15
|
+
/** Local Observer port; 0 picks a free one. */
|
|
16
|
+
port: number;
|
|
17
|
+
}
|
|
18
|
+
/** What a remote run ended as, for the command's result and exit code. */
|
|
19
|
+
export interface RemoteOutcome {
|
|
20
|
+
runId: string;
|
|
21
|
+
exitCode: number;
|
|
22
|
+
/** The run's own result, as the runner's command wrote it. */
|
|
23
|
+
result?: unknown;
|
|
24
|
+
/** Why the remote run did not finish, when it did not. */
|
|
25
|
+
error?: string;
|
|
26
|
+
/** Left running on purpose: the terminal closed while the study went on. */
|
|
27
|
+
detached?: boolean;
|
|
28
|
+
files?: number;
|
|
29
|
+
}
|
|
30
|
+
/** Launch, follow and copy back one remote run. */
|
|
31
|
+
export declare function launchRemoteRun(launch: RemoteLaunch): Promise<RemoteOutcome>;
|
|
32
|
+
/** Follows the run until it ends, is stopped, detaches or stops answering. */
|
|
33
|
+
export declare function followRemoteRun(args: {
|
|
34
|
+
launch: RemoteContext;
|
|
35
|
+
runner: Runner;
|
|
36
|
+
paths: PreparedRunArtifactPaths;
|
|
37
|
+
runId: string;
|
|
38
|
+
signals: Signals;
|
|
39
|
+
proxyUrl: string;
|
|
40
|
+
}): Promise<RemoteOutcome>;
|
|
41
|
+
/** Two sandbox journals as one: every line of each, in order, one per line. */
|
|
42
|
+
export declare function joinJournals(...journals: string[]): string;
|
|
43
|
+
export interface Signals {
|
|
44
|
+
readonly requested: "stop" | "detach" | undefined;
|
|
45
|
+
/** The exit code a local run gives for the stop signal received: 130 for SIGINT, 143 for SIGTERM. */
|
|
46
|
+
readonly exitCode: number;
|
|
47
|
+
dispose(): void;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Ctrl-C and SIGTERM stop the run, as they stop a local one. A closed terminal (SIGHUP) leaves it
|
|
51
|
+
* running; the runner ends it at its deadline.
|
|
52
|
+
*/
|
|
53
|
+
export declare function watchSignals(): Signals;
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
// `humanish run|watch <study> --remote`: run the study's own process in a runner sandbox on its
|
|
2
|
+
// participants' provider, follow it from here, and copy the finished run back. This machine only
|
|
3
|
+
// plans, launches and watches, so it may sleep or lose its network while the study goes on.
|
|
4
|
+
import { execFile } from "node:child_process";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { fileURLToPath } from "node:url";
|
|
10
|
+
import { promisify } from "node:util";
|
|
11
|
+
import { readContainedRegularFile, writeContainedOutputFile } from "../run/contained-output.js";
|
|
12
|
+
import { createRunArtifactPaths } from "../run/paths.js";
|
|
13
|
+
import { reclaimRunSandboxes } from "../run/reclaim.js";
|
|
14
|
+
import { pointLatestRunAt } from "../run/run.js";
|
|
15
|
+
import { appendSandboxOwner, SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
16
|
+
import { loadE2BBaseModule } from "../substrates/e2b/base-module.js";
|
|
17
|
+
import { e2bConnection } from "../substrates/e2b/connection.js";
|
|
18
|
+
import { acquireE2BShellSandbox } from "../substrates/e2b/sandbox.js";
|
|
19
|
+
import { loadModalShellModule } from "../substrates/modal/sandbox.js";
|
|
20
|
+
import { serveObserverProxy } from "./observer-proxy.js";
|
|
21
|
+
import { installHumanish, listRemoteRun, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
|
|
22
|
+
/** How often the launcher reads the runner's run while it follows it. */
|
|
23
|
+
const POLL_MS = 5_000;
|
|
24
|
+
/** Reads in a row the runner may leave unanswered before the launcher gives it up. */
|
|
25
|
+
const UNANSWERED_LIMIT = 24;
|
|
26
|
+
/** How long the run has to write its status record after it is started. */
|
|
27
|
+
const START_DEADLINE_MS = 3 * 60_000;
|
|
28
|
+
/** How long a stopped run has to end and write its result. */
|
|
29
|
+
const STOP_DEADLINE_MS = 2 * 60_000;
|
|
30
|
+
/** Launch, follow and copy back one remote run. */
|
|
31
|
+
export async function launchRemoteRun(launch) {
|
|
32
|
+
const { cwd, remote, say } = launch;
|
|
33
|
+
const runId = `cua-${new Date().toISOString().replace(/[:.]/g, "-")}-${randomBytes(4).toString("hex")}`;
|
|
34
|
+
const ownerKey = randomBytes(8).toString("hex");
|
|
35
|
+
const created = await createRunArtifactPaths(cwd, runId);
|
|
36
|
+
if (!created.ok)
|
|
37
|
+
throw new Error(`Run ${runId} already exists here.`);
|
|
38
|
+
const paths = created.paths;
|
|
39
|
+
// The participants carry the key the runner's run pins; this directory records it as an owner
|
|
40
|
+
// line, so reclaim here finds them. The runner itself carries this directory's own key, so the
|
|
41
|
+
// remote run, which reclaims by its key when it stops, never matches the sandbox it runs in.
|
|
42
|
+
const owned = await appendSandboxOwner(paths, { tool: "humanish", runId, runKey: ownerKey }, remote.provider);
|
|
43
|
+
if (!owned)
|
|
44
|
+
throw new Error(`Could not record run ${runId}'s owner tags in its run directory, so a later reclaim could not find its sandboxes. Nothing was created.`);
|
|
45
|
+
// Installed before the first sandbox exists, so a signal never ends this process while one is
|
|
46
|
+
// being created without a cleanup.
|
|
47
|
+
const signals = watchSignals();
|
|
48
|
+
let tarballDir;
|
|
49
|
+
let proxy;
|
|
50
|
+
try {
|
|
51
|
+
const tarball = await packThisHumanish();
|
|
52
|
+
tarballDir = tarball.dir;
|
|
53
|
+
const where = remote.provider === "modal" ? "a Modal sandbox" : "an E2B sandbox";
|
|
54
|
+
say(`Running this study on ${where} as run ${runId}; creating the runner.`);
|
|
55
|
+
const runner = runnerOf((await acquireRunner(launch, paths)).sandbox);
|
|
56
|
+
if (signals.requested !== undefined)
|
|
57
|
+
return await endRun(launch, runId, {
|
|
58
|
+
exitCode: signals.exitCode,
|
|
59
|
+
error: "stopped before the run started",
|
|
60
|
+
});
|
|
61
|
+
await writeRemoteRecord(paths, {
|
|
62
|
+
runId,
|
|
63
|
+
provider: remote.provider,
|
|
64
|
+
deadline: new Date(Date.now() + remote.runnerMs).toISOString(),
|
|
65
|
+
});
|
|
66
|
+
// Bound before the run starts, so a taken --port fails before any participant exists.
|
|
67
|
+
proxy = await serveObserverProxy(await runner.sandbox.portAccess(RUNNER_OBSERVER_PORT), launch.port);
|
|
68
|
+
const version = await installHumanish(runner, tarball.bytes, [await peerSpec(remote.provider)]);
|
|
69
|
+
say(`Installed humanish ${version} in the runner; uploading the study.`);
|
|
70
|
+
await uploadProject(runner, cwd, remote.uploads, remote.uploadDirectories);
|
|
71
|
+
if (signals.requested !== undefined)
|
|
72
|
+
return await endRun(launch, runId, {
|
|
73
|
+
exitCode: signals.exitCode,
|
|
74
|
+
error: "stopped before the run started",
|
|
75
|
+
});
|
|
76
|
+
await startRemoteRun(runner, {
|
|
77
|
+
study: launch.study,
|
|
78
|
+
runId,
|
|
79
|
+
ownerKey,
|
|
80
|
+
env: Object.fromEntries(remote.envNames.flatMap((name) => {
|
|
81
|
+
const value = launch.env[name];
|
|
82
|
+
return value === undefined || value === "" ? [] : [[name, value]];
|
|
83
|
+
})),
|
|
84
|
+
});
|
|
85
|
+
return await followRemoteRun({ launch, runner, paths, runId, signals, proxyUrl: proxy.url });
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
await reclaimRunSandboxes(cwd, runId, { env: launch.env }).catch(() => undefined);
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
signals.dispose();
|
|
93
|
+
await proxy?.close();
|
|
94
|
+
if (tarballDir !== undefined)
|
|
95
|
+
await rm(tarballDir, { recursive: true, force: true });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
async function acquireRunner(launch, paths) {
|
|
99
|
+
const { provider, runnerMs } = launch.remote;
|
|
100
|
+
const module = provider === "modal" ? await loadModalShellModule(launch.env) : await loadE2BBaseModule();
|
|
101
|
+
// A Modal module takes its connection from launch.env and ignores these two.
|
|
102
|
+
const { apiKey = "", ...account } = e2bConnection(launch.env);
|
|
103
|
+
return acquireE2BShellSandbox({
|
|
104
|
+
module,
|
|
105
|
+
provider,
|
|
106
|
+
options: {
|
|
107
|
+
apiKey,
|
|
108
|
+
...account,
|
|
109
|
+
timeoutMs: runnerMs,
|
|
110
|
+
metadata: { participantId: "runner" },
|
|
111
|
+
network: { allowPublicTraffic: false },
|
|
112
|
+
},
|
|
113
|
+
receipt: { root: paths, participantId: "runner" },
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
/** Follows the run until it ends, is stopped, detaches or stops answering. */
|
|
117
|
+
export async function followRemoteRun(args) {
|
|
118
|
+
const { launch, runner, paths, runId, signals } = args;
|
|
119
|
+
const { say } = launch;
|
|
120
|
+
const copy = () => copyRunBack(runner, paths, runId);
|
|
121
|
+
let state;
|
|
122
|
+
let stopAt;
|
|
123
|
+
let unanswered = 0;
|
|
124
|
+
const startBy = Date.now() + START_DEADLINE_MS;
|
|
125
|
+
for (;;) {
|
|
126
|
+
const run = await readRemoteRun(runner, runId).catch(() => undefined);
|
|
127
|
+
unanswered = run === undefined ? unanswered + 1 : 0;
|
|
128
|
+
if (run?.state !== undefined && state === undefined) {
|
|
129
|
+
say(`Run ${runId} is running. Watch it live at ${args.proxyUrl}`);
|
|
130
|
+
say("Closing this terminal leaves the study running; Ctrl-C stops it.");
|
|
131
|
+
}
|
|
132
|
+
if (run?.state !== undefined && run.state !== state) {
|
|
133
|
+
state = run.state;
|
|
134
|
+
if (state !== "running")
|
|
135
|
+
say(`Run ${runId} is ${state}.`);
|
|
136
|
+
}
|
|
137
|
+
if (run?.result !== undefined) {
|
|
138
|
+
say("The run ended; copying its evidence back.");
|
|
139
|
+
const copied = await copy().then((files) => ({ files }), (error) => ({ error }));
|
|
140
|
+
// The run reclaimed its participants as it ended; only the runner is left, and it holds the
|
|
141
|
+
// only whole copy of the evidence until a copy succeeds.
|
|
142
|
+
if ("error" in copied && stopAt === undefined)
|
|
143
|
+
return {
|
|
144
|
+
runId,
|
|
145
|
+
exitCode: 2,
|
|
146
|
+
error: `copying the evidence back failed (${message(copied.error)}). The runner keeps it until the deadline in its remote.json; stop it sooner with humanish reclaim --run ${runId}`,
|
|
147
|
+
};
|
|
148
|
+
return await endRun(launch, runId, {
|
|
149
|
+
// A run stopped from here exits as a local one does on that signal.
|
|
150
|
+
exitCode: stopAt === undefined ? run.result.exitCode : signals.exitCode,
|
|
151
|
+
result: run.result.result,
|
|
152
|
+
...("files" in copied
|
|
153
|
+
? { files: copied.files }
|
|
154
|
+
: { error: `copying the evidence back failed: ${message(copied.error)}` }),
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
if (signals.requested === "detach")
|
|
158
|
+
return { runId, exitCode: 0, detached: true };
|
|
159
|
+
if (signals.requested === "stop" && stopAt === undefined) {
|
|
160
|
+
say("Stopping the remote run; its participants are reclaimed before it ends.");
|
|
161
|
+
stopAt = Date.now() + STOP_DEADLINE_MS;
|
|
162
|
+
if (state === undefined)
|
|
163
|
+
return await endRun(launch, runId, {
|
|
164
|
+
exitCode: signals.exitCode,
|
|
165
|
+
error: "stopped before the run started",
|
|
166
|
+
});
|
|
167
|
+
await stopRemoteRun(runner).catch(() => undefined);
|
|
168
|
+
}
|
|
169
|
+
const ended = unanswered >= UNANSWERED_LIMIT
|
|
170
|
+
? `the runner sandbox stopped answering for ${(UNANSWERED_LIMIT * POLL_MS) / 1000} s`
|
|
171
|
+
: run !== undefined && state !== undefined && !run.alive
|
|
172
|
+
? stopAt === undefined
|
|
173
|
+
? `the run ended without a result: ${run.errTail || "no output"}`
|
|
174
|
+
: "stopped"
|
|
175
|
+
: stopAt !== undefined && Date.now() > stopAt
|
|
176
|
+
? "stopped, and the run did not end in time"
|
|
177
|
+
: state === undefined && Date.now() > startBy
|
|
178
|
+
? `the run did not start in the runner: ${run?.errTail || "no output"}`
|
|
179
|
+
: undefined;
|
|
180
|
+
if (ended !== undefined) {
|
|
181
|
+
// Participants may still be running, so everything is reclaimed after whatever copies.
|
|
182
|
+
const copied = await copy().then((files) => ({ files }), (error) => ({ error }));
|
|
183
|
+
return await endRun(launch, runId, {
|
|
184
|
+
exitCode: stopAt === undefined ? 2 : signals.exitCode,
|
|
185
|
+
error: "files" in copied
|
|
186
|
+
? ended
|
|
187
|
+
: `${ended}; copying its evidence back failed: ${message(copied.error)}`,
|
|
188
|
+
...("files" in copied ? { files: copied.files } : {}),
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
await new Promise((resolve) => setTimeout(resolve, POLL_MS));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Copy every file of the runner's run directory into this machine's run directory, each checked
|
|
196
|
+
* against the digest the runner listed. The sandbox journal keeps this machine's lines (the
|
|
197
|
+
* runner's own receipt) and adds the runner's (its participants').
|
|
198
|
+
*/
|
|
199
|
+
async function copyRunBack(runner, paths, runId) {
|
|
200
|
+
const files = await listRemoteRun(runner, runId);
|
|
201
|
+
for (const file of files) {
|
|
202
|
+
if (file.path === "owner-key")
|
|
203
|
+
continue;
|
|
204
|
+
const bytes = await readRemoteFile(runner, runId, file);
|
|
205
|
+
if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
|
|
206
|
+
const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
|
|
207
|
+
const mine = local.status === "read" ? Buffer.from(local.bytes).toString("utf8") : "";
|
|
208
|
+
await writeContainedOutputFile(paths, file.path, joinJournals(mine, Buffer.from(bytes).toString("utf8")), "utf8");
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
await writeContainedOutputFile(paths, file.path, bytes);
|
|
212
|
+
}
|
|
213
|
+
await pointLatestRunAt(paths, runId);
|
|
214
|
+
return files.length;
|
|
215
|
+
}
|
|
216
|
+
/** Two sandbox journals as one: every line of each, in order, one per line. */
|
|
217
|
+
export function joinJournals(...journals) {
|
|
218
|
+
const lines = journals
|
|
219
|
+
.flatMap((journal) => journal.split("\n"))
|
|
220
|
+
.filter((line) => line.trim() !== "");
|
|
221
|
+
return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Reclaim every sandbox of the run, by receipt and by its tags, and report a reclaim that did not
|
|
225
|
+
* come back clean as an error and a failing exit code.
|
|
226
|
+
*/
|
|
227
|
+
async function endRun(launch, runId, outcome) {
|
|
228
|
+
const reclaimed = await reclaimRunSandboxes(launch.cwd, runId, { env: launch.env }).then((result) => result.ok, () => false);
|
|
229
|
+
if (reclaimed)
|
|
230
|
+
return { runId, ...outcome };
|
|
231
|
+
return {
|
|
232
|
+
runId,
|
|
233
|
+
...outcome,
|
|
234
|
+
exitCode: outcome.exitCode === 0 ? 2 : outcome.exitCode,
|
|
235
|
+
error: [
|
|
236
|
+
outcome.error,
|
|
237
|
+
`its sandboxes were not confirmed stopped; run humanish reclaim --run ${runId}`,
|
|
238
|
+
]
|
|
239
|
+
.filter(Boolean)
|
|
240
|
+
.join("; "),
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
function message(error) {
|
|
244
|
+
return error instanceof Error ? error.message : String(error);
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* The run directory's record of where its run executes. It holds no credential and no raw sandbox
|
|
248
|
+
* id: the runner's id is in the sandbox journal with every other, as its `runner` receipt.
|
|
249
|
+
*/
|
|
250
|
+
async function writeRemoteRecord(paths, record) {
|
|
251
|
+
await writeContainedOutputFile(paths, "remote.json", `${JSON.stringify({ schema: "humanish.remote-run.v1", ...record }, null, 2)}\n`, "utf8");
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Ctrl-C and SIGTERM stop the run, as they stop a local one. A closed terminal (SIGHUP) leaves it
|
|
255
|
+
* running; the runner ends it at its deadline.
|
|
256
|
+
*/
|
|
257
|
+
export function watchSignals() {
|
|
258
|
+
let requested;
|
|
259
|
+
let exitCode = 130;
|
|
260
|
+
const interrupt = () => {
|
|
261
|
+
requested ??= "stop";
|
|
262
|
+
};
|
|
263
|
+
const terminate = () => {
|
|
264
|
+
if (requested === undefined)
|
|
265
|
+
exitCode = 143;
|
|
266
|
+
requested ??= "stop";
|
|
267
|
+
};
|
|
268
|
+
const detach = () => {
|
|
269
|
+
requested ??= "detach";
|
|
270
|
+
};
|
|
271
|
+
process.on("SIGINT", interrupt);
|
|
272
|
+
process.on("SIGTERM", terminate);
|
|
273
|
+
process.on("SIGHUP", detach);
|
|
274
|
+
return {
|
|
275
|
+
get requested() {
|
|
276
|
+
return requested;
|
|
277
|
+
},
|
|
278
|
+
get exitCode() {
|
|
279
|
+
return exitCode;
|
|
280
|
+
},
|
|
281
|
+
dispose() {
|
|
282
|
+
process.off("SIGINT", interrupt);
|
|
283
|
+
process.off("SIGTERM", terminate);
|
|
284
|
+
process.off("SIGHUP", detach);
|
|
285
|
+
},
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* This machine's humanish as an npm tarball, so the runner runs exactly the same code. A run from
|
|
290
|
+
* source has no built package to pack.
|
|
291
|
+
*/
|
|
292
|
+
async function packThisHumanish() {
|
|
293
|
+
const here = fileURLToPath(import.meta.url);
|
|
294
|
+
const root = path.resolve(path.dirname(here), "..", "..");
|
|
295
|
+
if (path.relative(root, here).split(path.sep)[0] !== "dist")
|
|
296
|
+
throw new Error("A remote run installs this humanish in the runner, and a run from source has no built package. Run `pnpm build`, then `node dist/cli.js`, or use an installed humanish.");
|
|
297
|
+
const dir = await mkdtemp(path.join(tmpdir(), "humanish-pack-"));
|
|
298
|
+
const { stdout } = await promisify(execFile)("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", dir, root], { maxBuffer: 16 * 1024 * 1024 });
|
|
299
|
+
const [packed] = JSON.parse(stdout);
|
|
300
|
+
return { bytes: await readFile(path.join(dir, packed.filename)), dir };
|
|
301
|
+
}
|
|
302
|
+
/** The provider SDK the runner's participants need, at the version this machine has. */
|
|
303
|
+
async function peerSpec(provider) {
|
|
304
|
+
const name = provider === "modal" ? "modal" : "@e2b/desktop";
|
|
305
|
+
try {
|
|
306
|
+
const entry = fileURLToPath(import.meta.resolve(name));
|
|
307
|
+
for (let dir = path.dirname(entry); dir !== path.dirname(dir); dir = path.dirname(dir)) {
|
|
308
|
+
const manifest = await readFile(path.join(dir, "package.json"), "utf8").catch(() => undefined);
|
|
309
|
+
if (manifest === undefined)
|
|
310
|
+
continue;
|
|
311
|
+
const parsed = JSON.parse(manifest);
|
|
312
|
+
if (parsed.name === name && parsed.version !== undefined)
|
|
313
|
+
return `${name}@${parsed.version}`;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
// Not installed here, which planning already refused; the runner takes the latest.
|
|
318
|
+
}
|
|
319
|
+
return name;
|
|
320
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { SandboxPortAccess } from "../substrates/e2b/sdk.js";
|
|
2
|
+
export interface ObserverProxy {
|
|
3
|
+
readonly url: string;
|
|
4
|
+
close(): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Serve `access` on 127.0.0.1:`port` (0 picks a free port). A request whose Host is not this
|
|
8
|
+
* loopback address is refused, so a page that rebinds a name to 127.0.0.1 cannot read the runner's
|
|
9
|
+
* Observer through it; a write must also come from this origin.
|
|
10
|
+
*/
|
|
11
|
+
export declare function serveObserverProxy(access: SandboxPortAccess, port: number): Promise<ObserverProxy>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// The local end of watching a remote run: a server on 127.0.0.1 that forwards each request to the
|
|
2
|
+
// runner's Observer through the provider's proxy, adding the credential the provider requires. The
|
|
3
|
+
// credential stays in this process; the browser only ever talks to loopback.
|
|
4
|
+
import { createServer, request as httpRequest, } from "node:http";
|
|
5
|
+
import { request as httpsRequest } from "node:https";
|
|
6
|
+
import { listenOnLoopback } from "../observer/listen.js";
|
|
7
|
+
/** The hop-by-hop headers a proxy does not forward, and the ones this proxy sets itself. */
|
|
8
|
+
const DROPPED = new Set(["host", "connection", "keep-alive", "upgrade", "proxy-authorization"]);
|
|
9
|
+
/**
|
|
10
|
+
* Serve `access` on 127.0.0.1:`port` (0 picks a free port). A request whose Host is not this
|
|
11
|
+
* loopback address is refused, so a page that rebinds a name to 127.0.0.1 cannot read the runner's
|
|
12
|
+
* Observer through it; a write must also come from this origin.
|
|
13
|
+
*/
|
|
14
|
+
export async function serveObserverProxy(access, port) {
|
|
15
|
+
const target = new URL(access.url);
|
|
16
|
+
let origin = "";
|
|
17
|
+
const server = createServer((request, response) => {
|
|
18
|
+
if (!sameOrigin(request, origin)) {
|
|
19
|
+
response.writeHead(421, { "content-type": "text/plain; charset=utf-8" });
|
|
20
|
+
response.end("This address only serves 127.0.0.1.\n");
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
forward(request, response, target, access.headers);
|
|
24
|
+
});
|
|
25
|
+
const bound = await listenOnLoopback(server, port);
|
|
26
|
+
origin = `127.0.0.1:${bound}`;
|
|
27
|
+
return {
|
|
28
|
+
url: `http://${origin}/`,
|
|
29
|
+
close: () => new Promise((resolve) => {
|
|
30
|
+
server.close(() => resolve());
|
|
31
|
+
server.closeAllConnections();
|
|
32
|
+
}),
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function sameOrigin(request, host) {
|
|
36
|
+
const allowed = new Set([host, host.replace("127.0.0.1", "localhost")]);
|
|
37
|
+
if (!allowed.has(request.headers.host ?? ""))
|
|
38
|
+
return false;
|
|
39
|
+
if (request.method === "GET" || request.method === "HEAD")
|
|
40
|
+
return true;
|
|
41
|
+
const origin = request.headers.origin;
|
|
42
|
+
return origin !== undefined && [...allowed].some((name) => origin === `http://${name}`);
|
|
43
|
+
}
|
|
44
|
+
function forward(request, response, target, credentials) {
|
|
45
|
+
const headers = {};
|
|
46
|
+
for (const [name, value] of Object.entries(request.headers))
|
|
47
|
+
if (value !== undefined && !DROPPED.has(name))
|
|
48
|
+
headers[name] = value;
|
|
49
|
+
const secure = target.protocol === "https:";
|
|
50
|
+
const upstream = (secure ? httpsRequest : httpRequest)({
|
|
51
|
+
host: target.hostname,
|
|
52
|
+
port: target.port === "" ? (secure ? 443 : 80) : Number(target.port),
|
|
53
|
+
path: `${target.pathname.replace(/\/$/, "")}${request.url ?? "/"}`,
|
|
54
|
+
method: request.method,
|
|
55
|
+
headers: { ...headers, ...credentials, host: target.host },
|
|
56
|
+
}, (answer) => {
|
|
57
|
+
response.writeHead(answer.statusCode ?? 502, answer.headers);
|
|
58
|
+
answer.pipe(response);
|
|
59
|
+
});
|
|
60
|
+
upstream.on("error", () => {
|
|
61
|
+
if (!response.headersSent)
|
|
62
|
+
response.writeHead(502, { "content-type": "text/plain; charset=utf-8" });
|
|
63
|
+
response.end("The runner's Observer did not answer. The run may still be going; retry shortly.\n");
|
|
64
|
+
});
|
|
65
|
+
request.pipe(upstream);
|
|
66
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { SandboxProviderId } from "../run/sandbox-receipts.js";
|
|
2
|
+
import type { StudyConfig } from "../study/types.js";
|
|
3
|
+
import type { StudyPlan } from "../study/plan-types.js";
|
|
4
|
+
export interface RemotePlan {
|
|
5
|
+
readonly provider: SandboxProviderId;
|
|
6
|
+
/** The runner's lifetime: setup, the participants' last possible end, analysis and teardown. */
|
|
7
|
+
readonly runnerMs: number;
|
|
8
|
+
/** Paths relative to the project, uploaded to the same place in the runner's project. */
|
|
9
|
+
readonly uploads: readonly string[];
|
|
10
|
+
/** Directories, relative to the project, uploaded whole when they exist. */
|
|
11
|
+
readonly uploadDirectories: readonly string[];
|
|
12
|
+
/** The environment names the runner's run may receive, and nothing else. */
|
|
13
|
+
readonly envNames: readonly string[];
|
|
14
|
+
}
|
|
15
|
+
export type RemotePlanResult = {
|
|
16
|
+
ok: true;
|
|
17
|
+
plan: RemotePlan;
|
|
18
|
+
} | {
|
|
19
|
+
ok: false;
|
|
20
|
+
reason: string;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* The remote plan for `config`, whose live plan is `plan`, or why it cannot run remotely.
|
|
24
|
+
* `studyPath` is the study file, `cwd` the project it runs in. `ceilingMs` is the longest the
|
|
25
|
+
* provider keeps a sandbox.
|
|
26
|
+
*/
|
|
27
|
+
export declare function planRemoteRun(args: {
|
|
28
|
+
config: StudyConfig;
|
|
29
|
+
plan: StudyPlan;
|
|
30
|
+
cwd: string;
|
|
31
|
+
studyPath: string;
|
|
32
|
+
ceilingMs: number;
|
|
33
|
+
/** How many sandboxes the E2B account runs at once; the runner takes one of them. */
|
|
34
|
+
concurrentSandboxes: number;
|
|
35
|
+
}): RemotePlanResult;
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Whether a study can run on a remote machine, the runner sandbox, and what the runner needs: how
|
|
2
|
+
// long it must live, which files it must receive and which keys it may hold. A pure function of the
|
|
3
|
+
// study and its plan, so a refusal happens before anything is created.
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { keyNamesOf } from "../study/requirements.js";
|
|
6
|
+
import { participantList } from "../study/study-fields.js";
|
|
7
|
+
/** Time the runner needs before the study starts: create, Node and humanish install, uploads. */
|
|
8
|
+
const RUNNER_SETUP_MS = 10 * 60_000;
|
|
9
|
+
/** Time the runner needs after the study ends: syncing the run back and a margin. */
|
|
10
|
+
const RUNNER_TEARDOWN_MS = 15 * 60_000;
|
|
11
|
+
/** Settings the plan reads from the environment, which the remote run must read the same way. */
|
|
12
|
+
const PLAN_SETTINGS = [
|
|
13
|
+
"HUMANISH_E2B_MAX_SANDBOX_MINUTES",
|
|
14
|
+
"HUMANISH_E2B_MAX_CONCURRENT_SANDBOXES",
|
|
15
|
+
"HUMANISH_CUA_MAX_CONCURRENCY",
|
|
16
|
+
"HUMANISH_E2B_REQUEST_TIMEOUT_MS",
|
|
17
|
+
"E2B_DOMAIN",
|
|
18
|
+
];
|
|
19
|
+
/** The connection names each provider's runner and participants are reached with. */
|
|
20
|
+
const PROVIDER_KEYS = {
|
|
21
|
+
e2b: ["E2B_API_KEY"],
|
|
22
|
+
modal: ["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "MODAL_ENVIRONMENT"],
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* The remote plan for `config`, whose live plan is `plan`, or why it cannot run remotely.
|
|
26
|
+
* `studyPath` is the study file, `cwd` the project it runs in. `ceilingMs` is the longest the
|
|
27
|
+
* provider keeps a sandbox.
|
|
28
|
+
*/
|
|
29
|
+
export function planRemoteRun(args) {
|
|
30
|
+
const { config, plan, cwd } = args;
|
|
31
|
+
const refuse = (reason) => ({ ok: false, reason });
|
|
32
|
+
if (plan.route !== "computer-use")
|
|
33
|
+
return refuse(`a ${plan.route} study runs on this machine; only computer-use studies run remotely.`);
|
|
34
|
+
const runner = plan.runner;
|
|
35
|
+
if (runner.desktop !== "e2b-desktop")
|
|
36
|
+
return refuse("its participants use this machine's browser, so the study runs here.");
|
|
37
|
+
if (runner.brain.kind !== "openai")
|
|
38
|
+
return refuse("its participants are driven by an agent on this machine, which a remote run cannot reach.");
|
|
39
|
+
if (runner.subject.kind !== "app-url")
|
|
40
|
+
return refuse("its subject is built or served from this machine; only app-url subjects run remotely for now.");
|
|
41
|
+
if (servedHere(runner.subject.appUrl))
|
|
42
|
+
return refuse("its app URL points at this machine or a private network, which the runner cannot reach.");
|
|
43
|
+
if (plan.analysis?.config.provider === "codex")
|
|
44
|
+
return refuse("its analysis uses this machine's Codex login.");
|
|
45
|
+
if (config.review?.scorer !== undefined)
|
|
46
|
+
return refuse("it loads a scorer module, which does not run remotely yet.");
|
|
47
|
+
if (plan.rerun !== undefined)
|
|
48
|
+
return refuse("a rerun reads a run on this machine.");
|
|
49
|
+
const study = inside(cwd, args.studyPath);
|
|
50
|
+
if (study === undefined)
|
|
51
|
+
return refuse("the study file is outside the project directory.");
|
|
52
|
+
const files = [study];
|
|
53
|
+
for (const participant of participantList(config) ?? [])
|
|
54
|
+
for (const file of participant.files ?? []) {
|
|
55
|
+
const from = inside(cwd, path.resolve(path.dirname(args.studyPath), file.from));
|
|
56
|
+
if (from === undefined)
|
|
57
|
+
return refuse(`participant file ${file.from} is outside the project directory.`);
|
|
58
|
+
files.push(from);
|
|
59
|
+
}
|
|
60
|
+
if (runner.provider === "e2b" && plan.concurrency + 1 > args.concurrentSandboxes)
|
|
61
|
+
return refuse(`its ${plan.concurrency} participants at once and the runner need ${plan.concurrency + 1} E2B sandboxes at once, more than the ${args.concurrentSandboxes} your account runs. Lower execution.concurrency to ${args.concurrentSandboxes - 1}.`);
|
|
62
|
+
const runnerMs = runnerLifetimeMs(plan);
|
|
63
|
+
if (runnerMs > args.ceilingMs)
|
|
64
|
+
return refuse(`the study needs its runner for up to ${minutes(runnerMs)} minutes, longer than the ${minutes(args.ceilingMs)} minutes ${runner.provider === "modal" ? "Modal" : "E2B"} keeps a sandbox.`);
|
|
65
|
+
return {
|
|
66
|
+
ok: true,
|
|
67
|
+
plan: {
|
|
68
|
+
provider: runner.provider,
|
|
69
|
+
runnerMs,
|
|
70
|
+
uploads: [...new Set(files)],
|
|
71
|
+
uploadDirectories: ["humanish/personas", ".humanish/local/personas"],
|
|
72
|
+
envNames: [
|
|
73
|
+
...new Set([...keyNamesOf(plan), ...PROVIDER_KEYS[runner.provider], ...PLAN_SETTINGS]),
|
|
74
|
+
].sort(),
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/** Setup, the participants' last possible end, the analysis's own limit and teardown. */
|
|
79
|
+
function runnerLifetimeMs(plan) {
|
|
80
|
+
const analysisMs = plan.analysis?.config.timeoutMs ?? 0;
|
|
81
|
+
return RUNNER_SETUP_MS + plan.arrivals.lastEndMs + analysisMs + RUNNER_TEARDOWN_MS;
|
|
82
|
+
}
|
|
83
|
+
/** `target` relative to `cwd`, or undefined when it is outside it. */
|
|
84
|
+
function inside(cwd, target) {
|
|
85
|
+
const relative = path.relative(path.resolve(cwd), path.resolve(cwd, target));
|
|
86
|
+
if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative))
|
|
87
|
+
return undefined;
|
|
88
|
+
return relative.split(path.sep).join("/");
|
|
89
|
+
}
|
|
90
|
+
const minutes = (ms) => Math.ceil(ms / 60_000);
|
|
91
|
+
/** Whether `appUrl` names loopback, a private address, or a name only this machine resolves. */
|
|
92
|
+
function servedHere(appUrl) {
|
|
93
|
+
let host;
|
|
94
|
+
try {
|
|
95
|
+
host = new URL(appUrl).hostname.replace(/^\[|\]$/g, "").toLowerCase();
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local"))
|
|
101
|
+
return true;
|
|
102
|
+
if (host === "::1" || host.startsWith("fc") || host.startsWith("fd") || host.startsWith("fe80:"))
|
|
103
|
+
return host.includes(":");
|
|
104
|
+
const octets = host.split(".").map(Number);
|
|
105
|
+
if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet)))
|
|
106
|
+
return false;
|
|
107
|
+
const [a, b] = octets;
|
|
108
|
+
return (a === 127 ||
|
|
109
|
+
a === 10 ||
|
|
110
|
+
a === 0 ||
|
|
111
|
+
(a === 172 && b >= 16 && b <= 31) ||
|
|
112
|
+
(a === 192 && b === 168) ||
|
|
113
|
+
(a === 169 && b === 254));
|
|
114
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
2
|
+
import { type SandboxProviderId } from "../run/sandbox-receipts.js";
|
|
3
|
+
import { type RemoteContext, type RemoteOutcome } from "./launch.js";
|
|
4
|
+
/** A run directory here whose run executes in a runner and has not been copied back. */
|
|
5
|
+
export interface RemoteRunHere {
|
|
6
|
+
paths: PreparedRunArtifactPaths;
|
|
7
|
+
runId: string;
|
|
8
|
+
provider: SandboxProviderId;
|
|
9
|
+
runnerId: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The remote run `runId` names, when its directory here has a remote.json and no run.json yet;
|
|
13
|
+
* undefined for any other run, which the Observer serves from its files as before.
|
|
14
|
+
*/
|
|
15
|
+
export declare function remoteRunHere(cwd: string, runId: string): Promise<RemoteRunHere | undefined>;
|
|
16
|
+
/** Reconnect to `run`'s runner, serve its Observer on 127.0.0.1:`port` and follow it to the end. */
|
|
17
|
+
export declare function reattachRemoteRun(context: RemoteContext, run: RemoteRunHere, port: number): Promise<RemoteOutcome>;
|