humanish 0.140.0 → 0.142.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/ARCHITECTURE.md +1 -0
  2. package/CHANGELOG.md +23 -0
  3. package/README.md +2 -0
  4. package/dist/actors/codex/restricted-participant.js +1 -1
  5. package/dist/actors/computer-use/external-handoff.d.ts +22 -0
  6. package/dist/actors/computer-use/external-handoff.js +83 -0
  7. package/dist/actors/computer-use/loop/observation.d.ts +7 -0
  8. package/dist/actors/computer-use/loop/observation.js +17 -1
  9. package/dist/actors/computer-use/loop/types.d.ts +6 -0
  10. package/dist/analysis/admission.d.ts +8 -0
  11. package/dist/analysis/admission.js +15 -4
  12. package/dist/analysis/analysis-limits.d.ts +6 -0
  13. package/dist/analysis/analysis-limits.js +6 -0
  14. package/dist/analysis/analysis-lock.d.ts +8 -0
  15. package/dist/analysis/analysis-lock.js +129 -0
  16. package/dist/analysis/automatic-completion.js +7 -0
  17. package/dist/analysis/automatic-config.d.ts +5 -2
  18. package/dist/analysis/automatic-config.js +23 -8
  19. package/dist/analysis/automatic.js +13 -7
  20. package/dist/analysis/execute.d.ts +17 -6
  21. package/dist/analysis/execute.js +37 -19
  22. package/dist/analysis/gemini-wire.d.ts +8 -0
  23. package/dist/analysis/gemini-wire.js +165 -0
  24. package/dist/analysis/image-tokens.d.ts +4 -2
  25. package/dist/analysis/image-tokens.js +24 -3
  26. package/dist/analysis/job.d.ts +2 -0
  27. package/dist/analysis/job.js +7 -2
  28. package/dist/analysis/openai-wire.d.ts +5 -0
  29. package/dist/analysis/openai-wire.js +101 -0
  30. package/dist/analysis/provider.d.ts +15 -7
  31. package/dist/analysis/provider.js +21 -101
  32. package/dist/analysis/recording-ended.d.ts +16 -0
  33. package/dist/analysis/recording-ended.js +29 -0
  34. package/dist/analysis/responses.d.ts +3 -0
  35. package/dist/analysis/responses.js +19 -0
  36. package/dist/analysis/service.d.ts +1 -3
  37. package/dist/analysis/service.js +34 -53
  38. package/dist/analysis/source.d.ts +1 -1
  39. package/dist/analysis/types.d.ts +8 -2
  40. package/dist/analysis/validation.d.ts +1 -0
  41. package/dist/analysis/validation.js +13 -12
  42. package/dist/browser-control/protocol.d.ts +8 -0
  43. package/dist/browser-control/protocol.js +21 -0
  44. package/dist/cli/commands/analysis-signals.d.ts +10 -4
  45. package/dist/cli/commands/analysis-signals.js +28 -6
  46. package/dist/cli/commands/analyze.js +9 -7
  47. package/dist/cli/commands/observe.js +26 -1
  48. package/dist/cli/commands/run-command.d.ts +1 -1
  49. package/dist/cli/commands/run-command.js +12 -0
  50. package/dist/cli/commands/runs.js +8 -2
  51. package/dist/cli/commands/study-remote.d.ts +13 -0
  52. package/dist/cli/commands/study-remote.js +125 -0
  53. package/dist/cli/commands/study-route-computer-use.js +28 -4
  54. package/dist/cli/commands/study-route-scripted.js +3 -2
  55. package/dist/cli/commands/study-route-shared-world.js +3 -2
  56. package/dist/cli/commands/study-route-terminal.js +3 -2
  57. package/dist/cli/commands/study-run.js +34 -0
  58. package/dist/cli/commands/watch.js +5 -0
  59. package/dist/cli/doctor.js +7 -2
  60. package/dist/cli/findings.d.ts +30 -0
  61. package/dist/cli/findings.js +114 -15
  62. package/dist/cli/io.d.ts +8 -0
  63. package/dist/cli/io.js +3 -1
  64. package/dist/evidence/browser-hygiene.d.ts +12 -1
  65. package/dist/evidence/browser-hygiene.js +45 -2
  66. package/dist/feedback/export-bundle.js +1 -1
  67. package/dist/guest/external-links.d.ts +14 -0
  68. package/dist/guest/external-links.js +48 -0
  69. package/dist/guest/runtime-desktop.js +5 -2
  70. package/dist/keys/key-resolution.d.ts +10 -0
  71. package/dist/keys/key-resolution.js +33 -5
  72. package/dist/observer/data.d.ts +4 -0
  73. package/dist/observer/listen.d.ts +7 -2
  74. package/dist/observer/listen.js +10 -4
  75. package/dist/observer/render.d.ts +2 -0
  76. package/dist/observer/render.js +1 -1
  77. package/dist/observer/run-routes.js +12 -3
  78. package/dist/observer-app.html +2 -2
  79. package/dist/remote/launch.d.ts +53 -0
  80. package/dist/remote/launch.js +320 -0
  81. package/dist/remote/observer-proxy.d.ts +11 -0
  82. package/dist/remote/observer-proxy.js +66 -0
  83. package/dist/remote/plan.d.ts +35 -0
  84. package/dist/remote/plan.js +114 -0
  85. package/dist/remote/reattach.d.ts +17 -0
  86. package/dist/remote/reattach.js +81 -0
  87. package/dist/remote/runner.d.ts +68 -0
  88. package/dist/remote/runner.js +216 -0
  89. package/dist/routes/computer-use/e2b-desktop/desktop.js +27 -17
  90. package/dist/routes/computer-use/e2b-desktop/start.js +1 -1
  91. package/dist/routes/computer-use/e2b-desktop/teardown.js +1 -1
  92. package/dist/routes/computer-use/in-process-desktop.js +2 -2
  93. package/dist/routes/computer-use/live-phase.js +1 -1
  94. package/dist/routes/computer-use/setup.js +2 -2
  95. package/dist/routes/computer-use/types.d.ts +2 -0
  96. package/dist/routes/scripted/types.d.ts +2 -0
  97. package/dist/routes/shared-world/route.js +1 -1
  98. package/dist/routes/shared-world/types.d.ts +2 -0
  99. package/dist/routes/terminal/types.d.ts +2 -0
  100. package/dist/run/costs.d.ts +2 -1
  101. package/dist/run/display.d.ts +6 -1
  102. package/dist/run/display.js +11 -0
  103. package/dist/run/host-suspension.js +4 -1
  104. package/dist/run/preflight-receipts.d.ts +3 -12
  105. package/dist/run/preflight-receipts.js +7 -67
  106. package/dist/run/pricing.d.ts +13 -1
  107. package/dist/run/pricing.js +50 -4
  108. package/dist/run/process-owner.d.ts +28 -0
  109. package/dist/run/process-owner.js +83 -0
  110. package/dist/run/projection.js +1 -1
  111. package/dist/run/reclaim.d.ts +6 -0
  112. package/dist/run/reclaim.js +32 -2
  113. package/dist/run/results.d.ts +1 -1
  114. package/dist/run/route-shell.js +1 -1
  115. package/dist/run/run-cost.d.ts +2 -1
  116. package/dist/run/run-cost.js +6 -1
  117. package/dist/run/run-index.d.ts +3 -1
  118. package/dist/run/run-index.js +23 -11
  119. package/dist/run/run.d.ts +29 -5
  120. package/dist/run/run.js +43 -17
  121. package/dist/run/sandbox-creates.d.ts +10 -3
  122. package/dist/run/sandbox-creates.js +48 -6
  123. package/dist/run/status.d.ts +57 -2
  124. package/dist/run/status.js +87 -1
  125. package/dist/run/stored-runs.d.ts +3 -0
  126. package/dist/run/stored-runs.js +9 -4
  127. package/dist/run-study.d.ts +5 -0
  128. package/dist/study/doctor.js +14 -1
  129. package/dist/study/requirements.d.ts +1 -1
  130. package/dist/study/requirements.js +8 -6
  131. package/dist/study/route-inputs.js +4 -0
  132. package/dist/study/run-study-events.d.ts +8 -2
  133. package/dist/study/run-study-events.js +9 -2
  134. package/dist/substrates/e2b/base-module.d.ts +3 -0
  135. package/dist/substrates/e2b/base-module.js +40 -0
  136. package/dist/substrates/e2b/desktop-browser.d.ts +1 -1
  137. package/dist/substrates/e2b/desktop-browser.js +27 -7
  138. package/dist/substrates/e2b/desktop-executor.d.ts +7 -0
  139. package/dist/substrates/e2b/desktop-executor.js +3 -0
  140. package/dist/substrates/e2b/desktop-handoff.d.ts +22 -0
  141. package/dist/substrates/e2b/desktop-handoff.js +86 -0
  142. package/dist/substrates/e2b/sdk.d.ts +24 -0
  143. package/dist/substrates/e2b/sdk.js +5 -1
  144. package/dist/substrates/e2b/shell-only.d.ts +5 -0
  145. package/dist/substrates/e2b/shell-only.js +27 -0
  146. package/dist/substrates/local/runtime-release.js +8 -8
  147. package/dist/substrates/modal/sandbox.js +27 -28
  148. package/dist/substrates/modal/sdk.d.ts +7 -0
  149. package/dist/tui-app.js +115 -115
  150. package/docs/architecture/observer.md +7 -3
  151. package/docs/contracts/schemas.md +24 -3
  152. package/docs/contracts/study-analysis.md +52 -4
  153. package/docs/product/automatic-analysis.md +55 -9
  154. package/package.json +2 -2
  155. package/skills/humanish/SKILL.md +118 -5
@@ -6,8 +6,8 @@
6
6
  // key (`key-one-of`) stays with buildRuntimeAuth at run time, which also picks its placement.
7
7
  import { detectLocalAgents } from "../actors/local-agent/cli.js";
8
8
  import { localAgentRefusal } from "../actors/local-agent/readiness.js";
9
- import { describeMissingKeys } from "../keys/key-resolution.js";
10
- import { MODEL_RATES, unpricedCapMessage } from "../run/pricing.js";
9
+ import { analysisKeyNames, describeMissingKeys } from "../keys/key-resolution.js";
10
+ import { participantRate, unpricedCapMessage } from "../run/pricing.js";
11
11
  /** Provider keys in the order the routes name them in a refusal. */
12
12
  const KEY_ORDER = [
13
13
  "OPENAI_API_KEY",
@@ -45,7 +45,7 @@ export function requiredSubjectEnv(requirements) {
45
45
  /**
46
46
  * The provider keys a live run of `plan` is known to read: its `key` and `key-one-of` names, its
47
47
  * subject env, the external catch's token variable, ANTHROPIC_API_KEY for a Claude Code
48
- * participant, and OPENAI_API_KEY when automatic analysis runs on OpenAI. `run` prints a
48
+ * participant, and the analyst's key when automatic analysis runs on OpenAI or Gemini. `run` prints a
49
49
  * key-source line for these only. Discovery still fills every key it finds: other readers, such as
50
50
  * a declared scorer's host code, are not in the plan.
51
51
  */
@@ -63,8 +63,10 @@ export function keyNamesOf(plan) {
63
63
  const catchToken = plan.residual.comms?.email?.external?.authTokenEnv;
64
64
  if (catchToken !== undefined)
65
65
  names.add(catchToken);
66
- if (plan.analysis !== undefined && plan.analysis.config.provider !== "codex")
67
- names.add("OPENAI_API_KEY");
66
+ const analyst = plan.analysis?.config.provider;
67
+ if (plan.analysis !== undefined && analyst !== "codex")
68
+ for (const name of analysisKeyNames(analyst ?? "openai"))
69
+ names.add(name);
68
70
  return names;
69
71
  }
70
72
  /**
@@ -140,7 +142,7 @@ export function unpricedCapCheck(args) {
140
142
  const { caps, model } = args;
141
143
  if (caps.maxUsd === undefined && caps.maxTotalUsd === undefined)
142
144
  return undefined;
143
- if (MODEL_RATES[model.trim().toLowerCase()])
145
+ if (participantRate(model))
144
146
  return undefined;
145
147
  return { code: args.code, message: unpricedCapMessage(model) };
146
148
  }
@@ -10,6 +10,7 @@ export function computerUseInput(options, deps, emit) {
10
10
  ...(options.open === undefined ? {} : { open: options.open }),
11
11
  ...(options.onObserverReady === undefined ? {} : { onObserverReady: options.onObserverReady }),
12
12
  ...(options.runId === undefined ? {} : { runId: options.runId }),
13
+ ...(options.ownerKey === undefined ? {} : { ownerKey: options.ownerKey }),
13
14
  ...(options.rerun === undefined ? {} : { rerun: options.rerun }),
14
15
  ...(options.env === undefined ? {} : { env: options.env }),
15
16
  ...(options.prepareDesktop === undefined ? {} : { prepareDesktop: options.prepareDesktop }),
@@ -28,6 +29,7 @@ export function scriptedInput(options, deps, emit) {
28
29
  cwd: options.cwd,
29
30
  ...(options.open === undefined ? {} : { open: options.open }),
30
31
  ...(options.runId === undefined ? {} : { runId: options.runId }),
32
+ ...(options.ownerKey === undefined ? {} : { ownerKey: options.ownerKey }),
31
33
  ...(options.env === undefined ? {} : { env: options.env }),
32
34
  ...(options.prepareDesktop === undefined ? {} : { prepareDesktop: options.prepareDesktop }),
33
35
  deps,
@@ -40,6 +42,7 @@ export function terminalInput(options, deps, emit) {
40
42
  cwd: options.cwd,
41
43
  ...(options.open === undefined ? {} : { open: options.open }),
42
44
  ...(options.runId === undefined ? {} : { runId: options.runId }),
45
+ ...(options.ownerKey === undefined ? {} : { ownerKey: options.ownerKey }),
43
46
  ...(options.env === undefined ? {} : { env: options.env }),
44
47
  deps,
45
48
  ...scorerOf(options, terminalRouteScorer),
@@ -52,6 +55,7 @@ export function sharedWorldInput(options, deps, emit) {
52
55
  ...(options.open === undefined ? {} : { open: options.open }),
53
56
  ...(options.onObserverReady === undefined ? {} : { onObserverReady: options.onObserverReady }),
54
57
  ...(options.runId === undefined ? {} : { runId: options.runId }),
58
+ ...(options.ownerKey === undefined ? {} : { ownerKey: options.ownerKey }),
55
59
  ...(options.env === undefined ? {} : { env: options.env }),
56
60
  ...(options.prepareDesktop === undefined ? {} : { prepareDesktop: options.prepareDesktop }),
57
61
  ...observersOf(options, emit),
@@ -19,7 +19,8 @@ export type SetupTarget = {
19
19
  readonly participant: ParticipantRef;
20
20
  };
21
21
  /**
22
- * What a run reports while it runs. `plan` comes from computer use only; the other routes run from
22
+ * What a run reports while it runs. `run-started` comes from every route that records a run, once
23
+ * its run id exists. `plan` comes from computer use only; the other routes run from
23
24
  * the study plan but do not emit it. `subject-phase` comes from computer use (participant target)
24
25
  * and shared world (subject target).
25
26
  */
@@ -40,6 +41,11 @@ export type StudyEvent = {
40
41
  at: string;
41
42
  ok?: boolean;
42
43
  durationMs?: number;
44
+ }
45
+ /** The run's directory and status.json exist: `runId` names it from here on. */
46
+ | {
47
+ type: "run-started";
48
+ runId: string;
43
49
  } | {
44
50
  type: "analysis-started";
45
51
  } | {
@@ -48,7 +54,7 @@ export type StudyEvent = {
48
54
  /**
49
55
  * The literal values to scrub from an onEvent warning: the provider keys and the declared subject
50
56
  * env from every env the run could read (the `env` option and process.env), and the
51
- * analysis API key. A callback can hold any of them, and its warning is appended after the route
57
+ * analysis API keys, OpenAI's and Gemini's. A callback can hold any of them, and its warning is appended after the route
52
58
  * sanitized its own.
53
59
  */
54
60
  export declare function knownSecrets(config: StudyConfig, options: InternalRunStudyOptions, forwardedEnv: Readonly<Record<string, string | undefined>> | undefined): RunSecrets;
@@ -1,17 +1,24 @@
1
1
  // What a run reports through RunStudyOptions.onEvent, and how a callback failure becomes a run
2
2
  // warning. normalizeRunStudyOptions (run-study-options.ts) builds the emitter and hands it to the
3
3
  // route hook bags it maps; the event values are built here.
4
+ import { analysisKeyNames } from "../keys/key-resolution.js";
4
5
  import { redactText, toErrorMessage } from "../evidence/redaction.js";
5
6
  import { RunSecrets } from "../run/secrets.js";
6
7
  /**
7
8
  * The literal values to scrub from an onEvent warning: the provider keys and the declared subject
8
9
  * env from every env the run could read (the `env` option and process.env), and the
9
- * analysis API key. A callback can hold any of them, and its warning is appended after the route
10
+ * analysis API keys, OpenAI's and Gemini's. A callback can hold any of them, and its warning is appended after the route
10
11
  * sanitized its own.
11
12
  */
12
13
  export function knownSecrets(config, options, forwardedEnv) {
13
14
  const sources = [forwardedEnv, options.env, process.env];
14
- const names = ["OPENAI_API_KEY", "E2B_API_KEY", "CODEX_API_KEY", ...(config.subject.env ?? [])];
15
+ const names = [
16
+ "OPENAI_API_KEY",
17
+ "E2B_API_KEY",
18
+ "CODEX_API_KEY",
19
+ ...analysisKeyNames("gemini"),
20
+ ...(config.subject.env ?? []),
21
+ ];
15
22
  const secrets = new RunSecrets([]);
16
23
  for (const env of sources)
17
24
  for (const name of names)
@@ -0,0 +1,3 @@
1
+ import { type E2BDesktopModule } from "./sdk.js";
2
+ /** A sandbox port that creates and connects to E2B base-template sandboxes. */
3
+ export declare function loadE2BBaseModule(): Promise<E2BDesktopModule>;
@@ -0,0 +1,40 @@
1
+ // E2B's base sandbox, the `base` template with no desktop, behind the same sandbox port as the
2
+ // desktop, so acquire(), receipts, release and reclaim work on it unchanged. A sandbox that runs
3
+ // humanish itself needs commands, files and one authenticated port; the desktop template would
4
+ // start an X session nobody uses and bill its 8 vCPU.
5
+ //
6
+ // @e2b/desktop re-exports the e2b SDK but replaces its Sandbox with the desktop subclass, and a
7
+ // strict install does not let humanish import `e2b`, which is only @e2b/desktop's dependency. The
8
+ // base class is the desktop class's parent.
9
+ import { guardDesktopSandboxCreate, importE2BDesktop, } from "./sdk.js";
10
+ import { shellOnlySurface } from "./shell-only.js";
11
+ /** The header E2B's proxy reads a sandbox's traffic access token from (spike, 2026-10-21). */
12
+ const TRAFFIC_TOKEN_HEADER = "e2b-traffic-access-token";
13
+ class E2BDesktopOnlyError extends Error {
14
+ constructor(what) {
15
+ super(`${what} needs a desktop sandbox; this E2B sandbox runs the base template.`);
16
+ this.name = "E2BDesktopOnlyError";
17
+ }
18
+ }
19
+ /** A sandbox port that creates and connects to E2B base-template sandboxes. */
20
+ export async function loadE2BBaseModule() {
21
+ return guardDesktopSandboxCreate({ Sandbox: baseSandbox(await importE2BDesktop()) });
22
+ }
23
+ function baseSandbox(desktop) {
24
+ const Base = Object.getPrototypeOf(desktop.Sandbox);
25
+ if (typeof Base.create !== "function")
26
+ throw new Error("The installed @e2b/desktop does not extend the e2b SDK's Sandbox; update @e2b/desktop.");
27
+ class BaseSandbox extends Base {
28
+ constructor(...args) {
29
+ super(...args);
30
+ Object.assign(this, shellOnlySurface((what) => new E2BDesktopOnlyError(what)));
31
+ }
32
+ async portAccess(port) {
33
+ const token = this.trafficAccessToken;
34
+ if (token === undefined || token === "")
35
+ throw new Error("This E2B sandbox accepts public traffic, so it has no traffic access token. Create it with network.allowPublicTraffic set to false.");
36
+ return { url: `https://${this.getHost(port)}`, headers: { [TRAFFIC_TOKEN_HEADER]: token } };
37
+ }
38
+ }
39
+ return BaseSandbox;
40
+ }
@@ -81,4 +81,4 @@ extraChromiumFlags?: readonly string[], environment?: Readonly<Record<string, st
81
81
  * from it run in that zone.
82
82
  */
83
83
  export declare function openDesktopTerminal(desktop: E2BDesktopSandbox, requestTimeoutMs: number, workdir: string | undefined, timeZone?: string): Promise<void>;
84
- export declare function startDesktopStream(desktop: E2BDesktopSandbox, browserWindowId: string | undefined): Promise<void>;
84
+ export declare function startDesktopStream(desktop: E2BDesktopSandbox, browserWindowId: string | undefined, requestTimeoutMs: number): Promise<void>;
@@ -5,6 +5,7 @@ import { failureTail } from "../../evidence/redaction.js";
5
5
  import { isHttpUrl } from "../../study/parse/subject.js";
6
6
  import { runDetachedStep } from "../detached.js";
7
7
  import { shellQuote } from "../shell.js";
8
+ import { externalLinkHandlerSetup } from "./desktop-handoff.js";
8
9
  import { e2bShell } from "./shell.js";
9
10
  // Settle after opening the browser or the terminal, before the first screenshot: long enough for
10
11
  // a cold browser and page load to paint (2 s captured a blank desktop; the render needs 6-9 s).
@@ -210,7 +211,8 @@ extraChromiumFlags = [], environment) {
210
211
  `target_url=${shellQuote(targetUrl)}`,
211
212
  `browser_preference=${shellQuote(requestedBrowser)}`,
212
213
  "profile_dir=",
213
- `chrome_preferences_json=${shellQuote(chromiumEvidenceProfilePreferencesJson())}`,
214
+ `chrome_preferences_json=${shellQuote(chromiumEvidenceProfilePreferencesJson(targetUrl))}`,
215
+ externalLinkHandlerSetup(),
214
216
  "prepare_profile() {",
215
217
  ' profile_dir="$(mktemp -d /tmp/humanish-chrome-profile.XXXXXX)"',
216
218
  ' mkdir -p "$profile_dir/Default"',
@@ -400,15 +402,33 @@ export async function openDesktopTerminal(desktop, requestTimeoutMs, workdir, ti
400
402
  throw new Error("desktop-cli participant could not open a terminal on this desktop template");
401
403
  }
402
404
  }
403
- export async function startDesktopStream(desktop, browserWindowId) {
405
+ export async function startDesktopStream(desktop, browserWindowId, requestTimeoutMs) {
404
406
  if (!browserWindowId) {
405
407
  await desktop.stream.start({ requireAuth: true });
406
- return;
407
408
  }
408
- try {
409
- await desktop.stream.start({ requireAuth: true, windowId: browserWindowId });
409
+ else {
410
+ try {
411
+ await desktop.stream.start({ requireAuth: true, windowId: browserWindowId });
412
+ }
413
+ catch {
414
+ await desktop.stream.start({ requireAuth: true });
415
+ }
410
416
  }
411
- catch {
412
- await desktop.stream.start({ requireAuth: true });
417
+ await makeStreamViewOnly(desktop, requestTimeoutMs);
418
+ }
419
+ /**
420
+ * Switches the stream's VNC server to view-only and confirms it. Both providers start x11vnc
421
+ * without `-viewonly`, and the URL's `view_only` is a noVNC client setting, so anyone holding the
422
+ * URL could otherwise type into the participant's desktop. x11vnc takes the change over its X
423
+ * property channel and answers the query with `ans=viewonly:1`; anything else throws, and the
424
+ * caller treats the stream as unavailable.
425
+ */
426
+ async function makeStreamViewOnly(desktop, requestTimeoutMs) {
427
+ const shell = e2bShell(desktop);
428
+ const options = { timeoutMs: 20_000, requestTimeoutMs };
429
+ await shell.run("x11vnc -display :0 -sync -R viewonly", options);
430
+ const query = await shell.run("x11vnc -display :0 -Q viewonly", options);
431
+ if (!/^ans=viewonly:1$/m.test(query.stdout)) {
432
+ throw new Error("the live stream could not be made view-only");
413
433
  }
414
434
  }
@@ -1,4 +1,5 @@
1
1
  import type { CuaExecutor, CuaObservation } from "../../actors/computer-use/loop.js";
2
+ import type { ExternalHandoff } from "../../actors/computer-use/external-handoff.js";
2
3
  /**
3
4
  * The minimal slice of the @e2b/desktop Sandbox the executor depends on: a structural subset of
4
5
  * the SDK class (checked against 2.4.0), so a Sandbox satisfies it with no adapter. Methods are typed to return `Promise<void> | void` (and the
@@ -65,6 +66,12 @@ export interface E2BDesktopExecutorOptions {
65
66
  * never persists raw URL/title/text; it only uses them in memory to decide whether to stop.
66
67
  */
67
68
  observeBrowserState?: () => Promise<Pick<CuaObservation, "url" | "title" | "text" | "scrollY">>;
69
+ /**
70
+ * Optional read of the links the browser handed to another app since the last read
71
+ * (desktop-handoff.ts). It runs beside the browser probe and the screenshot; a read that fails
72
+ * leaves its links for the next one.
73
+ */
74
+ readExternalHandoffs?: () => Promise<readonly ExternalHandoff[]>;
68
75
  }
69
76
  /** The stage of a `type` action that failed. */
70
77
  export type CuaTypePhase = "desktop-write" | "text-tempfile" | "text-command";
@@ -151,10 +151,13 @@ export function createE2BDesktopExecutor(desktop, options = {}) {
151
151
  // Probe browser state before the screenshot so a deterministic stopWhen match
152
152
  // and its persisted frame describe the same settled surface as closely as the
153
153
  // desktop substrate allows.
154
+ const handoffs = options.readExternalHandoffs?.().catch(() => []);
154
155
  const browserState = await options.observeBrowserState?.().catch(() => undefined);
155
156
  const raw = await desktop.screenshot();
156
157
  const screenshot = toBuffer(raw);
158
+ const externalHandoffs = (await handoffs) ?? [];
157
159
  return {
160
+ ...(externalHandoffs.length === 0 ? {} : { externalHandoffs }),
158
161
  screenshot,
159
162
  stateSignature: perceptualSignature(screenshot),
160
163
  ...(browserState?.url === undefined ? {} : { url: browserState.url }),
@@ -0,0 +1,22 @@
1
+ import { type ExternalHandoff } from "../../actors/computer-use/external-handoff.js";
2
+ import { type Shell } from "../shell.js";
3
+ /**
4
+ * One stand-in, installed in `bin`, appending to `log` (a shell word). `$0` names the command it
5
+ * stands in for. The local guest image ships the same script
6
+ * (runtime/browser-guest/control/root/opt/humanish/control/bin/xdg-open).
7
+ */
8
+ export declare function externalLinkStandIn(bin: string, log: string): string;
9
+ /**
10
+ * Shell lines for the browser launch script, run before the browser starts: they install the
11
+ * stand-ins and put them first on the search path for every command the script starts after them.
12
+ */
13
+ export declare function externalLinkHandlerSetup(dir?: string): string;
14
+ /**
15
+ * Returns a read of the links handed off since the previous read, oldest first and at most
16
+ * EXTERNAL_HANDOFF_LIMITS.perObservation at a time. Reads run one at a time, so a retried
17
+ * observation cannot return a link twice. A line still being written is left for the next read.
18
+ */
19
+ export declare function externalHandoffReader(shell: Pick<Shell, "run">, options: {
20
+ requestTimeoutMs: number;
21
+ dir?: string;
22
+ }): () => Promise<ExternalHandoff[]>;
@@ -0,0 +1,86 @@
1
+ // Links the participant's browser hands to another app (mailto:, tel:, sms:) on a hosted desktop.
2
+ //
3
+ // Chrome on Linux opens such a link by running `xdg-email` (mailto) or `xdg-open` (the rest) by
4
+ // name, unless an XDG desktop portal answers on the session bus; the stock desktop runs none.
5
+ // The stock desktop has no mail, phone or messaging app, so `xdg-open` ended in XFCE's "Choose
6
+ // Preferred Application" dialog, which participants reported as a fault of the app under test.
7
+ // The browser launch puts stand-ins for both commands first on the browser's search path. A stand-in
8
+ // appends the link to a log and exits; a web or file link goes on to the desktop's own command.
9
+ // The executor reads the log on each observation, and the participant loop records each link
10
+ // and tells the participant what happened. Chrome's own "Open xdg-open?" prompt for tel: and
11
+ // the other non-mailto schemes is answered by the profile preference in
12
+ // src/evidence/browser-hygiene.ts; Chrome never prompts for mailto.
13
+ import { EXTERNAL_HANDOFF_LIMITS, externalHandoffOf, } from "../../actors/computer-use/external-handoff.js";
14
+ import { shellQuote } from "../shell.js";
15
+ /** Where the stand-ins and their log live on the desktop. */
16
+ const EXTERNAL_LINK_DIR = "/tmp/humanish-external-links";
17
+ const LINK_LENGTH = 4096;
18
+ /**
19
+ * One stand-in, installed in `bin`, appending to `log` (a shell word). `$0` names the command it
20
+ * stands in for. The local guest image ships the same script
21
+ * (runtime/browser-guest/control/root/opt/humanish/control/bin/xdg-open).
22
+ */
23
+ export function externalLinkStandIn(bin, log) {
24
+ return [
25
+ "#!/bin/sh",
26
+ "# Installed by humanish for a study participant's browser: this desktop has no app for this",
27
+ "# link, so the link is written down for the study instead of opening a chooser dialog.",
28
+ `bin=${shellQuote(bin)}`,
29
+ "url=",
30
+ 'for arg in "$@"; do case "$arg" in -*) ;; *) url=$arg; break ;; esac; done',
31
+ 'case "$url" in',
32
+ " [Hh][Tt][Tt][Pp]:*|[Hh][Tt][Tt][Pp][Ss]:*|[Ff][Ii][Ll][Ee]:*) ;;",
33
+ " [A-Za-z]*:*)",
34
+ ` printf '%s\\n' "$(printf '%s' "$url" | tr -d '\\r\\n' | cut -c 1-${LINK_LENGTH})" >> ${log}`,
35
+ " exit 0 ;;",
36
+ "esac",
37
+ // Anything else goes to the desktop's own command, found on the search path without this
38
+ // directory.
39
+ "set -f",
40
+ "real=",
41
+ "IFS=:",
42
+ 'for entry in $PATH; do [ "$entry" = "$bin" ] || real=${real:+$real:}$entry; done',
43
+ 'PATH=$real exec "${0##*/}" "$@"',
44
+ "",
45
+ ].join("\n");
46
+ }
47
+ /**
48
+ * Shell lines for the browser launch script, run before the browser starts: they install the
49
+ * stand-ins and put them first on the search path for every command the script starts after them.
50
+ */
51
+ export function externalLinkHandlerSetup(dir = EXTERNAL_LINK_DIR) {
52
+ const bin = `${dir}/bin`;
53
+ return [
54
+ `mkdir -p ${shellQuote(bin)}`,
55
+ `touch ${shellQuote(`${dir}/links.log`)}`,
56
+ `printf '%s' ${shellQuote(externalLinkStandIn(bin, shellQuote(`${dir}/links.log`)))} > ${shellQuote(`${bin}/xdg-open`)}`,
57
+ `cp ${shellQuote(`${bin}/xdg-open`)} ${shellQuote(`${bin}/xdg-email`)}`,
58
+ `chmod 755 ${shellQuote(`${bin}/xdg-open`)} ${shellQuote(`${bin}/xdg-email`)}`,
59
+ `export PATH=${shellQuote(bin)}:"$PATH"`,
60
+ ].join("\n");
61
+ }
62
+ /**
63
+ * Returns a read of the links handed off since the previous read, oldest first and at most
64
+ * EXTERNAL_HANDOFF_LIMITS.perObservation at a time. Reads run one at a time, so a retried
65
+ * observation cannot return a link twice. A line still being written is left for the next read.
66
+ */
67
+ export function externalHandoffReader(shell, options) {
68
+ const log = `${options.dir ?? EXTERNAL_LINK_DIR}/links.log`;
69
+ let consumed = 0;
70
+ let previous = Promise.resolve();
71
+ const read = async () => {
72
+ const result = await shell.run(`tail -n +${consumed + 1} ${shellQuote(log)} 2>/dev/null | head -n ${EXTERNAL_HANDOFF_LIMITS.perObservation}`, { requestTimeoutMs: options.requestTimeoutMs, timeoutMs: 10_000 });
73
+ const lines = result.stdout.split("\n");
74
+ lines.pop();
75
+ consumed += lines.length;
76
+ return lines.flatMap((line) => {
77
+ const handoff = externalHandoffOf(line);
78
+ return handoff === undefined ? [] : [handoff];
79
+ });
80
+ };
81
+ return () => {
82
+ const next = previous.then(read, read);
83
+ previous = next.catch(() => undefined);
84
+ return next;
85
+ };
86
+ }
@@ -30,6 +30,12 @@ export interface E2BDesktopModule {
30
30
  * (2.38 and later) returns a paginator.
31
31
  */
32
32
  list?(options?: E2BSandboxListOptions): E2BSandboxPaginator;
33
+ /**
34
+ * A handle on a sandbox that exists, by exact id, with the same calls a create returns. Throws
35
+ * a SandboxNotFoundError-shaped error when the id is gone. A process that did not create the
36
+ * sandbox uses it to run commands there and reach its ports.
37
+ */
38
+ connect?(sandboxId: string, options?: E2BRequestOptions): Promise<E2BDesktopSandbox>;
33
39
  };
34
40
  }
35
41
  /**
@@ -84,6 +90,17 @@ export interface E2BNetworkOptions {
84
90
  headers?: Record<string, string>;
85
91
  };
86
92
  }[]>;
93
+ /**
94
+ * False makes every URL `getHost` gives answer only requests that carry the sandbox's traffic
95
+ * access token. E2B's own default is true.
96
+ */
97
+ allowPublicTraffic?: boolean;
98
+ }
99
+ /** A URL that reaches a port inside a sandbox, and the headers the provider requires on each request. */
100
+ export interface SandboxPortAccess {
101
+ url: string;
102
+ /** Credentials: never persist or log them. */
103
+ headers: Record<string, string>;
87
104
  }
88
105
  /**
89
106
  * A create names its account. e2bAccount (connection.ts) gives every later call on the sandbox the
@@ -172,6 +189,11 @@ export interface E2BDesktopSandbox {
172
189
  * the concurrent backend fails closed when it is absent rather than calling a missing method.
173
190
  */
174
191
  getHost?(port: number): string;
192
+ /**
193
+ * The URL and credential headers that reach `port`, for a sandbox whose provider authenticates
194
+ * port traffic: an E2B sandbox created with `allowPublicTraffic: false`, or any Modal sandbox.
195
+ */
196
+ portAccess?(port: number): Promise<SandboxPortAccess>;
175
197
  screenshot(format?: "bytes"): Promise<Uint8Array>;
176
198
  wait(ms: number): Promise<void>;
177
199
  leftClick(x?: number, y?: number): Promise<void>;
@@ -202,6 +224,8 @@ export interface E2BDesktopSandbox {
202
224
  };
203
225
  }
204
226
  export declare function loadE2BDesktopModule(): Promise<E2BDesktopModule>;
227
+ /** The optional @e2b/desktop peer as it is, or an error that says how to install it. */
228
+ export declare function importE2BDesktop(): Promise<E2BDesktopModule>;
205
229
  export declare const DESKTOP_CREATE_CLEANUP_TIMEOUT_MS = 10000;
206
230
  /**
207
231
  * Call `observer` with the sandbox id as soon as the guarded SDK constructs the handle for a
@@ -8,8 +8,12 @@
8
8
  import { protectDesktopScreenshotCleanup } from "./desktop-screenshot-cleanup.js";
9
9
  import { desktopPeerAdvice, humanishInstall } from "./peer-install.js";
10
10
  export async function loadE2BDesktopModule() {
11
+ return guardDesktopSandboxCreate(await importE2BDesktop());
12
+ }
13
+ /** The optional @e2b/desktop peer as it is, or an error that says how to install it. */
14
+ export async function importE2BDesktop() {
11
15
  try {
12
- return guardDesktopSandboxCreate((await import("@e2b/desktop")));
16
+ return (await import("@e2b/desktop"));
13
17
  }
14
18
  catch (error) {
15
19
  if (isMissingE2BDesktopDependency(error)) {
@@ -0,0 +1,5 @@
1
+ import type { E2BDesktopSandbox } from "./sdk.js";
2
+ type DesktopSurface = Pick<E2BDesktopSandbox, "wait" | "launch" | "screenshot" | "leftClick" | "rightClick" | "middleClick" | "doubleClick" | "moveMouse" | "scroll" | "write" | "press" | "drag" | "stream">;
3
+ /** Desktop calls that reject with `unsupported(what)`; `wait` still waits. */
4
+ export declare function shellOnlySurface(unsupported: (what: string) => Error): DesktopSurface;
5
+ export {};
@@ -0,0 +1,27 @@
1
+ /** Desktop calls that reject with `unsupported(what)`; `wait` still waits. */
2
+ export function shellOnlySurface(unsupported) {
3
+ const reject = (what) => () => Promise.reject(unsupported(what));
4
+ return {
5
+ wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
6
+ launch: reject("A desktop application launch"),
7
+ screenshot: reject("A desktop screenshot"),
8
+ leftClick: reject("Desktop input"),
9
+ rightClick: reject("Desktop input"),
10
+ middleClick: reject("Desktop input"),
11
+ doubleClick: reject("Desktop input"),
12
+ moveMouse: reject("Desktop input"),
13
+ scroll: reject("Desktop input"),
14
+ write: reject("Desktop input"),
15
+ press: reject("Desktop input"),
16
+ drag: reject("Desktop input"),
17
+ stream: {
18
+ getAuthKey: () => {
19
+ throw unsupported("A desktop stream");
20
+ },
21
+ getUrl: () => {
22
+ throw unsupported("A desktop stream");
23
+ },
24
+ start: reject("A desktop stream"),
25
+ },
26
+ };
27
+ }
@@ -1,18 +1,18 @@
1
1
  /** Updated with the verified, public runtime artifact before release. */
2
2
  const LOCAL_RUNTIME_RELEASE = {
3
- url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.1/runtime-linux-amd64.tar.gz",
4
- sha256: "1de53db000f661e91ff6ea9a2eb8198b08b850621cf63d8ce7c1b046f5035faf",
5
- bytes: 597029770,
6
- image: "sha256:c805c310b7b086276efa74ea5735e8b559457c7edba85c5c9e3ac7934e8760f2",
3
+ url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.3/runtime-linux-amd64.tar.gz",
4
+ sha256: "6fc19792e9b8f414165a73470526cc414c98d0ad368210c5e95daf29469b8dbd",
5
+ bytes: 597049520,
6
+ image: "sha256:4672388737bd7d136910fca6f8b6d99e09fbe96c13e1b100491c05444e7d267a",
7
7
  };
8
8
  /** Architecture-specific artifacts; each retains matching sources and notices. */
9
9
  export const LOCAL_RUNTIME_RELEASES = {
10
10
  amd64: LOCAL_RUNTIME_RELEASE,
11
11
  arm64: {
12
- url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.2/runtime-linux-arm64.tar.gz",
13
- sha256: "ea7a42e12eae3b4d9140fff41e4e3512930202f9ccbb688497b146634d1cef10",
14
- bytes: 583555960,
15
- image: "sha256:e21c33b4745afdb3bac6a69c76606814348032ee661228c9bfdd9c7d06591a28",
12
+ url: "https://github.com/danielgwilson/humanish/releases/download/runtime-2026.10.21.4/runtime-linux-arm64.tar.gz",
13
+ sha256: "d590de49a86eabe1cd09d7556aa75b2ebd62bc2c88fe37122e2af315c478c901",
14
+ bytes: 583551803,
15
+ image: "sha256:4606122520504f7f587f52b872165e6ab900be0ba3eca933b2f6dd3b9dcde723",
16
16
  },
17
17
  };
18
18
  /** Media assets are separate so ordinary browser studies keep the smaller download. */
@@ -7,6 +7,7 @@
7
7
  // default gVisor runtime sets no_new_privs on every process, so the user's sudo, which the Node
8
8
  // bootstrap and product installs run, cannot work there.
9
9
  import { E2BDesktopStartupError, reportDesktopAllocation } from "../e2b/sdk.js";
10
+ import { shellOnlySurface } from "../e2b/shell-only.js";
10
11
  import { readBytes, readFile, runCommand, sandboxPath, wholeSeconds, writeFile, } from "./commands.js";
11
12
  import { MODAL_DESKTOP_DISPLAY, MODAL_STREAM_PORT, modalDesktopSurface, startModalDesktop, } from "./desktop.js";
12
13
  import { MODAL_SANDBOX_USER, modalDesktopImage, modalShellImage } from "./image.js";
@@ -70,9 +71,7 @@ export function modalSandboxModule(client, connection, profile) {
70
71
  ...(options.resolution === undefined ? {} : { resolution: options.resolution }),
71
72
  ...(options.dpi === undefined ? {} : { dpi: options.dpi }),
72
73
  });
73
- const tunnels = await sandbox.tunnels(30_000);
74
- const hosts = new Map(Object.entries(tunnels).map(([port, tunnel]) => [Number(port), new URL(tunnel.url).host]));
75
- return modalSandboxHandle(sandbox, profile, connection, hosts);
74
+ return modalSandboxHandle(sandbox, profile, connection, await tunnelHosts(sandbox));
76
75
  }
77
76
  catch (error) {
78
77
  const gone = await modalSandboxes(client, connection)
@@ -82,13 +81,33 @@ export function modalSandboxModule(client, connection, profile) {
82
81
  throw new E2BDesktopStartupError(error, gone);
83
82
  }
84
83
  }
84
+ async function connect(sandboxId, options) {
85
+ const ms = options?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
86
+ let sandbox;
87
+ try {
88
+ sandbox = await within(client.sandboxes.fromId(sandboxId), ms, "a sandbox lookup");
89
+ if ((await within(sandbox.poll(), ms, "a sandbox poll")) !== null)
90
+ throw sandboxNotFound(sandboxId);
91
+ }
92
+ catch (error) {
93
+ throw isModalNotFoundError(error) ? sandboxNotFound(sandboxId) : error;
94
+ }
95
+ const hosts = profile.desktop === true ? await tunnelHosts(sandbox) : new Map();
96
+ return modalSandboxHandle(sandbox, profile, connection, hosts);
97
+ }
85
98
  return {
86
99
  Sandbox: {
87
100
  create: hidingTokens(connection, create),
101
+ connect: hidingTokens(connection, connect),
88
102
  ...modalSandboxes(client, connection),
89
103
  },
90
104
  };
91
105
  }
106
+ /** The host of each tunnel a desktop sandbox exposes, by port. */
107
+ async function tunnelHosts(sandbox) {
108
+ const tunnels = await sandbox.tunnels(30_000);
109
+ return new Map(Object.entries(tunnels).map(([port, tunnel]) => [Number(port), new URL(tunnel.url).host]));
110
+ }
92
111
  /** The humanish App in the connection's environment; only a create may make it. */
93
112
  function appLookup(client, connection, createIfMissing) {
94
113
  let app;
@@ -240,6 +259,10 @@ function modalSandboxHandle(sandbox, profile, connection, hosts) {
240
259
  const run = hidingTokens(connection, (command, options) => runCommand(sandbox, command, options));
241
260
  const shell = {
242
261
  sandboxId: sandbox.sandboxId,
262
+ portAccess: hidingTokens(connection, async (port) => {
263
+ const { url, token } = await sandbox.createConnectToken({ port });
264
+ return { url, headers: { Authorization: `Bearer ${token}` } };
265
+ }),
243
266
  // Modal reports no allocation for a sandbox; the create's request is what it bills.
244
267
  getInfo: () => Promise.resolve({ cpuCount: profile.cpu * 2, memoryMB: profile.memoryMiB }),
245
268
  commands: { run },
@@ -257,31 +280,7 @@ function modalSandboxHandle(sandbox, profile, connection, hosts) {
257
280
  ...shell,
258
281
  ...modalDesktopSurface(run, hidingTokens(connection, (path) => readBytes(sandbox, path)), hosts),
259
282
  };
260
- const desktopOnly = (what) => () => Promise.reject(new ModalUnsupportedError(what));
261
- return {
262
- ...shell,
263
- wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
264
- launch: desktopOnly("A desktop application launch"),
265
- screenshot: desktopOnly("A desktop screenshot"),
266
- leftClick: desktopOnly("Desktop input"),
267
- rightClick: desktopOnly("Desktop input"),
268
- middleClick: desktopOnly("Desktop input"),
269
- doubleClick: desktopOnly("Desktop input"),
270
- moveMouse: desktopOnly("Desktop input"),
271
- scroll: desktopOnly("Desktop input"),
272
- write: desktopOnly("Desktop input"),
273
- press: desktopOnly("Desktop input"),
274
- drag: desktopOnly("Desktop input"),
275
- stream: {
276
- getAuthKey: () => {
277
- throw new ModalUnsupportedError("A desktop stream");
278
- },
279
- getUrl: () => {
280
- throw new ModalUnsupportedError("A desktop stream");
281
- },
282
- start: desktopOnly("A desktop stream"),
283
- },
284
- };
283
+ return { ...shell, ...shellOnlySurface((what) => new ModalUnsupportedError(what)) };
285
284
  }
286
285
  /** The Modal account `env` names, for release by id and reclaim. Throws when it names none. */
287
286
  export async function modalAccount(env, load = loadModalModule) {
@@ -51,6 +51,13 @@ export interface ModalSandbox {
51
51
  tunnels(timeoutMs?: number): Promise<Record<number, {
52
52
  url: string;
53
53
  }>>;
54
+ /** A URL to `port` and a token Modal's proxy requires as `Authorization: Bearer <token>`. */
55
+ createConnectToken(params?: {
56
+ port?: number;
57
+ }): Promise<{
58
+ url: string;
59
+ token: string;
60
+ }>;
54
61
  }
55
62
  export interface ModalSandboxCreateParams {
56
63
  cpu?: number;