humanish 0.136.1 → 0.138.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/dist/actors/computer-use/actor.d.ts +2 -0
  3. package/dist/actors/computer-use/actor.js +1 -0
  4. package/dist/actors/computer-use/loop/observation.js +2 -0
  5. package/dist/actors/computer-use/loop/session.d.ts +3 -2
  6. package/dist/actors/computer-use/loop/session.js +8 -12
  7. package/dist/actors/computer-use/loop/types.d.ts +14 -1
  8. package/dist/cli/commands/runs.js +2 -2
  9. package/dist/cli/doctor.js +24 -0
  10. package/dist/evidence/browser-network.d.ts +108 -0
  11. package/dist/evidence/browser-network.js +516 -0
  12. package/dist/evidence/trace-page.d.ts +31 -1
  13. package/dist/evidence/trace-page.js +68 -2
  14. package/dist/keys/key-resolution.js +53 -2
  15. package/dist/observer-app.html +2 -2
  16. package/dist/routes/computer-use/bundle.js +1 -0
  17. package/dist/routes/computer-use/e2b-desktop/desktop.js +6 -0
  18. package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +12 -1
  19. package/dist/routes/computer-use/e2b-desktop/fidelity.js +7 -2
  20. package/dist/routes/computer-use/e2b-desktop/network.d.ts +52 -0
  21. package/dist/routes/computer-use/e2b-desktop/network.js +153 -0
  22. package/dist/routes/computer-use/e2b-desktop/start.js +18 -1
  23. package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -0
  24. package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
  25. package/dist/routes/computer-use/e2b-desktop/teardown.js +6 -3
  26. package/dist/routes/computer-use/fanout-records.js +10 -0
  27. package/dist/routes/computer-use/participant-desktop.d.ts +7 -2
  28. package/dist/routes/computer-use/participant-model.js +2 -1
  29. package/dist/routes/computer-use/plan.js +2 -1
  30. package/dist/routes/computer-use/single-bundle.d.ts +3 -0
  31. package/dist/routes/computer-use/single-bundle.js +4 -0
  32. package/dist/routes/computer-use/types.d.ts +7 -0
  33. package/dist/routes/scripted/plan.js +2 -2
  34. package/dist/routes/shared-world/plan.js +3 -2
  35. package/dist/routes/terminal/live-sandbox.d.ts +2 -0
  36. package/dist/routes/terminal/live-sandbox.js +8 -3
  37. package/dist/routes/terminal/plan.js +51 -8
  38. package/dist/routes/terminal/session.js +8 -1
  39. package/dist/run/bundle.d.ts +2 -1
  40. package/dist/run/cost-summary.js +9 -4
  41. package/dist/run/live-items.d.ts +20 -2
  42. package/dist/run/live-items.js +23 -14
  43. package/dist/run/pricing.d.ts +11 -0
  44. package/dist/run/pricing.js +18 -5
  45. package/dist/run/reclaim-outcomes.d.ts +4 -1
  46. package/dist/run/reclaim-outcomes.js +8 -6
  47. package/dist/run/reclaim.d.ts +4 -1
  48. package/dist/run/reclaim.js +81 -34
  49. package/dist/run/sandbox-creates.d.ts +7 -4
  50. package/dist/run/sandbox-creates.js +17 -9
  51. package/dist/run/sandbox-receipts.d.ts +13 -5
  52. package/dist/run/sandbox-receipts.js +18 -6
  53. package/dist/run/streams.d.ts +7 -0
  54. package/dist/study/composition-rules.js +11 -10
  55. package/dist/study/config.js +4 -1
  56. package/dist/study/doctor.js +1 -1
  57. package/dist/study/keys.d.ts +3 -0
  58. package/dist/study/keys.js +2 -0
  59. package/dist/study/parse/comms.js +1 -20
  60. package/dist/study/parse/execution.js +35 -3
  61. package/dist/study/parse/values.d.ts +5 -0
  62. package/dist/study/parse/values.js +22 -0
  63. package/dist/study/plan-base.js +6 -1
  64. package/dist/study/plan-types.d.ts +5 -2
  65. package/dist/study/requirements.js +7 -2
  66. package/dist/study/routing.d.ts +1 -1
  67. package/dist/study/routing.js +1 -1
  68. package/dist/study/study-fields.d.ts +7 -1
  69. package/dist/study/study-fields.js +10 -0
  70. package/dist/study/types.d.ts +12 -1
  71. package/dist/study/validation.d.ts +8 -0
  72. package/dist/study/validation.js +31 -2
  73. package/dist/substrates/detached.d.ts +5 -0
  74. package/dist/substrates/detached.js +9 -0
  75. package/dist/substrates/e2b/cdp-network-capture.d.ts +23 -0
  76. package/dist/substrates/e2b/cdp-network-capture.js +194 -0
  77. package/dist/substrates/e2b/cdp-probe.d.ts +20 -5
  78. package/dist/substrates/e2b/cdp-probe.js +98 -34
  79. package/dist/substrates/e2b/desktop-cdp.d.ts +32 -2
  80. package/dist/substrates/e2b/desktop-cdp.js +87 -5
  81. package/dist/substrates/e2b/desktop-resources.d.ts +4 -2
  82. package/dist/substrates/e2b/desktop-resources.js +2 -2
  83. package/dist/substrates/e2b/peer-install.js +1 -1
  84. package/dist/substrates/e2b/sandbox.d.ts +3 -0
  85. package/dist/substrates/e2b/sandbox.js +5 -4
  86. package/dist/substrates/modal/image.d.ts +9 -0
  87. package/dist/substrates/modal/image.js +36 -0
  88. package/dist/substrates/modal/sandbox.d.ts +26 -0
  89. package/dist/substrates/modal/sandbox.js +367 -0
  90. package/dist/substrates/modal/sdk.d.ts +112 -0
  91. package/dist/substrates/modal/sdk.js +61 -0
  92. package/dist/verify/network.d.ts +10 -0
  93. package/dist/verify/network.js +162 -0
  94. package/dist/verify/pages.js +1 -16
  95. package/dist/verify/verify.js +8 -2
  96. package/docs/architecture/terminal-product-route.md +1 -1
  97. package/docs/contracts/run-bundle.md +90 -3
  98. package/docs/contracts/schemas.md +6 -5
  99. package/package.json +8 -2
@@ -20,7 +20,9 @@ function residualOf(config) {
20
20
  ...(policies === undefined ? {} : { policies }),
21
21
  ...(defaults === undefined ? {} : { defaults }),
22
22
  ...(review === undefined ? {} : { review }),
23
- ...(config.execution?.desktop === undefined && config.execution?.target === undefined
23
+ ...(config.execution?.desktop === undefined &&
24
+ config.execution?.target === undefined &&
25
+ config.execution?.network === undefined
24
26
  ? {}
25
27
  : {
26
28
  execution: {
@@ -30,6 +32,9 @@ function residualOf(config) {
30
32
  ...(config.execution?.target === undefined
31
33
  ? {}
32
34
  : { target: config.execution.target }),
35
+ ...(config.execution?.network === undefined
36
+ ? {}
37
+ : { network: config.execution.network }),
33
38
  },
34
39
  }),
35
40
  subject: {
@@ -4,6 +4,7 @@ import type { ReasoningEffort } from "../actors/reasoning-effort.js";
4
4
  import type { BrowserSurface } from "../actors/scripted-browser/types.js";
5
5
  import type { ScriptedRefusal } from "../routes/scripted/plan.js";
6
6
  import type { TerminalRefusal } from "../routes/terminal/plan.js";
7
+ import type { SandboxProviderId } from "../run/sandbox-receipts.js";
7
8
  import type { RunStudyProvenance } from "../run/study-provenance.js";
8
9
  import type { ArrivalPlan } from "./arrivals.js";
9
10
  import type { BundleEstimate } from "./bundle-estimate.js";
@@ -19,7 +20,7 @@ type AtLeastTwo<T> = readonly [T, T, ...T[]];
19
20
  */
20
21
  export type Requirement = {
21
22
  readonly kind: "key";
22
- readonly name: "OPENAI_API_KEY" | "E2B_API_KEY";
23
+ readonly name: "OPENAI_API_KEY" | "E2B_API_KEY" | "MODAL_TOKEN_ID" | "MODAL_TOKEN_SECRET";
23
24
  }
24
25
  /** The terminal runtime key: CODEX_API_KEY, else OPENAI_API_KEY. */
25
26
  | {
@@ -51,7 +52,7 @@ export type Requirement = {
51
52
  * fields at run time.
52
53
  */
53
54
  export type ResidualConfig = Pick<StudyConfig, "comms" | "policies" | "defaults" | "review"> & {
54
- readonly execution?: Pick<NonNullable<StudyConfig["execution"]>, "desktop" | "target">;
55
+ readonly execution?: Pick<NonNullable<StudyConfig["execution"]>, "desktop" | "target" | "network">;
55
56
  readonly subject: Pick<StudyConfig["subject"], "clone" | "localTree" | "repos" | "envValues">;
56
57
  };
57
58
  export interface PlannedAnalysis {
@@ -223,6 +224,8 @@ export interface SharedWorldPlan extends PlanBase {
223
224
  }
224
225
  export type TerminalPlan = PlanBase & {
225
226
  readonly route: "terminal";
227
+ /** Where the shell sandbox runs: `execution.target` modal-terminal, else E2B. */
228
+ readonly sandboxProvider: SandboxProviderId;
226
229
  /** The registered terminal actor that runs. */
227
230
  readonly actor: string;
228
231
  readonly product: StudySubjectProduct & {
@@ -9,9 +9,14 @@ import { localAgentRefusal } from "../actors/local-agent/readiness.js";
9
9
  import { describeMissingKeys } from "../keys/key-resolution.js";
10
10
  import { MODEL_RATES, unpricedCapMessage } from "../run/pricing.js";
11
11
  /** Provider keys in the order the routes name them in a refusal. */
12
- const KEY_ORDER = ["OPENAI_API_KEY", "E2B_API_KEY"];
12
+ const KEY_ORDER = [
13
+ "OPENAI_API_KEY",
14
+ "E2B_API_KEY",
15
+ "MODAL_TOKEN_ID",
16
+ "MODAL_TOKEN_SECRET",
17
+ ];
13
18
  const present = (env, name) => (env[name]?.trim() ?? "") !== "";
14
- /** The `key` requirements whose value `env` lacks, OPENAI_API_KEY before E2B_API_KEY. */
19
+ /** The `key` requirements whose value `env` lacks, in KEY_ORDER. */
15
20
  function missingKeys(requirements, env) {
16
21
  const required = new Set(requirements.flatMap((requirement) => (requirement.kind === "key" ? [requirement.name] : [])));
17
22
  return KEY_ORDER.filter((name) => required.has(name) && !present(env, name));
@@ -69,7 +69,7 @@ export declare function isScriptedBrowserComposition(config: StudyConfig): boole
69
69
  export declare function isProvisionedScriptedBrowserComposition(config: StudyConfig): boolean;
70
70
  /**
71
71
  * True when this config routes to the terminal-product backend: a terminal-product subject whose
72
- * first actor resolves to a registered terminal actor (execution.target e2b-terminal or absent;
72
+ * first actor resolves to a registered terminal actor (execution.target e2b-terminal, modal-terminal or absent;
73
73
  * the parse layer enforces that pairing). Mirror of isComputerUseComposition/isScriptedBrowserComposition;
74
74
  * the single source of truth for routeOf and the warning logic.
75
75
  */
@@ -177,7 +177,7 @@ export function isProvisionedScriptedBrowserComposition(config) {
177
177
  }
178
178
  /**
179
179
  * True when this config routes to the terminal-product backend: a terminal-product subject whose
180
- * first actor resolves to a registered terminal actor (execution.target e2b-terminal or absent;
180
+ * first actor resolves to a registered terminal actor (execution.target e2b-terminal, modal-terminal or absent;
181
181
  * the parse layer enforces that pairing). Mirror of isComputerUseComposition/isScriptedBrowserComposition;
182
182
  * the single source of truth for routeOf and the warning logic.
183
183
  */
@@ -1,7 +1,13 @@
1
- import type { StudyConfig, StudyParticipantEntry } from "./types.js";
1
+ import type { StudyConfig, StudyNetworkCapture, StudyParticipantEntry } from "./types.js";
2
2
  /** The participants listed one by one: a `participants` list. Undefined when none is listed. */
3
3
  export declare function participantList(config: StudyConfig): readonly StudyParticipantEntry[] | undefined;
4
4
  /** The declared number of identical participants: `participants: <n>` or `participants.count`. */
5
5
  export declare function declaredParticipantCount(config: StudyConfig): number | undefined;
6
6
  /** The steer each of a counted group of participants gets: `participants.instruction`. */
7
7
  export declare function participantInstruction(config: StudyConfig): string | undefined;
8
+ /**
9
+ * The origins each participant's network log keeps beside its target's
10
+ * (`execution.network.origins`), or undefined when capture is off: `execution.network` absent or
11
+ * false.
12
+ */
13
+ export declare function networkCaptureOrigins(network: StudyNetworkCapture | undefined): readonly string[] | undefined;
@@ -17,6 +17,16 @@ export function declaredParticipantCount(config) {
17
17
  export function participantInstruction(config) {
18
18
  return isGroup(config.participants) ? config.participants.instruction : undefined;
19
19
  }
20
+ /**
21
+ * The origins each participant's network log keeps beside its target's
22
+ * (`execution.network.origins`), or undefined when capture is off: `execution.network` absent or
23
+ * false.
24
+ */
25
+ export function networkCaptureOrigins(network) {
26
+ if (network === undefined || network === false)
27
+ return undefined;
28
+ return network === true ? [] : (network.origins ?? []);
29
+ }
20
30
  function isGroup(participants) {
21
31
  return typeof participants === "object" && participants !== null && !Array.isArray(participants);
22
32
  }
@@ -467,7 +467,7 @@ export interface StudyActor {
467
467
  */
468
468
  dwell?: DwellWindow;
469
469
  }
470
- type StudyExecutionTarget = "local" | "e2b-desktop" | "e2b-terminal";
470
+ type StudyExecutionTarget = "local" | "e2b-desktop" | "e2b-terminal" | "modal-terminal";
471
471
  /** Terminal transport: the captured non-interactive exec stream (stdin disabled). It is not an
472
472
  * interactive duplex PTY; labeling captured exec output "pty" would be a claim/mechanism
473
473
  * mismatch, so this route uses "exec-stream". */
@@ -570,6 +570,15 @@ export interface StudyDesktopFidelity {
570
570
  /** Full user-agent string to present; default: a mobile Safari string. */
571
571
  userAgent?: string;
572
572
  }
573
+ /**
574
+ * `execution.network`: record each browser participant's requests to `network/<id>.ndjson`.
575
+ * `true` keeps the requests to the participant's target origin; `origins` adds up to 16 more exact
576
+ * http(s) origins. Only computer-use participants in hosted Chrome or Chromium on
577
+ * `execution.target: e2b-desktop` read it; every other route refuses it.
578
+ */
579
+ export type StudyNetworkCapture = boolean | {
580
+ origins?: string[];
581
+ };
573
582
  export interface StudyExecution {
574
583
  target?: StudyExecutionTarget;
575
584
  /** Actor session wall-clock budget. Consumed on the app-url route. */
@@ -582,6 +591,8 @@ export interface StudyExecution {
582
591
  */
583
592
  concurrency?: number;
584
593
  desktop?: StudyExecutionDesktop;
594
+ /** Each browser participant's network log; see StudyNetworkCapture. Absent or false: off. */
595
+ network?: StudyNetworkCapture;
585
596
  /** `terminal-product` route: the terminal transport + stdin posture. Consumed on that route. */
586
597
  terminal?: StudyExecutionTerminal;
587
598
  /** `terminal-product` route: runtime key placement, defaulting to openai-egress, an external
@@ -1,3 +1,4 @@
1
+ import { type StudyRoute } from "./routing.js";
1
2
  import type { StudyConfig } from "./types.js";
2
3
  /**
3
4
  * Cross-validate the computer-use fan-out declaration (`per-lane-worlds`). Returns the failure
@@ -21,6 +22,13 @@ export declare function smtpValidationReason(config: StudyConfig): string | unde
21
22
  * topology may not identify the backend that is actually executing.
22
23
  */
23
24
  export declare function desktopMediaValidationReason(config: StudyConfig, supportsMedia?: boolean): string | undefined;
25
+ /**
26
+ * Why a study's `execution.network` cannot run where it is going, or undefined. Only a
27
+ * computer-use participant in hosted Chrome or Chromium has the in-sandbox DevTools client that
28
+ * records requests. Enforced at parse and again by each route's planner, which passes its own
29
+ * route, for a library caller's config.
30
+ */
31
+ export declare function networkCaptureValidationReason(config: StudyConfig, route?: StudyRoute): string | undefined;
24
32
  /** Reused by direct library runners so unsupported receiving never becomes inert configuration. */
25
33
  export declare function receivingEmailValidationReason(config: StudyConfig): string | undefined;
26
34
  /** Refuse task declarations that the selected execution path would discard.
@@ -4,8 +4,8 @@ import { waitFieldsReason } from "../actors/computer-use/wait.js";
4
4
  import { PARTICIPANT_ID_MAX_CHARS, PARTICIPANT_ID_PATTERN } from "./parse/actors.js";
5
5
  import { isHttpUrl, isLoopbackUrl } from "./parse/subject.js";
6
6
  import { declaredTargets } from "./plan-participants.js";
7
- import { actorResolvesToComputerUse, computerUseParticipantCount, participantCountReason, registeredComputerUseActors, resolveEntryUrl, isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, } from "./routing.js";
8
- import { participantList } from "./study-fields.js";
7
+ import { actorResolvesToComputerUse, computerUseParticipantCount, participantCountReason, registeredComputerUseActors, resolveEntryUrl, isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, routeOf, } from "./routing.js";
8
+ import { networkCaptureOrigins, participantList } from "./study-fields.js";
9
9
  import { isStartAfterMs, rosterStartsReason, startAfterMessage, windowReason } from "./arrivals.js";
10
10
  /**
11
11
  * Cross-validate the computer-use fan-out declaration (`per-lane-worlds`). Returns the failure
@@ -206,6 +206,35 @@ export function desktopMediaValidationReason(config, supportsMedia = isComputerU
206
206
  }
207
207
  return undefined;
208
208
  }
209
+ /**
210
+ * Why a study's `execution.network` cannot run where it is going, or undefined. Only a
211
+ * computer-use participant in hosted Chrome or Chromium has the in-sandbox DevTools client that
212
+ * records requests. Enforced at parse and again by each route's planner, which passes its own
213
+ * route, for a library caller's config.
214
+ */
215
+ export function networkCaptureValidationReason(config, route = routeOf(config)) {
216
+ if (networkCaptureOrigins(config.execution?.network) === undefined)
217
+ return undefined;
218
+ const hosted = "only computer-use participants in hosted Chrome or Chromium on `execution.target: e2b-desktop` record their requests";
219
+ if (route === "terminal")
220
+ return "`execution.network` records each browser participant's requests, and route: terminal launches no browser: its participant works in a shell. Remove `execution.network`.";
221
+ if (route === "shared-world")
222
+ return "`execution.network` is not supported on route: shared-world yet: a shared world's lobby code sits in a page address the network log would keep, and that route keeps observed addresses only as digests. Remove `execution.network`, or run independent participants on route: computer-use.";
223
+ if (route === "scripted")
224
+ return `\`execution.network\` is not supported on route: scripted yet: ${hosted}. Remove \`execution.network\`.`;
225
+ if (route !== "computer-use")
226
+ return `\`execution.network\` is not read on route: ${route}: ${hosted}. Remove \`execution.network\`.`;
227
+ if (config.subject.source === "desktop-cli")
228
+ return "`execution.network` records each browser participant's requests, and a desktop-cli subject opens a terminal window and no browser, so there are none to record. Remove `execution.network`.";
229
+ if (config.subject.source === "local-app")
230
+ return `\`execution.network\` is not supported for a local-app subject: humanish does not launch that participant's browser, and ${hosted}. Remove \`execution.network\`.`;
231
+ const target = config.execution?.target;
232
+ if (target !== "e2b-desktop")
233
+ return `\`execution.network\` is not supported with \`execution.target: ${target ?? "(absent)"}\` yet: ${hosted}. Set \`execution.target: e2b-desktop\`, or remove \`execution.network\`.`;
234
+ if (config.execution?.desktop?.browser === "firefox")
235
+ return "`execution.network` needs Chrome or Chromium, and `execution.desktop.browser: firefox` launches Firefox with no DevTools port to record requests through. Set `execution.desktop.browser` to chrome, chromium or default, or remove `execution.network`.";
236
+ return undefined;
237
+ }
209
238
  /** Reused by direct library runners so unsupported receiving never becomes inert configuration. */
210
239
  export function receivingEmailValidationReason(config) {
211
240
  if (config.comms?.email?.kind !== "real")
@@ -27,6 +27,11 @@ export interface DetachedStepResult {
27
27
  }
28
28
  /** Read the capped log tail for a step (raw; caller redacts). */
29
29
  export declare function readDetachedLog(shell: Shell, name: string, requestTimeoutMs?: number): Promise<string>;
30
+ /**
31
+ * Whether a process started by startDetachedProcess or runDetachedStep has exited. Its wrapper
32
+ * writes a status file however the command ended, a signal included.
33
+ */
34
+ export declare function detachedProcessExited(shell: Shell, name: string, requestTimeoutMs?: number): Promise<boolean>;
30
35
  /**
31
36
  * Run a bounded step (install/build) detached, polling its atomic status file until it
32
37
  * exits or the budget runs out. On timeout the process group is killed and the log tail is
@@ -75,6 +75,15 @@ export async function readDetachedLog(shell, name, requestTimeoutMs = DEFAULT_RE
75
75
  const result = await runOrThrow(shell, `tail -c ${LOG_TAIL_BYTES} ${shellQuote(`${stepDir(name)}/log.txt`)} 2>/dev/null || true`, { requestTimeoutMs });
76
76
  return result.stdout;
77
77
  }
78
+ /**
79
+ * Whether a process started by startDetachedProcess or runDetachedStep has exited. Its wrapper
80
+ * writes a status file however the command ended, a signal included.
81
+ */
82
+ export async function detachedProcessExited(shell, name, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
83
+ assertName(name);
84
+ const result = await runOrThrow(shell, `test -e ${shellQuote(`${stepDir(name)}/status`)} && echo exited || true`, { requestTimeoutMs });
85
+ return result.stdout.trim() === "exited";
86
+ }
78
87
  /**
79
88
  * Run a bounded step (install/build) detached, polling its atomic status file until it
80
89
  * exits or the budget runs out. On timeout the process group is killed and the log tail is
@@ -0,0 +1,23 @@
1
+ /** The `network` field of a hold request. */
2
+ export interface ChromeNetworkCaptureRequest {
3
+ /**
4
+ * Sandbox file the holder appends to; its folder is created. Defaults to
5
+ * /tmp/humanish-network/events.ndjson.
6
+ */
7
+ eventsPath?: string;
8
+ /** Origins, as `new URL(url).origin` writes them, whose request addresses the holder writes. */
9
+ origins: string[];
10
+ }
11
+ /**
12
+ * The capture half of the holder, spliced into CHROME_CDP_PROBE_PY so the shipped script stays one
13
+ * string. Line kinds: `start` (Network answered on the launch page), `request`, `response`,
14
+ * `servedFromCache`, `finished`, `failed`, `backForward` (a main-frame restore from the
15
+ * back-forward cache, which sends no request), `cap` (the file reached 32 MiB and writing stopped)
16
+ * and `exit` (the holder stopped; written in a finally). Ids are `<session>:<requestId>`, the
17
+ * session numbered in attach order from 0 for the launch page, because request ids repeat across
18
+ * renderers. `wallTime` is seconds since the epoch on the sandbox clock; `timestamp` is CDP's
19
+ * monotonic clock, for durations. A path longer than 4,096 characters ends at its last whole
20
+ * segment within them, then `/[TRUNCATED]`: the host redacts each segment, and a token cut in two
21
+ * can fall below the length its rules withhold.
22
+ */
23
+ export declare const NETWORK_CAPTURE_PY: string;
@@ -0,0 +1,194 @@
1
+ // What the in-sandbox holder keeps from a participant's browser traffic. The holder (cdp-probe.ts,
2
+ // `mode: "hold"`) already owns the one long-lived DevTools socket on a hosted desktop; with a
3
+ // network request it also enables the Network domain and appends one short JSON line per kept
4
+ // event to a file the host reads at teardown.
5
+ //
6
+ // The CDP events carry headers, cookies, post data, initiator stacks and full URLs. A line copies
7
+ // none of them: only the request id, the two clocks, method, resource type, status, cache flags,
8
+ // error text, and an address for an allowed origin, cut at the first `?` or `#` with user info
9
+ // removed. A request to any other origin gets a line with no address, method or type, so the host
10
+ // can count it, and nothing more is written for it. A method or resource type that is not plain
11
+ // letters is written as `Other`, and error text outside Chrome's `net::ERR_*` form as
12
+ // `net::ERR_OTHER`, so no free text reaches the file.
13
+ /** The segment a path the holder cut ends with: the marker tracePage ends a cut path with. */
14
+ const PATH_CUT = "[TRUNCATED]";
15
+ /**
16
+ * The capture half of the holder, spliced into CHROME_CDP_PROBE_PY so the shipped script stays one
17
+ * string. Line kinds: `start` (Network answered on the launch page), `request`, `response`,
18
+ * `servedFromCache`, `finished`, `failed`, `backForward` (a main-frame restore from the
19
+ * back-forward cache, which sends no request), `cap` (the file reached 32 MiB and writing stopped)
20
+ * and `exit` (the holder stopped; written in a finally). Ids are `<session>:<requestId>`, the
21
+ * session numbered in attach order from 0 for the launch page, because request ids repeat across
22
+ * renderers. `wallTime` is seconds since the epoch on the sandbox clock; `timestamp` is CDP's
23
+ * monotonic clock, for durations. A path longer than 4,096 characters ends at its last whole
24
+ * segment within them, then `/[TRUNCATED]`: the host redacts each segment, and a token cut in two
25
+ * can fall below the length its rules withhold.
26
+ */
27
+ export const NETWORK_CAPTURE_PY = String.raw `
28
+ NETWORK_ENABLE = {"maxTotalBufferSize": 0, "maxResourceBufferSize": 0, "maxPostDataSize": 0}
29
+ NETWORK_EVENTS_CAP = 32 * 1024 * 1024
30
+ # Kept free under the cap so the cap and exit lines always fit.
31
+ NETWORK_TAIL_RESERVE = 512
32
+ NETWORK_PATH_LIMIT = 4096
33
+ WEB_ADDRESS = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://(?:[^/?#]*@)?([^/?#]*)([^?#]*)")
34
+ NETWORK_WORD = re.compile(r"^[A-Za-z]{1,32}$")
35
+ NETWORK_ERROR = re.compile(r"^net::ERR_[A-Z0-9_]{1,64}$")
36
+
37
+ def allowed_address(url, origins):
38
+ """origin + path for an allowed origin, without user info, query or fragment; else None."""
39
+ match = WEB_ADDRESS.match(str(url or ""))
40
+ if not match:
41
+ return None
42
+ origin = ("%s://%s" % (match.group(1), match.group(2))).lower()
43
+ if origin not in origins:
44
+ return None
45
+ path = match.group(3) or "/"
46
+ if len(path) > NETWORK_PATH_LIMIT:
47
+ path = path[:path.rfind("/", 0, NETWORK_PATH_LIMIT + 1)] + "/${PATH_CUT}"
48
+ return origin + path
49
+
50
+ def network_word(value, fallback):
51
+ text = str(value or "")
52
+ return text if NETWORK_WORD.match(text) else fallback
53
+
54
+ def clock(value):
55
+ return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
56
+
57
+ def response_fields(response):
58
+ fields = {}
59
+ if not isinstance(response, dict):
60
+ return fields
61
+ status = clock(response.get("status"))
62
+ if status is not None and 100 <= status <= 599:
63
+ fields["status"] = int(status)
64
+ for flag in ("fromDiskCache", "fromPrefetchCache", "fromServiceWorker"):
65
+ if response.get(flag) is True:
66
+ fields[flag] = True
67
+ return fields
68
+
69
+ class NetworkLog:
70
+ """The events file of one holder. Writes go straight to the file, one line per write, so a
71
+ read while the holder runs sees whole lines except possibly the last."""
72
+ def __init__(self, request):
73
+ self.path = str(request.get("eventsPath") or "/tmp/humanish-network/events.ndjson")
74
+ self.origins = set(str(origin).rstrip("/").lower() for origin in request.get("origins") or [] if isinstance(origin, str))
75
+ folder = os.path.dirname(self.path)
76
+ if folder:
77
+ os.makedirs(folder, exist_ok=True)
78
+ self.handle = open(self.path, "ab", buffering=0)
79
+ self.written = os.fstat(self.handle.fileno()).st_size
80
+ self.capped = False
81
+ self.sessions = {}
82
+ # Ids whose current hop has an allowed address; only those get lines after the request.
83
+ self.open = set()
84
+ self.faults = 0
85
+
86
+ def session(self, session_id):
87
+ if session_id not in self.sessions:
88
+ self.sessions[session_id] = len(self.sessions)
89
+ return self.sessions[session_id]
90
+
91
+ def write(self, line, tail=False):
92
+ if self.capped and not tail:
93
+ return
94
+ data = (json.dumps({key: value for key, value in line.items() if value is not None}, separators=(",", ":")) + "\n").encode("utf-8")
95
+ room = NETWORK_EVENTS_CAP - (0 if tail else NETWORK_TAIL_RESERVE)
96
+ if self.written + len(data) > room:
97
+ if not tail:
98
+ self.capped = True
99
+ self.write({"e": "cap", "wallTime": time.time()}, tail=True)
100
+ return
101
+ self.handle.write(data)
102
+ self.written += len(data)
103
+
104
+ def event(self, method, params, session):
105
+ """Record one CDP event. A fault is counted, never raised: emulation must keep running."""
106
+ try:
107
+ self.record(method, params, session)
108
+ except Exception:
109
+ self.faults += 1
110
+
111
+ def record(self, method, params, session):
112
+ # Past the cap nothing more is written, so nothing more is tracked.
113
+ if self.capped:
114
+ return
115
+ if method == "Page.frameNavigated":
116
+ frame = params.get("frame") or {}
117
+ if params.get("type") == "BackForwardCacheRestore" and not frame.get("parentId"):
118
+ self.write({"e": "backForward", "wallTime": time.time(), "url": allowed_address(frame.get("url"), self.origins)})
119
+ return
120
+ if not str(method).startswith("Network."):
121
+ return
122
+ kind = method[len("Network."):]
123
+ request_id = "%d:%s" % (self.session(session), str(params.get("requestId") or "")[:64])
124
+ timestamp = clock(params.get("timestamp"))
125
+ if kind == "requestWillBeSent":
126
+ request = params.get("request") or {}
127
+ line = {"e": "request", "id": request_id, "wallTime": clock(params.get("wallTime")), "timestamp": timestamp}
128
+ if "redirectResponse" in params:
129
+ line["redirectResponse"] = response_fields(params.get("redirectResponse"))
130
+ address = allowed_address(request.get("url"), self.origins)
131
+ if address is None:
132
+ self.open.discard(request_id)
133
+ else:
134
+ self.open.add(request_id)
135
+ line["url"] = address
136
+ line["method"] = network_word(request.get("method"), "Other")
137
+ line["type"] = network_word(params.get("type"), "Other")
138
+ self.write(line)
139
+ elif request_id not in self.open:
140
+ return
141
+ elif kind == "responseReceived":
142
+ line = {"e": "response", "id": request_id, "timestamp": timestamp}
143
+ line.update(response_fields(params.get("response")))
144
+ self.write(line)
145
+ elif kind == "requestServedFromCache":
146
+ self.write({"e": "servedFromCache", "id": request_id})
147
+ elif kind == "loadingFinished":
148
+ self.open.discard(request_id)
149
+ self.write({"e": "finished", "id": request_id, "timestamp": timestamp})
150
+ elif kind == "loadingFailed":
151
+ self.open.discard(request_id)
152
+ error = str(params.get("errorText") or "")
153
+ self.write({"e": "failed", "id": request_id, "timestamp": timestamp, "errorText": error if NETWORK_ERROR.match(error) else "net::ERR_OTHER"})
154
+
155
+ def close(self, reason):
156
+ line = {"e": "exit", "reason": str(reason)[:200]}
157
+ if self.faults:
158
+ line["faults"] = self.faults
159
+ try:
160
+ self.write(line, tail=True)
161
+ self.handle.close()
162
+ except Exception:
163
+ pass
164
+
165
+ def start_capture(ws, session_id, network):
166
+ """Network on the launch session, answered before any Emulation message goes out, so a mobile
167
+ participant's launch reload is captured. Returns the log, or None, and the announce field."""
168
+ try:
169
+ log = NetworkLog(network)
170
+ except Exception as error:
171
+ return None, {"unavailable": "the events file could not be opened (%s)" % type(error).__name__}
172
+ # Read before the call, so a request whose events queued behind the reply started after it.
173
+ started = time.time()
174
+ try:
175
+ reply = ws.call("Network.enable", NETWORK_ENABLE, session_id)
176
+ except Exception as error:
177
+ reply = {"error": {"message": type(error).__name__}}
178
+ if not isinstance(reply, dict) or "error" in reply:
179
+ detail = str(((reply or {}).get("error") or {}).get("message")) if isinstance(reply, dict) else "no reply"
180
+ log.close("Network.enable failed: %s" % detail)
181
+ return None, {"unavailable": "Network.enable failed on the launch page: %s" % detail}
182
+ try:
183
+ log.session(session_id)
184
+ log.write({"e": "start", "wallTime": started})
185
+ except Exception as error:
186
+ log.close("the start line could not be written: %s" % type(error).__name__)
187
+ return None, {"unavailable": "the events file could not be written (%s)" % type(error).__name__}
188
+ # Page events report back-forward restores; without them only those lines are missing.
189
+ try:
190
+ ws.call("Page.enable", {}, session_id)
191
+ except Exception:
192
+ pass
193
+ return log, {"startedAt": started}
194
+ `;
@@ -1,3 +1,4 @@
1
+ import { type ChromeNetworkCaptureRequest } from "./cdp-network-capture.js";
1
2
  /** Which target to attribute when the endpoint lists several pages. */
2
3
  export type ChromeCdpPagePreference = "pinned" | "active";
3
4
  export interface ChromeCdpProbeArgs {
@@ -23,15 +24,17 @@ export interface ChromeCdpProbeArgs {
23
24
  /**
24
25
  * "state": url/title/text/scrollY. "geometry": outer window + CSS viewport. "port": resolution
25
26
  * only. "emulate": apply mobile emulation to the selected page and exit (the overrides that
26
- * are session-scoped, UA / touch / DPR, lapse when the socket closes). "hold": the same over a browser-level
27
- * socket, then stay attached until killed (how a participant keeps them for its whole life) and attach
28
- * to every page target Chrome opens later, sending it the same overrides and a reload the moment
29
- * it exists, never pausing it. "fidelity": read
27
+ * are session-scoped, UA / touch / DPR, lapse when the socket closes). "hold": stay attached over
28
+ * a browser-level socket until killed and attach to every page target Chrome opens later, never
29
+ * pausing it; it applies `emulation` (how a participant keeps the overrides for its whole life,
30
+ * with a reload on each page) and `network` capture, either or both. "fidelity": read
30
31
  * back what the page reports about itself (UA, DPR, viewport, touch), the proof for the bundle.
31
32
  */
32
33
  mode: "state" | "geometry" | "port" | "emulate" | "hold" | "fidelity";
33
- /** For "emulate": what to apply. */
34
+ /** For "emulate" and "hold": what to apply. */
34
35
  emulation?: ChromeMobileEmulationRequest;
36
+ /** For "hold": capture each held page's requests to a sandbox file. */
37
+ network?: ChromeNetworkCaptureRequest;
35
38
  }
36
39
  /** Mobile emulation request: the CDP Emulation domain applied to one page target. */
37
40
  export interface ChromeMobileEmulationRequest {
@@ -75,6 +78,18 @@ export interface ChromeCdpProbeResult {
75
78
  applied?: string[];
76
79
  /** "fidelity": the read-back. */
77
80
  fidelity?: ChromeFidelityRead;
81
+ /** "hold" with `network`: when capture started on the launch page, or why it could not. */
82
+ network?: {
83
+ startedAt: number;
84
+ } | {
85
+ unavailable: string;
86
+ };
87
+ /**
88
+ * Seconds since the epoch on the sandbox clock when the probe printed, on every answer including
89
+ * an unavailable one. It is the clock CDP `wallTime` uses, so a turn mark compares with request
90
+ * times without host clock skew.
91
+ */
92
+ now?: number;
78
93
  }
79
94
  /**
80
95
  * The probe itself. Kept as one string so the shipped command is exactly what the tests execute