humanish 0.136.1 → 0.138.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/actors/computer-use/actor.d.ts +2 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/observation.js +2 -0
- package/dist/actors/computer-use/loop/session.d.ts +3 -2
- package/dist/actors/computer-use/loop/session.js +8 -12
- package/dist/actors/computer-use/loop/types.d.ts +14 -1
- package/dist/cli/commands/runs.js +2 -2
- package/dist/cli/doctor.js +24 -0
- package/dist/evidence/browser-network.d.ts +108 -0
- package/dist/evidence/browser-network.js +516 -0
- package/dist/evidence/trace-page.d.ts +31 -1
- package/dist/evidence/trace-page.js +68 -2
- package/dist/keys/key-resolution.js +53 -2
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/bundle.js +1 -0
- package/dist/routes/computer-use/e2b-desktop/desktop.js +6 -0
- package/dist/routes/computer-use/e2b-desktop/fidelity.d.ts +12 -1
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +7 -2
- package/dist/routes/computer-use/e2b-desktop/network.d.ts +52 -0
- package/dist/routes/computer-use/e2b-desktop/network.js +153 -0
- package/dist/routes/computer-use/e2b-desktop/start.js +18 -1
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -0
- package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
- package/dist/routes/computer-use/e2b-desktop/teardown.js +6 -3
- package/dist/routes/computer-use/fanout-records.js +10 -0
- package/dist/routes/computer-use/participant-desktop.d.ts +7 -2
- package/dist/routes/computer-use/participant-model.js +2 -1
- package/dist/routes/computer-use/plan.js +2 -1
- package/dist/routes/computer-use/single-bundle.d.ts +3 -0
- package/dist/routes/computer-use/single-bundle.js +4 -0
- package/dist/routes/computer-use/types.d.ts +7 -0
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/shared-world/plan.js +3 -2
- package/dist/routes/terminal/live-sandbox.d.ts +2 -0
- package/dist/routes/terminal/live-sandbox.js +8 -3
- package/dist/routes/terminal/plan.js +51 -8
- package/dist/routes/terminal/session.js +8 -1
- package/dist/run/bundle.d.ts +2 -1
- package/dist/run/cost-summary.js +9 -4
- package/dist/run/live-items.d.ts +20 -2
- package/dist/run/live-items.js +23 -14
- package/dist/run/pricing.d.ts +11 -0
- package/dist/run/pricing.js +18 -5
- package/dist/run/reclaim-outcomes.d.ts +4 -1
- package/dist/run/reclaim-outcomes.js +8 -6
- package/dist/run/reclaim.d.ts +4 -1
- package/dist/run/reclaim.js +81 -34
- package/dist/run/sandbox-creates.d.ts +7 -4
- package/dist/run/sandbox-creates.js +17 -9
- package/dist/run/sandbox-receipts.d.ts +13 -5
- package/dist/run/sandbox-receipts.js +18 -6
- package/dist/run/streams.d.ts +7 -0
- package/dist/study/composition-rules.js +11 -10
- package/dist/study/config.js +4 -1
- package/dist/study/doctor.js +1 -1
- package/dist/study/keys.d.ts +3 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/parse/comms.js +1 -20
- package/dist/study/parse/execution.js +35 -3
- package/dist/study/parse/values.d.ts +5 -0
- package/dist/study/parse/values.js +22 -0
- package/dist/study/plan-base.js +6 -1
- package/dist/study/plan-types.d.ts +5 -2
- package/dist/study/requirements.js +7 -2
- package/dist/study/routing.d.ts +1 -1
- package/dist/study/routing.js +1 -1
- package/dist/study/study-fields.d.ts +7 -1
- package/dist/study/study-fields.js +10 -0
- package/dist/study/types.d.ts +12 -1
- package/dist/study/validation.d.ts +8 -0
- package/dist/study/validation.js +31 -2
- package/dist/substrates/detached.d.ts +5 -0
- package/dist/substrates/detached.js +9 -0
- package/dist/substrates/e2b/cdp-network-capture.d.ts +23 -0
- package/dist/substrates/e2b/cdp-network-capture.js +194 -0
- package/dist/substrates/e2b/cdp-probe.d.ts +20 -5
- package/dist/substrates/e2b/cdp-probe.js +98 -34
- package/dist/substrates/e2b/desktop-cdp.d.ts +32 -2
- package/dist/substrates/e2b/desktop-cdp.js +87 -5
- package/dist/substrates/e2b/desktop-resources.d.ts +4 -2
- package/dist/substrates/e2b/desktop-resources.js +2 -2
- package/dist/substrates/e2b/peer-install.js +1 -1
- package/dist/substrates/e2b/sandbox.d.ts +3 -0
- package/dist/substrates/e2b/sandbox.js +5 -4
- package/dist/substrates/modal/image.d.ts +9 -0
- package/dist/substrates/modal/image.js +36 -0
- package/dist/substrates/modal/sandbox.d.ts +26 -0
- package/dist/substrates/modal/sandbox.js +367 -0
- package/dist/substrates/modal/sdk.d.ts +112 -0
- package/dist/substrates/modal/sdk.js +61 -0
- package/dist/verify/network.d.ts +10 -0
- package/dist/verify/network.js +162 -0
- package/dist/verify/pages.js +1 -16
- package/dist/verify/verify.js +8 -2
- package/docs/architecture/terminal-product-route.md +1 -1
- package/docs/contracts/run-bundle.md +90 -3
- package/docs/contracts/schemas.md +6 -5
- package/package.json +8 -2
package/dist/study/plan-base.js
CHANGED
|
@@ -20,7 +20,9 @@ function residualOf(config) {
|
|
|
20
20
|
...(policies === undefined ? {} : { policies }),
|
|
21
21
|
...(defaults === undefined ? {} : { defaults }),
|
|
22
22
|
...(review === undefined ? {} : { review }),
|
|
23
|
-
...(config.execution?.desktop === undefined &&
|
|
23
|
+
...(config.execution?.desktop === undefined &&
|
|
24
|
+
config.execution?.target === undefined &&
|
|
25
|
+
config.execution?.network === undefined
|
|
24
26
|
? {}
|
|
25
27
|
: {
|
|
26
28
|
execution: {
|
|
@@ -30,6 +32,9 @@ function residualOf(config) {
|
|
|
30
32
|
...(config.execution?.target === undefined
|
|
31
33
|
? {}
|
|
32
34
|
: { target: config.execution.target }),
|
|
35
|
+
...(config.execution?.network === undefined
|
|
36
|
+
? {}
|
|
37
|
+
: { network: config.execution.network }),
|
|
33
38
|
},
|
|
34
39
|
}),
|
|
35
40
|
subject: {
|
|
@@ -4,6 +4,7 @@ import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
|
4
4
|
import type { BrowserSurface } from "../actors/scripted-browser/types.js";
|
|
5
5
|
import type { ScriptedRefusal } from "../routes/scripted/plan.js";
|
|
6
6
|
import type { TerminalRefusal } from "../routes/terminal/plan.js";
|
|
7
|
+
import type { SandboxProviderId } from "../run/sandbox-receipts.js";
|
|
7
8
|
import type { RunStudyProvenance } from "../run/study-provenance.js";
|
|
8
9
|
import type { ArrivalPlan } from "./arrivals.js";
|
|
9
10
|
import type { BundleEstimate } from "./bundle-estimate.js";
|
|
@@ -19,7 +20,7 @@ type AtLeastTwo<T> = readonly [T, T, ...T[]];
|
|
|
19
20
|
*/
|
|
20
21
|
export type Requirement = {
|
|
21
22
|
readonly kind: "key";
|
|
22
|
-
readonly name: "OPENAI_API_KEY" | "E2B_API_KEY";
|
|
23
|
+
readonly name: "OPENAI_API_KEY" | "E2B_API_KEY" | "MODAL_TOKEN_ID" | "MODAL_TOKEN_SECRET";
|
|
23
24
|
}
|
|
24
25
|
/** The terminal runtime key: CODEX_API_KEY, else OPENAI_API_KEY. */
|
|
25
26
|
| {
|
|
@@ -51,7 +52,7 @@ export type Requirement = {
|
|
|
51
52
|
* fields at run time.
|
|
52
53
|
*/
|
|
53
54
|
export type ResidualConfig = Pick<StudyConfig, "comms" | "policies" | "defaults" | "review"> & {
|
|
54
|
-
readonly execution?: Pick<NonNullable<StudyConfig["execution"]>, "desktop" | "target">;
|
|
55
|
+
readonly execution?: Pick<NonNullable<StudyConfig["execution"]>, "desktop" | "target" | "network">;
|
|
55
56
|
readonly subject: Pick<StudyConfig["subject"], "clone" | "localTree" | "repos" | "envValues">;
|
|
56
57
|
};
|
|
57
58
|
export interface PlannedAnalysis {
|
|
@@ -223,6 +224,8 @@ export interface SharedWorldPlan extends PlanBase {
|
|
|
223
224
|
}
|
|
224
225
|
export type TerminalPlan = PlanBase & {
|
|
225
226
|
readonly route: "terminal";
|
|
227
|
+
/** Where the shell sandbox runs: `execution.target` modal-terminal, else E2B. */
|
|
228
|
+
readonly sandboxProvider: SandboxProviderId;
|
|
226
229
|
/** The registered terminal actor that runs. */
|
|
227
230
|
readonly actor: string;
|
|
228
231
|
readonly product: StudySubjectProduct & {
|
|
@@ -9,9 +9,14 @@ import { localAgentRefusal } from "../actors/local-agent/readiness.js";
|
|
|
9
9
|
import { describeMissingKeys } from "../keys/key-resolution.js";
|
|
10
10
|
import { MODEL_RATES, unpricedCapMessage } from "../run/pricing.js";
|
|
11
11
|
/** Provider keys in the order the routes name them in a refusal. */
|
|
12
|
-
const KEY_ORDER = [
|
|
12
|
+
const KEY_ORDER = [
|
|
13
|
+
"OPENAI_API_KEY",
|
|
14
|
+
"E2B_API_KEY",
|
|
15
|
+
"MODAL_TOKEN_ID",
|
|
16
|
+
"MODAL_TOKEN_SECRET",
|
|
17
|
+
];
|
|
13
18
|
const present = (env, name) => (env[name]?.trim() ?? "") !== "";
|
|
14
|
-
/** The `key` requirements whose value `env` lacks,
|
|
19
|
+
/** The `key` requirements whose value `env` lacks, in KEY_ORDER. */
|
|
15
20
|
function missingKeys(requirements, env) {
|
|
16
21
|
const required = new Set(requirements.flatMap((requirement) => (requirement.kind === "key" ? [requirement.name] : [])));
|
|
17
22
|
return KEY_ORDER.filter((name) => required.has(name) && !present(env, name));
|
package/dist/study/routing.d.ts
CHANGED
|
@@ -69,7 +69,7 @@ export declare function isScriptedBrowserComposition(config: StudyConfig): boole
|
|
|
69
69
|
export declare function isProvisionedScriptedBrowserComposition(config: StudyConfig): boolean;
|
|
70
70
|
/**
|
|
71
71
|
* True when this config routes to the terminal-product backend: a terminal-product subject whose
|
|
72
|
-
* first actor resolves to a registered terminal actor (execution.target e2b-terminal or absent;
|
|
72
|
+
* first actor resolves to a registered terminal actor (execution.target e2b-terminal, modal-terminal or absent;
|
|
73
73
|
* the parse layer enforces that pairing). Mirror of isComputerUseComposition/isScriptedBrowserComposition;
|
|
74
74
|
* the single source of truth for routeOf and the warning logic.
|
|
75
75
|
*/
|
package/dist/study/routing.js
CHANGED
|
@@ -177,7 +177,7 @@ export function isProvisionedScriptedBrowserComposition(config) {
|
|
|
177
177
|
}
|
|
178
178
|
/**
|
|
179
179
|
* True when this config routes to the terminal-product backend: a terminal-product subject whose
|
|
180
|
-
* first actor resolves to a registered terminal actor (execution.target e2b-terminal or absent;
|
|
180
|
+
* first actor resolves to a registered terminal actor (execution.target e2b-terminal, modal-terminal or absent;
|
|
181
181
|
* the parse layer enforces that pairing). Mirror of isComputerUseComposition/isScriptedBrowserComposition;
|
|
182
182
|
* the single source of truth for routeOf and the warning logic.
|
|
183
183
|
*/
|
|
@@ -1,7 +1,13 @@
|
|
|
1
|
-
import type { StudyConfig, StudyParticipantEntry } from "./types.js";
|
|
1
|
+
import type { StudyConfig, StudyNetworkCapture, StudyParticipantEntry } from "./types.js";
|
|
2
2
|
/** The participants listed one by one: a `participants` list. Undefined when none is listed. */
|
|
3
3
|
export declare function participantList(config: StudyConfig): readonly StudyParticipantEntry[] | undefined;
|
|
4
4
|
/** The declared number of identical participants: `participants: <n>` or `participants.count`. */
|
|
5
5
|
export declare function declaredParticipantCount(config: StudyConfig): number | undefined;
|
|
6
6
|
/** The steer each of a counted group of participants gets: `participants.instruction`. */
|
|
7
7
|
export declare function participantInstruction(config: StudyConfig): string | undefined;
|
|
8
|
+
/**
|
|
9
|
+
* The origins each participant's network log keeps beside its target's
|
|
10
|
+
* (`execution.network.origins`), or undefined when capture is off: `execution.network` absent or
|
|
11
|
+
* false.
|
|
12
|
+
*/
|
|
13
|
+
export declare function networkCaptureOrigins(network: StudyNetworkCapture | undefined): readonly string[] | undefined;
|
|
@@ -17,6 +17,16 @@ export function declaredParticipantCount(config) {
|
|
|
17
17
|
export function participantInstruction(config) {
|
|
18
18
|
return isGroup(config.participants) ? config.participants.instruction : undefined;
|
|
19
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* The origins each participant's network log keeps beside its target's
|
|
22
|
+
* (`execution.network.origins`), or undefined when capture is off: `execution.network` absent or
|
|
23
|
+
* false.
|
|
24
|
+
*/
|
|
25
|
+
export function networkCaptureOrigins(network) {
|
|
26
|
+
if (network === undefined || network === false)
|
|
27
|
+
return undefined;
|
|
28
|
+
return network === true ? [] : (network.origins ?? []);
|
|
29
|
+
}
|
|
20
30
|
function isGroup(participants) {
|
|
21
31
|
return typeof participants === "object" && participants !== null && !Array.isArray(participants);
|
|
22
32
|
}
|
package/dist/study/types.d.ts
CHANGED
|
@@ -467,7 +467,7 @@ export interface StudyActor {
|
|
|
467
467
|
*/
|
|
468
468
|
dwell?: DwellWindow;
|
|
469
469
|
}
|
|
470
|
-
type StudyExecutionTarget = "local" | "e2b-desktop" | "e2b-terminal";
|
|
470
|
+
type StudyExecutionTarget = "local" | "e2b-desktop" | "e2b-terminal" | "modal-terminal";
|
|
471
471
|
/** Terminal transport: the captured non-interactive exec stream (stdin disabled). It is not an
|
|
472
472
|
* interactive duplex PTY; labeling captured exec output "pty" would be a claim/mechanism
|
|
473
473
|
* mismatch, so this route uses "exec-stream". */
|
|
@@ -570,6 +570,15 @@ export interface StudyDesktopFidelity {
|
|
|
570
570
|
/** Full user-agent string to present; default: a mobile Safari string. */
|
|
571
571
|
userAgent?: string;
|
|
572
572
|
}
|
|
573
|
+
/**
|
|
574
|
+
* `execution.network`: record each browser participant's requests to `network/<id>.ndjson`.
|
|
575
|
+
* `true` keeps the requests to the participant's target origin; `origins` adds up to 16 more exact
|
|
576
|
+
* http(s) origins. Only computer-use participants in hosted Chrome or Chromium on
|
|
577
|
+
* `execution.target: e2b-desktop` read it; every other route refuses it.
|
|
578
|
+
*/
|
|
579
|
+
export type StudyNetworkCapture = boolean | {
|
|
580
|
+
origins?: string[];
|
|
581
|
+
};
|
|
573
582
|
export interface StudyExecution {
|
|
574
583
|
target?: StudyExecutionTarget;
|
|
575
584
|
/** Actor session wall-clock budget. Consumed on the app-url route. */
|
|
@@ -582,6 +591,8 @@ export interface StudyExecution {
|
|
|
582
591
|
*/
|
|
583
592
|
concurrency?: number;
|
|
584
593
|
desktop?: StudyExecutionDesktop;
|
|
594
|
+
/** Each browser participant's network log; see StudyNetworkCapture. Absent or false: off. */
|
|
595
|
+
network?: StudyNetworkCapture;
|
|
585
596
|
/** `terminal-product` route: the terminal transport + stdin posture. Consumed on that route. */
|
|
586
597
|
terminal?: StudyExecutionTerminal;
|
|
587
598
|
/** `terminal-product` route: runtime key placement, defaulting to openai-egress, an external
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type StudyRoute } from "./routing.js";
|
|
1
2
|
import type { StudyConfig } from "./types.js";
|
|
2
3
|
/**
|
|
3
4
|
* Cross-validate the computer-use fan-out declaration (`per-lane-worlds`). Returns the failure
|
|
@@ -21,6 +22,13 @@ export declare function smtpValidationReason(config: StudyConfig): string | unde
|
|
|
21
22
|
* topology may not identify the backend that is actually executing.
|
|
22
23
|
*/
|
|
23
24
|
export declare function desktopMediaValidationReason(config: StudyConfig, supportsMedia?: boolean): string | undefined;
|
|
25
|
+
/**
|
|
26
|
+
* Why a study's `execution.network` cannot run where it is going, or undefined. Only a
|
|
27
|
+
* computer-use participant in hosted Chrome or Chromium has the in-sandbox DevTools client that
|
|
28
|
+
* records requests. Enforced at parse and again by each route's planner, which passes its own
|
|
29
|
+
* route, for a library caller's config.
|
|
30
|
+
*/
|
|
31
|
+
export declare function networkCaptureValidationReason(config: StudyConfig, route?: StudyRoute): string | undefined;
|
|
24
32
|
/** Reused by direct library runners so unsupported receiving never becomes inert configuration. */
|
|
25
33
|
export declare function receivingEmailValidationReason(config: StudyConfig): string | undefined;
|
|
26
34
|
/** Refuse task declarations that the selected execution path would discard.
|
package/dist/study/validation.js
CHANGED
|
@@ -4,8 +4,8 @@ import { waitFieldsReason } from "../actors/computer-use/wait.js";
|
|
|
4
4
|
import { PARTICIPANT_ID_MAX_CHARS, PARTICIPANT_ID_PATTERN } from "./parse/actors.js";
|
|
5
5
|
import { isHttpUrl, isLoopbackUrl } from "./parse/subject.js";
|
|
6
6
|
import { declaredTargets } from "./plan-participants.js";
|
|
7
|
-
import { actorResolvesToComputerUse, computerUseParticipantCount, participantCountReason, registeredComputerUseActors, resolveEntryUrl, isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, } from "./routing.js";
|
|
8
|
-
import { participantList } from "./study-fields.js";
|
|
7
|
+
import { actorResolvesToComputerUse, computerUseParticipantCount, participantCountReason, registeredComputerUseActors, resolveEntryUrl, isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, routeOf, } from "./routing.js";
|
|
8
|
+
import { networkCaptureOrigins, participantList } from "./study-fields.js";
|
|
9
9
|
import { isStartAfterMs, rosterStartsReason, startAfterMessage, windowReason } from "./arrivals.js";
|
|
10
10
|
/**
|
|
11
11
|
* Cross-validate the computer-use fan-out declaration (`per-lane-worlds`). Returns the failure
|
|
@@ -206,6 +206,35 @@ export function desktopMediaValidationReason(config, supportsMedia = isComputerU
|
|
|
206
206
|
}
|
|
207
207
|
return undefined;
|
|
208
208
|
}
|
|
209
|
+
/**
|
|
210
|
+
* Why a study's `execution.network` cannot run where it is going, or undefined. Only a
|
|
211
|
+
* computer-use participant in hosted Chrome or Chromium has the in-sandbox DevTools client that
|
|
212
|
+
* records requests. Enforced at parse and again by each route's planner, which passes its own
|
|
213
|
+
* route, for a library caller's config.
|
|
214
|
+
*/
|
|
215
|
+
export function networkCaptureValidationReason(config, route = routeOf(config)) {
|
|
216
|
+
if (networkCaptureOrigins(config.execution?.network) === undefined)
|
|
217
|
+
return undefined;
|
|
218
|
+
const hosted = "only computer-use participants in hosted Chrome or Chromium on `execution.target: e2b-desktop` record their requests";
|
|
219
|
+
if (route === "terminal")
|
|
220
|
+
return "`execution.network` records each browser participant's requests, and route: terminal launches no browser: its participant works in a shell. Remove `execution.network`.";
|
|
221
|
+
if (route === "shared-world")
|
|
222
|
+
return "`execution.network` is not supported on route: shared-world yet: a shared world's lobby code sits in a page address the network log would keep, and that route keeps observed addresses only as digests. Remove `execution.network`, or run independent participants on route: computer-use.";
|
|
223
|
+
if (route === "scripted")
|
|
224
|
+
return `\`execution.network\` is not supported on route: scripted yet: ${hosted}. Remove \`execution.network\`.`;
|
|
225
|
+
if (route !== "computer-use")
|
|
226
|
+
return `\`execution.network\` is not read on route: ${route}: ${hosted}. Remove \`execution.network\`.`;
|
|
227
|
+
if (config.subject.source === "desktop-cli")
|
|
228
|
+
return "`execution.network` records each browser participant's requests, and a desktop-cli subject opens a terminal window and no browser, so there are none to record. Remove `execution.network`.";
|
|
229
|
+
if (config.subject.source === "local-app")
|
|
230
|
+
return `\`execution.network\` is not supported for a local-app subject: humanish does not launch that participant's browser, and ${hosted}. Remove \`execution.network\`.`;
|
|
231
|
+
const target = config.execution?.target;
|
|
232
|
+
if (target !== "e2b-desktop")
|
|
233
|
+
return `\`execution.network\` is not supported with \`execution.target: ${target ?? "(absent)"}\` yet: ${hosted}. Set \`execution.target: e2b-desktop\`, or remove \`execution.network\`.`;
|
|
234
|
+
if (config.execution?.desktop?.browser === "firefox")
|
|
235
|
+
return "`execution.network` needs Chrome or Chromium, and `execution.desktop.browser: firefox` launches Firefox with no DevTools port to record requests through. Set `execution.desktop.browser` to chrome, chromium or default, or remove `execution.network`.";
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
209
238
|
/** Reused by direct library runners so unsupported receiving never becomes inert configuration. */
|
|
210
239
|
export function receivingEmailValidationReason(config) {
|
|
211
240
|
if (config.comms?.email?.kind !== "real")
|
|
@@ -27,6 +27,11 @@ export interface DetachedStepResult {
|
|
|
27
27
|
}
|
|
28
28
|
/** Read the capped log tail for a step (raw; caller redacts). */
|
|
29
29
|
export declare function readDetachedLog(shell: Shell, name: string, requestTimeoutMs?: number): Promise<string>;
|
|
30
|
+
/**
|
|
31
|
+
* Whether a process started by startDetachedProcess or runDetachedStep has exited. Its wrapper
|
|
32
|
+
* writes a status file however the command ended, a signal included.
|
|
33
|
+
*/
|
|
34
|
+
export declare function detachedProcessExited(shell: Shell, name: string, requestTimeoutMs?: number): Promise<boolean>;
|
|
30
35
|
/**
|
|
31
36
|
* Run a bounded step (install/build) detached, polling its atomic status file until it
|
|
32
37
|
* exits or the budget runs out. On timeout the process group is killed and the log tail is
|
|
@@ -75,6 +75,15 @@ export async function readDetachedLog(shell, name, requestTimeoutMs = DEFAULT_RE
|
|
|
75
75
|
const result = await runOrThrow(shell, `tail -c ${LOG_TAIL_BYTES} ${shellQuote(`${stepDir(name)}/log.txt`)} 2>/dev/null || true`, { requestTimeoutMs });
|
|
76
76
|
return result.stdout;
|
|
77
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Whether a process started by startDetachedProcess or runDetachedStep has exited. Its wrapper
|
|
80
|
+
* writes a status file however the command ended, a signal included.
|
|
81
|
+
*/
|
|
82
|
+
export async function detachedProcessExited(shell, name, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
|
|
83
|
+
assertName(name);
|
|
84
|
+
const result = await runOrThrow(shell, `test -e ${shellQuote(`${stepDir(name)}/status`)} && echo exited || true`, { requestTimeoutMs });
|
|
85
|
+
return result.stdout.trim() === "exited";
|
|
86
|
+
}
|
|
78
87
|
/**
|
|
79
88
|
* Run a bounded step (install/build) detached, polling its atomic status file until it
|
|
80
89
|
* exits or the budget runs out. On timeout the process group is killed and the log tail is
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/** The `network` field of a hold request. */
|
|
2
|
+
export interface ChromeNetworkCaptureRequest {
|
|
3
|
+
/**
|
|
4
|
+
* Sandbox file the holder appends to; its folder is created. Defaults to
|
|
5
|
+
* /tmp/humanish-network/events.ndjson.
|
|
6
|
+
*/
|
|
7
|
+
eventsPath?: string;
|
|
8
|
+
/** Origins, as `new URL(url).origin` writes them, whose request addresses the holder writes. */
|
|
9
|
+
origins: string[];
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The capture half of the holder, spliced into CHROME_CDP_PROBE_PY so the shipped script stays one
|
|
13
|
+
* string. Line kinds: `start` (Network answered on the launch page), `request`, `response`,
|
|
14
|
+
* `servedFromCache`, `finished`, `failed`, `backForward` (a main-frame restore from the
|
|
15
|
+
* back-forward cache, which sends no request), `cap` (the file reached 32 MiB and writing stopped)
|
|
16
|
+
* and `exit` (the holder stopped; written in a finally). Ids are `<session>:<requestId>`, the
|
|
17
|
+
* session numbered in attach order from 0 for the launch page, because request ids repeat across
|
|
18
|
+
* renderers. `wallTime` is seconds since the epoch on the sandbox clock; `timestamp` is CDP's
|
|
19
|
+
* monotonic clock, for durations. A path longer than 4,096 characters ends at its last whole
|
|
20
|
+
* segment within them, then `/[TRUNCATED]`: the host redacts each segment, and a token cut in two
|
|
21
|
+
* can fall below the length its rules withhold.
|
|
22
|
+
*/
|
|
23
|
+
export declare const NETWORK_CAPTURE_PY: string;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// What the in-sandbox holder keeps from a participant's browser traffic. The holder (cdp-probe.ts,
|
|
2
|
+
// `mode: "hold"`) already owns the one long-lived DevTools socket on a hosted desktop; with a
|
|
3
|
+
// network request it also enables the Network domain and appends one short JSON line per kept
|
|
4
|
+
// event to a file the host reads at teardown.
|
|
5
|
+
//
|
|
6
|
+
// The CDP events carry headers, cookies, post data, initiator stacks and full URLs. A line copies
|
|
7
|
+
// none of them: only the request id, the two clocks, method, resource type, status, cache flags,
|
|
8
|
+
// error text, and an address for an allowed origin, cut at the first `?` or `#` with user info
|
|
9
|
+
// removed. A request to any other origin gets a line with no address, method or type, so the host
|
|
10
|
+
// can count it, and nothing more is written for it. A method or resource type that is not plain
|
|
11
|
+
// letters is written as `Other`, and error text outside Chrome's `net::ERR_*` form as
|
|
12
|
+
// `net::ERR_OTHER`, so no free text reaches the file.
|
|
13
|
+
/** The segment a path the holder cut ends with: the marker tracePage ends a cut path with. */
|
|
14
|
+
const PATH_CUT = "[TRUNCATED]";
|
|
15
|
+
/**
|
|
16
|
+
* The capture half of the holder, spliced into CHROME_CDP_PROBE_PY so the shipped script stays one
|
|
17
|
+
* string. Line kinds: `start` (Network answered on the launch page), `request`, `response`,
|
|
18
|
+
* `servedFromCache`, `finished`, `failed`, `backForward` (a main-frame restore from the
|
|
19
|
+
* back-forward cache, which sends no request), `cap` (the file reached 32 MiB and writing stopped)
|
|
20
|
+
* and `exit` (the holder stopped; written in a finally). Ids are `<session>:<requestId>`, the
|
|
21
|
+
* session numbered in attach order from 0 for the launch page, because request ids repeat across
|
|
22
|
+
* renderers. `wallTime` is seconds since the epoch on the sandbox clock; `timestamp` is CDP's
|
|
23
|
+
* monotonic clock, for durations. A path longer than 4,096 characters ends at its last whole
|
|
24
|
+
* segment within them, then `/[TRUNCATED]`: the host redacts each segment, and a token cut in two
|
|
25
|
+
* can fall below the length its rules withhold.
|
|
26
|
+
*/
|
|
27
|
+
export const NETWORK_CAPTURE_PY = String.raw `
|
|
28
|
+
NETWORK_ENABLE = {"maxTotalBufferSize": 0, "maxResourceBufferSize": 0, "maxPostDataSize": 0}
|
|
29
|
+
NETWORK_EVENTS_CAP = 32 * 1024 * 1024
|
|
30
|
+
# Kept free under the cap so the cap and exit lines always fit.
|
|
31
|
+
NETWORK_TAIL_RESERVE = 512
|
|
32
|
+
NETWORK_PATH_LIMIT = 4096
|
|
33
|
+
WEB_ADDRESS = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://(?:[^/?#]*@)?([^/?#]*)([^?#]*)")
|
|
34
|
+
NETWORK_WORD = re.compile(r"^[A-Za-z]{1,32}$")
|
|
35
|
+
NETWORK_ERROR = re.compile(r"^net::ERR_[A-Z0-9_]{1,64}$")
|
|
36
|
+
|
|
37
|
+
def allowed_address(url, origins):
|
|
38
|
+
"""origin + path for an allowed origin, without user info, query or fragment; else None."""
|
|
39
|
+
match = WEB_ADDRESS.match(str(url or ""))
|
|
40
|
+
if not match:
|
|
41
|
+
return None
|
|
42
|
+
origin = ("%s://%s" % (match.group(1), match.group(2))).lower()
|
|
43
|
+
if origin not in origins:
|
|
44
|
+
return None
|
|
45
|
+
path = match.group(3) or "/"
|
|
46
|
+
if len(path) > NETWORK_PATH_LIMIT:
|
|
47
|
+
path = path[:path.rfind("/", 0, NETWORK_PATH_LIMIT + 1)] + "/${PATH_CUT}"
|
|
48
|
+
return origin + path
|
|
49
|
+
|
|
50
|
+
def network_word(value, fallback):
|
|
51
|
+
text = str(value or "")
|
|
52
|
+
return text if NETWORK_WORD.match(text) else fallback
|
|
53
|
+
|
|
54
|
+
def clock(value):
|
|
55
|
+
return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
|
56
|
+
|
|
57
|
+
def response_fields(response):
|
|
58
|
+
fields = {}
|
|
59
|
+
if not isinstance(response, dict):
|
|
60
|
+
return fields
|
|
61
|
+
status = clock(response.get("status"))
|
|
62
|
+
if status is not None and 100 <= status <= 599:
|
|
63
|
+
fields["status"] = int(status)
|
|
64
|
+
for flag in ("fromDiskCache", "fromPrefetchCache", "fromServiceWorker"):
|
|
65
|
+
if response.get(flag) is True:
|
|
66
|
+
fields[flag] = True
|
|
67
|
+
return fields
|
|
68
|
+
|
|
69
|
+
class NetworkLog:
|
|
70
|
+
"""The events file of one holder. Writes go straight to the file, one line per write, so a
|
|
71
|
+
read while the holder runs sees whole lines except possibly the last."""
|
|
72
|
+
def __init__(self, request):
|
|
73
|
+
self.path = str(request.get("eventsPath") or "/tmp/humanish-network/events.ndjson")
|
|
74
|
+
self.origins = set(str(origin).rstrip("/").lower() for origin in request.get("origins") or [] if isinstance(origin, str))
|
|
75
|
+
folder = os.path.dirname(self.path)
|
|
76
|
+
if folder:
|
|
77
|
+
os.makedirs(folder, exist_ok=True)
|
|
78
|
+
self.handle = open(self.path, "ab", buffering=0)
|
|
79
|
+
self.written = os.fstat(self.handle.fileno()).st_size
|
|
80
|
+
self.capped = False
|
|
81
|
+
self.sessions = {}
|
|
82
|
+
# Ids whose current hop has an allowed address; only those get lines after the request.
|
|
83
|
+
self.open = set()
|
|
84
|
+
self.faults = 0
|
|
85
|
+
|
|
86
|
+
def session(self, session_id):
|
|
87
|
+
if session_id not in self.sessions:
|
|
88
|
+
self.sessions[session_id] = len(self.sessions)
|
|
89
|
+
return self.sessions[session_id]
|
|
90
|
+
|
|
91
|
+
def write(self, line, tail=False):
|
|
92
|
+
if self.capped and not tail:
|
|
93
|
+
return
|
|
94
|
+
data = (json.dumps({key: value for key, value in line.items() if value is not None}, separators=(",", ":")) + "\n").encode("utf-8")
|
|
95
|
+
room = NETWORK_EVENTS_CAP - (0 if tail else NETWORK_TAIL_RESERVE)
|
|
96
|
+
if self.written + len(data) > room:
|
|
97
|
+
if not tail:
|
|
98
|
+
self.capped = True
|
|
99
|
+
self.write({"e": "cap", "wallTime": time.time()}, tail=True)
|
|
100
|
+
return
|
|
101
|
+
self.handle.write(data)
|
|
102
|
+
self.written += len(data)
|
|
103
|
+
|
|
104
|
+
def event(self, method, params, session):
|
|
105
|
+
"""Record one CDP event. A fault is counted, never raised: emulation must keep running."""
|
|
106
|
+
try:
|
|
107
|
+
self.record(method, params, session)
|
|
108
|
+
except Exception:
|
|
109
|
+
self.faults += 1
|
|
110
|
+
|
|
111
|
+
def record(self, method, params, session):
|
|
112
|
+
# Past the cap nothing more is written, so nothing more is tracked.
|
|
113
|
+
if self.capped:
|
|
114
|
+
return
|
|
115
|
+
if method == "Page.frameNavigated":
|
|
116
|
+
frame = params.get("frame") or {}
|
|
117
|
+
if params.get("type") == "BackForwardCacheRestore" and not frame.get("parentId"):
|
|
118
|
+
self.write({"e": "backForward", "wallTime": time.time(), "url": allowed_address(frame.get("url"), self.origins)})
|
|
119
|
+
return
|
|
120
|
+
if not str(method).startswith("Network."):
|
|
121
|
+
return
|
|
122
|
+
kind = method[len("Network."):]
|
|
123
|
+
request_id = "%d:%s" % (self.session(session), str(params.get("requestId") or "")[:64])
|
|
124
|
+
timestamp = clock(params.get("timestamp"))
|
|
125
|
+
if kind == "requestWillBeSent":
|
|
126
|
+
request = params.get("request") or {}
|
|
127
|
+
line = {"e": "request", "id": request_id, "wallTime": clock(params.get("wallTime")), "timestamp": timestamp}
|
|
128
|
+
if "redirectResponse" in params:
|
|
129
|
+
line["redirectResponse"] = response_fields(params.get("redirectResponse"))
|
|
130
|
+
address = allowed_address(request.get("url"), self.origins)
|
|
131
|
+
if address is None:
|
|
132
|
+
self.open.discard(request_id)
|
|
133
|
+
else:
|
|
134
|
+
self.open.add(request_id)
|
|
135
|
+
line["url"] = address
|
|
136
|
+
line["method"] = network_word(request.get("method"), "Other")
|
|
137
|
+
line["type"] = network_word(params.get("type"), "Other")
|
|
138
|
+
self.write(line)
|
|
139
|
+
elif request_id not in self.open:
|
|
140
|
+
return
|
|
141
|
+
elif kind == "responseReceived":
|
|
142
|
+
line = {"e": "response", "id": request_id, "timestamp": timestamp}
|
|
143
|
+
line.update(response_fields(params.get("response")))
|
|
144
|
+
self.write(line)
|
|
145
|
+
elif kind == "requestServedFromCache":
|
|
146
|
+
self.write({"e": "servedFromCache", "id": request_id})
|
|
147
|
+
elif kind == "loadingFinished":
|
|
148
|
+
self.open.discard(request_id)
|
|
149
|
+
self.write({"e": "finished", "id": request_id, "timestamp": timestamp})
|
|
150
|
+
elif kind == "loadingFailed":
|
|
151
|
+
self.open.discard(request_id)
|
|
152
|
+
error = str(params.get("errorText") or "")
|
|
153
|
+
self.write({"e": "failed", "id": request_id, "timestamp": timestamp, "errorText": error if NETWORK_ERROR.match(error) else "net::ERR_OTHER"})
|
|
154
|
+
|
|
155
|
+
def close(self, reason):
|
|
156
|
+
line = {"e": "exit", "reason": str(reason)[:200]}
|
|
157
|
+
if self.faults:
|
|
158
|
+
line["faults"] = self.faults
|
|
159
|
+
try:
|
|
160
|
+
self.write(line, tail=True)
|
|
161
|
+
self.handle.close()
|
|
162
|
+
except Exception:
|
|
163
|
+
pass
|
|
164
|
+
|
|
165
|
+
def start_capture(ws, session_id, network):
|
|
166
|
+
"""Network on the launch session, answered before any Emulation message goes out, so a mobile
|
|
167
|
+
participant's launch reload is captured. Returns the log, or None, and the announce field."""
|
|
168
|
+
try:
|
|
169
|
+
log = NetworkLog(network)
|
|
170
|
+
except Exception as error:
|
|
171
|
+
return None, {"unavailable": "the events file could not be opened (%s)" % type(error).__name__}
|
|
172
|
+
# Read before the call, so a request whose events queued behind the reply started after it.
|
|
173
|
+
started = time.time()
|
|
174
|
+
try:
|
|
175
|
+
reply = ws.call("Network.enable", NETWORK_ENABLE, session_id)
|
|
176
|
+
except Exception as error:
|
|
177
|
+
reply = {"error": {"message": type(error).__name__}}
|
|
178
|
+
if not isinstance(reply, dict) or "error" in reply:
|
|
179
|
+
detail = str(((reply or {}).get("error") or {}).get("message")) if isinstance(reply, dict) else "no reply"
|
|
180
|
+
log.close("Network.enable failed: %s" % detail)
|
|
181
|
+
return None, {"unavailable": "Network.enable failed on the launch page: %s" % detail}
|
|
182
|
+
try:
|
|
183
|
+
log.session(session_id)
|
|
184
|
+
log.write({"e": "start", "wallTime": started})
|
|
185
|
+
except Exception as error:
|
|
186
|
+
log.close("the start line could not be written: %s" % type(error).__name__)
|
|
187
|
+
return None, {"unavailable": "the events file could not be written (%s)" % type(error).__name__}
|
|
188
|
+
# Page events report back-forward restores; without them only those lines are missing.
|
|
189
|
+
try:
|
|
190
|
+
ws.call("Page.enable", {}, session_id)
|
|
191
|
+
except Exception:
|
|
192
|
+
pass
|
|
193
|
+
return log, {"startedAt": started}
|
|
194
|
+
`;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ChromeNetworkCaptureRequest } from "./cdp-network-capture.js";
|
|
1
2
|
/** Which target to attribute when the endpoint lists several pages. */
|
|
2
3
|
export type ChromeCdpPagePreference = "pinned" | "active";
|
|
3
4
|
export interface ChromeCdpProbeArgs {
|
|
@@ -23,15 +24,17 @@ export interface ChromeCdpProbeArgs {
|
|
|
23
24
|
/**
|
|
24
25
|
* "state": url/title/text/scrollY. "geometry": outer window + CSS viewport. "port": resolution
|
|
25
26
|
* only. "emulate": apply mobile emulation to the selected page and exit (the overrides that
|
|
26
|
-
* are session-scoped, UA / touch / DPR, lapse when the socket closes). "hold":
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* are session-scoped, UA / touch / DPR, lapse when the socket closes). "hold": stay attached over
|
|
28
|
+
* a browser-level socket until killed and attach to every page target Chrome opens later, never
|
|
29
|
+
* pausing it; it applies `emulation` (how a participant keeps the overrides for its whole life,
|
|
30
|
+
* with a reload on each page) and `network` capture, either or both. "fidelity": read
|
|
30
31
|
* back what the page reports about itself (UA, DPR, viewport, touch), the proof for the bundle.
|
|
31
32
|
*/
|
|
32
33
|
mode: "state" | "geometry" | "port" | "emulate" | "hold" | "fidelity";
|
|
33
|
-
/** For "emulate": what to apply. */
|
|
34
|
+
/** For "emulate" and "hold": what to apply. */
|
|
34
35
|
emulation?: ChromeMobileEmulationRequest;
|
|
36
|
+
/** For "hold": capture each held page's requests to a sandbox file. */
|
|
37
|
+
network?: ChromeNetworkCaptureRequest;
|
|
35
38
|
}
|
|
36
39
|
/** Mobile emulation request: the CDP Emulation domain applied to one page target. */
|
|
37
40
|
export interface ChromeMobileEmulationRequest {
|
|
@@ -75,6 +78,18 @@ export interface ChromeCdpProbeResult {
|
|
|
75
78
|
applied?: string[];
|
|
76
79
|
/** "fidelity": the read-back. */
|
|
77
80
|
fidelity?: ChromeFidelityRead;
|
|
81
|
+
/** "hold" with `network`: when capture started on the launch page, or why it could not. */
|
|
82
|
+
network?: {
|
|
83
|
+
startedAt: number;
|
|
84
|
+
} | {
|
|
85
|
+
unavailable: string;
|
|
86
|
+
};
|
|
87
|
+
/**
|
|
88
|
+
* Seconds since the epoch on the sandbox clock when the probe printed, on every answer including
|
|
89
|
+
* an unavailable one. It is the clock CDP `wallTime` uses, so a turn mark compares with request
|
|
90
|
+
* times without host clock skew.
|
|
91
|
+
*/
|
|
92
|
+
now?: number;
|
|
78
93
|
}
|
|
79
94
|
/**
|
|
80
95
|
* The probe itself. Kept as one string so the shipped command is exactly what the tests execute
|