humanish 0.124.0 → 0.126.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/CONTEXT.md +1 -1
- package/README.md +2 -2
- package/dist/actors/computer-use/actor.d.ts +2 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/clock.d.ts +9 -8
- package/dist/actors/computer-use/loop/clock.js +27 -17
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -1
- package/dist/actors/computer-use/loop/debrief.js +15 -5
- package/dist/actors/computer-use/loop/ending.d.ts +8 -5
- package/dist/actors/computer-use/loop/ending.js +8 -5
- package/dist/actors/computer-use/loop/observation.d.ts +6 -0
- package/dist/actors/computer-use/loop/observation.js +19 -1
- package/dist/actors/computer-use/loop/session.d.ts +4 -2
- package/dist/actors/computer-use/loop/session.js +4 -1
- package/dist/actors/computer-use/loop/trace.js +4 -2
- package/dist/actors/computer-use/loop/types.d.ts +22 -8
- package/dist/actors/computer-use/loop.js +5 -5
- package/dist/actors/contract.d.ts +31 -8
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/evidence.js +2 -2
- package/dist/cli/commands/study.js +8 -0
- package/dist/evidence/redaction.d.ts +1 -0
- package/dist/evidence/redaction.js +1 -0
- package/dist/evidence/trace-page.d.ts +6 -0
- package/dist/evidence/trace-page.js +90 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/observer/feed-cursors.d.ts +14 -0
- package/dist/observer/feed-cursors.js +61 -0
- package/dist/observer/participant-items.d.ts +7 -0
- package/dist/observer/pinned-files.d.ts +4 -1
- package/dist/observer/pinned-files.js +10 -7
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +39 -17
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle.js +4 -0
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/desktop.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +6 -0
- package/dist/routes/computer-use/e2b-desktop/prepare.js +47 -1
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +2 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +15 -3
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -1
- package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
- package/dist/routes/computer-use/fanout-records.js +4 -0
- package/dist/routes/computer-use/live-flush.d.ts +9 -4
- package/dist/routes/computer-use/live-flush.js +16 -3
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-desktop.d.ts +1 -1
- package/dist/routes/computer-use/participant-model.js +16 -6
- package/dist/routes/computer-use/participant-runs.d.ts +2 -2
- package/dist/routes/computer-use/participant-runs.js +11 -7
- package/dist/routes/computer-use/plan.js +39 -1
- package/dist/routes/computer-use/setup.d.ts +1 -0
- package/dist/routes/computer-use/setup.js +28 -0
- package/dist/routes/computer-use/single-bundle.d.ts +5 -1
- package/dist/routes/computer-use/single-bundle.js +2 -0
- package/dist/routes/computer-use/types.d.ts +16 -10
- package/dist/routes/shared-world/bundle-records.js +3 -0
- package/dist/routes/shared-world/participant-specs.js +3 -2
- package/dist/routes/shared-world/plan.js +5 -0
- package/dist/run/bundle-shape.js +2 -2
- package/dist/run/bundle.d.ts +10 -3
- package/dist/run/bundle.js +11 -2
- package/dist/run/contained-output.d.ts +26 -0
- package/dist/run/contained-output.js +101 -1
- package/dist/run/evidence-files.d.ts +12 -1
- package/dist/run/evidence-files.js +70 -40
- package/dist/run/judge.d.ts +7 -4
- package/dist/run/judge.js +13 -8
- package/dist/run/live-items.d.ts +67 -0
- package/dist/run/live-items.js +262 -0
- package/dist/run/locate.d.ts +10 -0
- package/dist/run/locate.js +10 -0
- package/dist/run/run.d.ts +3 -2
- package/dist/run/run.js +29 -8
- package/dist/run/sandbox-ids.d.ts +0 -5
- package/dist/run/sandbox-ids.js +0 -7
- package/dist/run/stream-trace.d.ts +40 -10
- package/dist/run/stream-trace.js +151 -22
- package/dist/run/streams.d.ts +33 -2
- package/dist/study/arrivals.d.ts +32 -14
- package/dist/study/arrivals.js +47 -21
- package/dist/study/bundle-estimate.d.ts +6 -5
- package/dist/study/bundle-estimate.js +12 -7
- package/dist/study/desktop-files.d.ts +37 -0
- package/dist/study/desktop-files.js +174 -0
- package/dist/study/keys.d.ts +8 -0
- package/dist/study/keys.js +3 -0
- package/dist/study/migrate/v2.js +12 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +51 -26
- package/dist/study/parse/clock.d.ts +18 -0
- package/dist/study/parse/clock.js +36 -0
- package/dist/study/parse/execution.js +6 -0
- package/dist/study/plan-base.js +4 -1
- package/dist/study/plan-participants.d.ts +19 -5
- package/dist/study/plan-participants.js +17 -1
- package/dist/study/plan.js +24 -13
- package/dist/study/preflight.js +4 -0
- package/dist/study/types.d.ts +34 -5
- package/dist/study/validation.js +3 -0
- package/dist/study/warnings.d.ts +2 -2
- package/dist/study/warnings.js +7 -0
- package/dist/subject/local-tree-archive.d.ts +7 -0
- package/dist/subject/local-tree-archive.js +12 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -1
- package/dist/substrates/e2b/desktop-browser.js +6 -2
- package/dist/substrates/e2b/desktop-clock.d.ts +10 -0
- package/dist/substrates/e2b/desktop-clock.js +29 -0
- package/dist/substrates/e2b/sdk.d.ts +6 -0
- package/dist/verify/artifacts.js +39 -51
- package/dist/verify/pages.d.ts +7 -0
- package/dist/verify/pages.js +50 -0
- package/dist/verify/screenshot-reads.d.ts +21 -0
- package/dist/verify/screenshot-reads.js +46 -0
- package/dist/verify/verify.js +17 -9
- package/docs/architecture/actor-contract.md +2 -0
- package/docs/architecture/external-public-shared-world.md +4 -1
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/observer.md +10 -1
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/policy.md +7 -0
- package/docs/contracts/run-bundle.md +106 -14
- package/docs/contracts/schemas.md +110 -40
- package/package.json +4 -2
|
@@ -37,12 +37,21 @@ export type ParticipantDeclaredOutcome = "reached" | "not_reached" | "blocked";
|
|
|
37
37
|
export declare const PARTICIPANT_OUTCOME_STATUSES: readonly ActorStatus[];
|
|
38
38
|
/** Optional precise interruption cause; completionReason and status retain their original meaning.
|
|
39
39
|
* One list, so the analysis schema and the diagnostics projection cannot drift from the trace. */
|
|
40
|
-
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end"];
|
|
40
|
+
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "session_cap"];
|
|
41
41
|
export type ActorStopCause = (typeof ACTOR_STOP_CAUSES)[number];
|
|
42
42
|
export type ActorCompletionReason = "goal_satisfied" | "turn_completed" | "gave_up" | "blocked_approval" | "timed_out" | "budget_reached" | "actor_error" | "step_failed" | "harness_error";
|
|
43
43
|
type ActorRunKind = "code" | "computer-use" | "scripted-browser" | "terminal";
|
|
44
44
|
type ActorProtocol = "json-rpc" | "json-stream" | "cua-loop" | "scripted-steps" | "terminal-exec";
|
|
45
45
|
export type ActorTraceItemKind = "message" | "reasoning" | "tool_call" | "command" | "file_change" | "approval" | "screenshot" | "ui_action" | "plan" | "notice";
|
|
46
|
+
/**
|
|
47
|
+
* Where a participant's browser was at one observation: the URL's origin and its path, redacted
|
|
48
|
+
* (src/evidence/trace-page.ts). No query string, fragment or user info. A non-web URL keeps only
|
|
49
|
+
* its scheme, as `{ origin: "about:", path: "" }`.
|
|
50
|
+
*/
|
|
51
|
+
export interface TracePage {
|
|
52
|
+
origin: string;
|
|
53
|
+
path: string;
|
|
54
|
+
}
|
|
46
55
|
export interface ActorTraceItem {
|
|
47
56
|
id: string;
|
|
48
57
|
kind: ActorTraceItemKind;
|
|
@@ -74,6 +83,9 @@ export interface ActorTraceItem {
|
|
|
74
83
|
x: number;
|
|
75
84
|
y: number;
|
|
76
85
|
};
|
|
86
|
+
/** On a computer-use `screenshot` item: the page that observation was on. Additive: absent when
|
|
87
|
+
* the observation read no URL, on other producers and in older bundles. */
|
|
88
|
+
page?: TracePage;
|
|
77
89
|
}
|
|
78
90
|
export interface ActorCapabilities {
|
|
79
91
|
headless: boolean;
|
|
@@ -338,19 +350,24 @@ export interface ActorTrace {
|
|
|
338
350
|
*/
|
|
339
351
|
taskFunnel?: TaskFunnel;
|
|
340
352
|
/**
|
|
341
|
-
* Additive + optional: the outcome the participant declared
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
353
|
+
* Additive + optional: the outcome the participant declared, in its provider's outcome field or
|
|
354
|
+
* in the fixed first line its prompt asks for. It comes from its last message when it ended the
|
|
355
|
+
* session itself, and from its accepted closing report at a scheduled end or a session cap.
|
|
356
|
+
* Absent when it declared none, after a stop condition or a dwell window, and on every older
|
|
357
|
+
* bundle. The route reads a declared `reached` or `blocked` before it reads the closing
|
|
358
|
+
* paragraph; three regex patches in one month each fixed a false refusal and each left the next
|
|
359
|
+
* shape unhandled. A declared `not_reached` leaves the paragraph to say whether a blocker
|
|
360
|
+
* stopped the participant.
|
|
345
361
|
*/
|
|
346
362
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
347
363
|
/**
|
|
348
364
|
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
349
|
-
* `dwell`, `scheduled_end`), or impressions only after the participant ended the
|
|
350
|
-
* (`participant_end`, never with `report`). Does not change task outcomes or
|
|
365
|
+
* `dwell`, `scheduled_end`, `session_cap`), or impressions only after the participant ended the
|
|
366
|
+
* session itself (`participant_end`, never with `report`). Does not change task outcomes or
|
|
367
|
+
* permit actions.
|
|
351
368
|
*/
|
|
352
369
|
debrief?: {
|
|
353
|
-
trigger: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
370
|
+
trigger: "stop_when" | "dwell" | "scheduled_end" | "session_cap" | "participant_end";
|
|
354
371
|
status: "completed" | "skipped" | "failed";
|
|
355
372
|
reason: string;
|
|
356
373
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
|
@@ -366,6 +383,12 @@ export interface ActorTrace {
|
|
|
366
383
|
*/
|
|
367
384
|
impressions?: ParticipantImpressions;
|
|
368
385
|
items: ActorTraceItem[];
|
|
386
|
+
/**
|
|
387
|
+
* Additive + optional: the page of the computer-use session's last observation, the closing
|
|
388
|
+
* task observation included. Absent when that observation read no URL, so a failed read never
|
|
389
|
+
* repeats an earlier page, and on other producers and older bundles.
|
|
390
|
+
*/
|
|
391
|
+
finalPage?: TracePage;
|
|
369
392
|
tokenUsage?: ActorTokenUsage;
|
|
370
393
|
/** A stalled or adapter-reported ambiguous interaction may have additional unreported usage.
|
|
371
394
|
* Known tokenUsage remains usable as a partial total. Absence is not proof of completeness. */
|
package/dist/actors/contract.js
CHANGED
|
@@ -56,6 +56,9 @@ export const ACTOR_STOP_CAUSES = [
|
|
|
56
56
|
// The participant's scheduled end (participants[].endAfterMs) came. A planned end: the session
|
|
57
57
|
// is goal_satisfied, and the closing report follows.
|
|
58
58
|
"scheduled_end",
|
|
59
|
+
// The participant was still working at its session cap (participants[].maxSessionMs). A planned
|
|
60
|
+
// end, as scheduled_end is: the session is goal_satisfied, and the closing report follows.
|
|
61
|
+
"session_cap",
|
|
59
62
|
];
|
|
60
63
|
/**
|
|
61
64
|
* The oldest Codex CLI release a recorded account profile may name. Readers accept every stable
|
|
@@ -11,6 +11,7 @@ const labels = {
|
|
|
11
11
|
provider_refused_prompt: "provider refused the prompt under its usage policy",
|
|
12
12
|
harness_aborted: "stopped by harness",
|
|
13
13
|
scheduled_end: "scheduled end",
|
|
14
|
+
session_cap: "session cap",
|
|
14
15
|
unspecified_limit: "limit reached",
|
|
15
16
|
};
|
|
16
17
|
/** Shared wording for consumers that already hold a validated, finite stop cause. */
|
|
@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { screenshotEvidenceError } from "../evidence/image.js";
|
|
4
4
|
import { runIdOf } from "../run/paths.js";
|
|
5
5
|
import { isRecord } from "../run/type-guards.js";
|
|
6
|
-
import {
|
|
6
|
+
import { isReadRunBundleSchema, RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
7
7
|
import { loadStreamTraces } from "../run/stream-trace.js";
|
|
8
8
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
9
9
|
import { digestAnalysisInput, hashAnalysisValue, validateAnalysisInputMetadata, } from "./validation.js";
|
|
@@ -30,7 +30,7 @@ function parseSource(prepared, bytes) {
|
|
|
30
30
|
throw new Error("ANALYSIS_SOURCE_TOO_LARGE");
|
|
31
31
|
const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
|
|
32
32
|
if (!isRecord(value) ||
|
|
33
|
-
(value.schema
|
|
33
|
+
!isReadRunBundleSchema(value.schema) ||
|
|
34
34
|
value.runId !== runIdOf(prepared) ||
|
|
35
35
|
!Array.isArray(value.streams) ||
|
|
36
36
|
value.streams.length > EVIDENCE_LIMITS.participants ||
|
|
@@ -2,6 +2,8 @@ import { formatAutomaticAnalysisBudget } from "../../analysis/automatic-config.j
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
5
|
+
import { describeTimeZones } from "../../study/plan-participants.js";
|
|
6
|
+
import { plannedParticipants } from "../../study/warnings.js";
|
|
5
7
|
import { runStudyPreflight, } from "../../study/preflight.js";
|
|
6
8
|
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, parsePositiveInteger, writeResult, humanError, } from "../io.js";
|
|
7
9
|
export function registerStudyCommands(parent, io) {
|
|
@@ -120,6 +122,7 @@ function formatStudyShowHuman(result) {
|
|
|
120
122
|
...(result.path ? [`path: ${result.path}`] : []),
|
|
121
123
|
...(result.origin ? [`origin: ${result.origin}`] : []),
|
|
122
124
|
...(config.subject.repos?.length ? [`repos: ${config.subject.repos.join(", ")}`] : []),
|
|
125
|
+
...timeZoneLines(config),
|
|
123
126
|
...(result.personas ?? []).map((persona) => `persona ${persona.id}: ${persona.brief ? `authored context (before runtime additions)\n${persona.brief.text}` : "unresolved; id only"}`),
|
|
124
127
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
125
128
|
].join("\n") + "\n");
|
|
@@ -145,3 +148,8 @@ function formatStudyCheckHuman(result) {
|
|
|
145
148
|
].join("\n") + "\n";
|
|
146
149
|
return result.error === undefined ? stdout : { stdout, error: result.error };
|
|
147
150
|
}
|
|
151
|
+
/** The `time zones:` line, for a study whose participants declare a time zone. */
|
|
152
|
+
function timeZoneLines(config) {
|
|
153
|
+
const zones = describeTimeZones(plannedParticipants(config));
|
|
154
|
+
return zones === undefined ? [] : [`time zones: ${zones}`];
|
|
155
|
+
}
|
|
@@ -8,6 +8,7 @@ export declare const REDACTION_MARKERS: {
|
|
|
8
8
|
readonly runtimePath: "[REDACTED_RUNTIME_PATH]";
|
|
9
9
|
readonly promptText: "[REDACTED_PROMPT_TEXT]";
|
|
10
10
|
readonly lobbyCode: "[REDACTED_LOBBY_CODE]";
|
|
11
|
+
readonly urlPart: "[REDACTED_URL_PART]";
|
|
11
12
|
};
|
|
12
13
|
/** Every pattern containsSensitive tests. A test pins each to ASCII, which scanEncodedText relies on. */
|
|
13
14
|
export declare function sensitivePatterns(): readonly RegExp[];
|
|
@@ -136,6 +136,7 @@ export const REDACTION_MARKERS = {
|
|
|
136
136
|
runtimePath: "[REDACTED_RUNTIME_PATH]",
|
|
137
137
|
promptText: "[REDACTED_PROMPT_TEXT]",
|
|
138
138
|
lobbyCode: "[REDACTED_LOBBY_CODE]",
|
|
139
|
+
urlPart: "[REDACTED_URL_PART]",
|
|
139
140
|
};
|
|
140
141
|
const LOCAL_PATH_PATTERNS = [
|
|
141
142
|
[/\/private\/var\/folders\/[^\s"'`<>)]*/g, REDACTION_MARKERS.localPath],
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { TracePage } from "../actors/contract.js";
|
|
2
|
+
/**
|
|
3
|
+
* The page of an observed URL, or undefined when there is none or the URL parser rejects it.
|
|
4
|
+
* `redact` is the trace's text redaction (the run's known-value scrub, then redactText).
|
|
5
|
+
*/
|
|
6
|
+
export declare function tracePage(url: string | undefined, redact: (text: string) => string): TracePage | undefined;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { readingsOf, scanEncodedText } from "./encoded-text.js";
|
|
2
|
+
import { REDACTION_MARKERS } from "./redaction.js";
|
|
3
|
+
// The page a computer-use observation was on, as a trace records it: the URL's origin and path,
|
|
4
|
+
// with no query string, fragment or user info, and any path segment that can carry a credential or
|
|
5
|
+
// an address withheld. See docs/contracts/schemas.md#actor-trace.
|
|
6
|
+
const WEB_SCHEMES = new Set(["http:", "https:"]);
|
|
7
|
+
// The longest path a page keeps, its truncation marker included.
|
|
8
|
+
const MAX_PATH = 1024;
|
|
9
|
+
const TRUNCATED = "[TRUNCATED]";
|
|
10
|
+
// A percent-encoded unreserved character (a letter, digit, `-`, `.`, `_` or `~`) or `%` itself.
|
|
11
|
+
// RFC 3986 section 2.3 says producers should not encode the first, and an encoded `%` is an
|
|
12
|
+
// encoding nested in another, so either only hides text from a literal match: the run's
|
|
13
|
+
// sandbox-id scrub, verify's raw-id check, a study secret's scrub.
|
|
14
|
+
const ENCODED_TO_HIDE = /%(?:3[0-9]|4[1-9A-F]|5[0-9AF]|6[1-9A-F]|7[0-9AE]|2[5DE])/i;
|
|
15
|
+
const EMAIL = /[^\s@/]@[^\s@/.]+\.[^\s@/]/;
|
|
16
|
+
// 16 characters is the floor redactText uses for a credential parameter's value. Each open-ended
|
|
17
|
+
// repeat is written `[...]{n}[...]*`, as in redaction.ts, so a megabyte path costs linear time.
|
|
18
|
+
const TOKEN_RUN = /[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g;
|
|
19
|
+
const LETTER = /[A-Za-z]/;
|
|
20
|
+
const DIGIT = /[0-9]/;
|
|
21
|
+
/**
|
|
22
|
+
* A run of 16 or more letters, digits, `-` and `_` in which some piece between separators is 8 or
|
|
23
|
+
* more characters long and mixes letters and digits: UUIDs, hex digests, base64url tokens and
|
|
24
|
+
* sandbox ids. A slug's pieces are words or numbers, so `how-to-build-a-thing-2024` stays.
|
|
25
|
+
*/
|
|
26
|
+
function tokenShaped(text) {
|
|
27
|
+
for (const [run] of text.matchAll(TOKEN_RUN))
|
|
28
|
+
for (const piece of run.split(/[-_]/))
|
|
29
|
+
if (piece.length >= 8 && LETTER.test(piece) && DIGIT.test(piece))
|
|
30
|
+
return true;
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
/** A segment is withheld when any reading of it holds an address, a token or redactable text. */
|
|
34
|
+
function withheld(segment, redact) {
|
|
35
|
+
if (ENCODED_TO_HIDE.test(segment))
|
|
36
|
+
return true;
|
|
37
|
+
return readingsOf(segment).some((reading) => EMAIL.test(reading) || tokenShaped(reading) || redact(reading) !== reading);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Each withheld segment as the marker, then the trace redaction over the whole path for patterns
|
|
41
|
+
* that span segments, then the cut. A path verify's trace scan would still read as sensitive or
|
|
42
|
+
* opaque is withheld whole, so a page never changes a run's grade.
|
|
43
|
+
*/
|
|
44
|
+
function redactPath(pathname, redact) {
|
|
45
|
+
const segments = pathname
|
|
46
|
+
.split("/")
|
|
47
|
+
.map((segment) => (withheld(segment, redact) ? REDACTION_MARKERS.urlPart : segment));
|
|
48
|
+
const redacted = redact(segments.join("/"));
|
|
49
|
+
// Cut after redaction: a cut first could shorten a token below the length the rules match.
|
|
50
|
+
const path = redacted.length > MAX_PATH
|
|
51
|
+
? `${redacted.slice(0, MAX_PATH - TRUNCATED.length)}${TRUNCATED}`
|
|
52
|
+
: redacted;
|
|
53
|
+
const scan = scanEncodedText(path);
|
|
54
|
+
return scan.sensitive || scan.opaque ? REDACTION_MARKERS.urlPart : path;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The origin with each token-shaped host label withheld, since a tunnel or preview host can be as
|
|
58
|
+
* unguessable as a shared link, then through the trace redaction, which withholds a sandbox host
|
|
59
|
+
* whole.
|
|
60
|
+
*/
|
|
61
|
+
function redactOrigin(url, redact) {
|
|
62
|
+
if (url.hostname.startsWith("["))
|
|
63
|
+
return redact(url.origin);
|
|
64
|
+
const host = url.hostname
|
|
65
|
+
.split(".")
|
|
66
|
+
.map((label) => (tokenShaped(label) ? REDACTION_MARKERS.urlPart : label))
|
|
67
|
+
.join(".");
|
|
68
|
+
return redact(`${url.protocol}//${host}${url.port === "" ? "" : `:${url.port}`}`);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The page of an observed URL, or undefined when there is none or the URL parser rejects it.
|
|
72
|
+
* `redact` is the trace's text redaction (the run's known-value scrub, then redactText).
|
|
73
|
+
*/
|
|
74
|
+
export function tracePage(url, redact) {
|
|
75
|
+
if (url === undefined)
|
|
76
|
+
return undefined;
|
|
77
|
+
const parsed = URL.parse(url);
|
|
78
|
+
if (parsed === null)
|
|
79
|
+
return undefined;
|
|
80
|
+
// A non-web URL keeps its scheme only: a file: path or a data: payload never lands.
|
|
81
|
+
if (!WEB_SCHEMES.has(parsed.protocol))
|
|
82
|
+
return { origin: parsed.protocol, path: "" };
|
|
83
|
+
const origin = redactOrigin(parsed, redact);
|
|
84
|
+
const path = redactPath(parsed.pathname, redact);
|
|
85
|
+
// A study secret can span both parts, such as a whole invite link, which neither part holds.
|
|
86
|
+
const joined = `${origin}${path}`;
|
|
87
|
+
if (redact(joined) !== joined)
|
|
88
|
+
return { origin: REDACTION_MARKERS.urlPart, path: REDACTION_MARKERS.urlPart };
|
|
89
|
+
return { origin, path };
|
|
90
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export { parseStudy } from "./study/config.js";
|
|
|
8
8
|
export { STUDY_SCHEMA } from "./study/types.js";
|
|
9
9
|
export type { StudyConfig } from "./study/types.js";
|
|
10
10
|
export { verifyRun } from "./verify/verify.js";
|
|
11
|
+
export { loadRunBundle } from "./run/locate.js";
|
|
11
12
|
export type { VerifyResult } from "./verify/verify.js";
|
|
12
13
|
export { renderObserver } from "./observer/render.js";
|
|
13
14
|
export type { ObserverResult } from "./observer/render.js";
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ export { parseStudy } from "./study/config.js";
|
|
|
8
8
|
export { STUDY_SCHEMA } from "./study/types.js";
|
|
9
9
|
// Read a run.
|
|
10
10
|
export { verifyRun } from "./verify/verify.js";
|
|
11
|
+
export { loadRunBundle } from "./run/locate.js";
|
|
11
12
|
export { renderObserver } from "./observer/render.js";
|
|
12
13
|
// Bring a participant.
|
|
13
14
|
export { runComputerUseLoop, stableProgressKey } from "./actors/computer-use/loop.js";
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { LiveItemsCursor } from "../run/live-items.js";
|
|
2
|
+
import type { LiveItemsRead } from "../run/stream-trace.js";
|
|
3
|
+
import type { ObserverItems } from "./participant-items.js";
|
|
4
|
+
/**
|
|
5
|
+
* The prefixes a poll's `after` parameter says the page holds, by live file path. A parameter with
|
|
6
|
+
* a cursor that is not spelled as the server spells them, more than 10,000 cursors, or two for one
|
|
7
|
+
* file is ignored whole, so the page gets every list in full.
|
|
8
|
+
*/
|
|
9
|
+
export declare function parseFeedCursors(after: string | null): ReadonlyMap<string, LiveItemsCursor>;
|
|
10
|
+
/**
|
|
11
|
+
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
12
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
13
|
+
*/
|
|
14
|
+
export declare function withFeedCursors(items: ObserverItems, live: readonly LiveItemsRead[]): ObserverItems;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The cursors of the Observer's feed. A cursor is `<position>.<generation>.<items>.<bytes>`: the
|
|
2
|
+
// live file live/<position>.<generation>.ndjson (src/run/live-items.ts names them) and the prefix of
|
|
3
|
+
// it a page holds. Each feed response gives a running participant's cursor as `live`; the page sends
|
|
4
|
+
// the cursors back in `after` and compares them only as strings, so a cursor has one spelling:
|
|
5
|
+
// decimal integers without leading zeros.
|
|
6
|
+
/** A live file's stem, `<position>.<generation>`, as isLiveItemsIndex accepts its path. */
|
|
7
|
+
const LIVE_PATH = /^live\/([1-9]\d{0,5}\.[1-9]\d{0,8})\.ndjson$/;
|
|
8
|
+
/** A cursor: the stem, then the line and byte counts. */
|
|
9
|
+
const CURSOR = /^([1-9]\d{0,5}\.[1-9]\d{0,8})\.(0|[1-9]\d{0,15})\.(0|[1-9]\d{0,15})$/;
|
|
10
|
+
/** More cursors than any page sends: a run has one live file per participant. */
|
|
11
|
+
const MAX_CURSORS = 10_000;
|
|
12
|
+
/**
|
|
13
|
+
* The prefixes a poll's `after` parameter says the page holds, by live file path. A parameter with
|
|
14
|
+
* a cursor that is not spelled as the server spells them, more than 10,000 cursors, or two for one
|
|
15
|
+
* file is ignored whole, so the page gets every list in full.
|
|
16
|
+
*/
|
|
17
|
+
export function parseFeedCursors(after) {
|
|
18
|
+
const cursors = new Map();
|
|
19
|
+
if (after === null || after === "")
|
|
20
|
+
return cursors;
|
|
21
|
+
const parts = after.split(",", MAX_CURSORS + 1);
|
|
22
|
+
if (parts.length > MAX_CURSORS)
|
|
23
|
+
return new Map();
|
|
24
|
+
for (const part of parts) {
|
|
25
|
+
const match = CURSOR.exec(part);
|
|
26
|
+
const path = `live/${match?.[1]}.ndjson`;
|
|
27
|
+
const cursor = { items: Number(match?.[2]), bytes: Number(match?.[3]) };
|
|
28
|
+
if (match === null ||
|
|
29
|
+
!Number.isSafeInteger(cursor.items) ||
|
|
30
|
+
!Number.isSafeInteger(cursor.bytes) ||
|
|
31
|
+
cursors.has(path))
|
|
32
|
+
return new Map();
|
|
33
|
+
cursors.set(path, cursor);
|
|
34
|
+
}
|
|
35
|
+
return cursors;
|
|
36
|
+
}
|
|
37
|
+
/** The cursor at `items` lines and `bytes` bytes into the live file at `path`. */
|
|
38
|
+
function cursorText(path, position) {
|
|
39
|
+
const stem = LIVE_PATH.exec(path)?.[1];
|
|
40
|
+
if (stem === undefined)
|
|
41
|
+
throw new Error(`${path} is not a live file name.`);
|
|
42
|
+
return `${stem}.${position.items}.${position.bytes}`;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
46
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
47
|
+
*/
|
|
48
|
+
export function withFeedCursors(items, live) {
|
|
49
|
+
const streams = Object.assign(Object.create(null), items.streams);
|
|
50
|
+
for (const read of live) {
|
|
51
|
+
const entry = Object.hasOwn(streams, read.streamId) ? streams[read.streamId] : undefined;
|
|
52
|
+
if (entry === undefined || !Array.isArray(entry.liveActor))
|
|
53
|
+
continue;
|
|
54
|
+
streams[read.streamId] = {
|
|
55
|
+
...entry,
|
|
56
|
+
live: cursorText(read.index.path, read.index),
|
|
57
|
+
...(read.after === undefined ? {} : { liveAfter: cursorText(read.index.path, read.after) }),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
return { ...items, streams };
|
|
61
|
+
}
|
|
@@ -23,6 +23,13 @@ export interface ObserverItems {
|
|
|
23
23
|
actor?: ActorTraceItem[];
|
|
24
24
|
liveActor?: ActorTraceItem[];
|
|
25
25
|
artifacts?: ObserverStream["artifacts"];
|
|
26
|
+
/**
|
|
27
|
+
* Where `liveActor` ends in the participant's live file, as a cursor the page sends back
|
|
28
|
+
* (src/observer/feed-cursors.ts). Only on a feed response, for a list read from a live file.
|
|
29
|
+
*/
|
|
30
|
+
live?: string;
|
|
31
|
+
/** Set when `liveActor` holds only the items past this cursor, one the page sent. */
|
|
32
|
+
liveAfter?: string;
|
|
26
33
|
}>;
|
|
27
34
|
}
|
|
28
35
|
/** The data as observer-data.json and the page's items carry it. */
|
|
@@ -17,10 +17,13 @@ export interface PinnedDirectory {
|
|
|
17
17
|
/**
|
|
18
18
|
* The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
|
|
19
19
|
* file larger than that is refused from its opened size and the read stops there, so a large file
|
|
20
|
-
* is never held in memory.
|
|
20
|
+
* is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
|
|
21
|
+
* a shorter file; with `start` as well, the bytes from `start` to `length`.
|
|
21
22
|
*/
|
|
22
23
|
export declare function readContainedFile(root: PinnedDirectory, filePathInput: string, options?: {
|
|
23
24
|
maxBytes?: number;
|
|
25
|
+
length?: number;
|
|
26
|
+
start?: number;
|
|
24
27
|
}): Promise<Buffer | null>;
|
|
25
28
|
export declare function openContainedFile(root: PinnedDirectory, filePathInput: string): Promise<{
|
|
26
29
|
handle: FileHandle;
|
|
@@ -5,23 +5,26 @@ import { createHash } from "node:crypto";
|
|
|
5
5
|
import { constants as fsConstants } from "node:fs";
|
|
6
6
|
import { lstat, open, realpath } from "node:fs/promises";
|
|
7
7
|
import path from "node:path";
|
|
8
|
-
import { readOpenedAtMost } from "../run/contained-output.js";
|
|
8
|
+
import { readOpenedAtMost, readOpenedPrefix } from "../run/contained-output.js";
|
|
9
9
|
import { isPathInside, isSafeRunIdSegment } from "../run/paths.js";
|
|
10
10
|
/**
|
|
11
11
|
* The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
|
|
12
12
|
* file larger than that is refused from its opened size and the read stops there, so a large file
|
|
13
|
-
* is never held in memory.
|
|
13
|
+
* is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
|
|
14
|
+
* a shorter file; with `start` as well, the bytes from `start` to `length`.
|
|
14
15
|
*/
|
|
15
16
|
export async function readContainedFile(root, filePathInput, options = {}) {
|
|
16
17
|
const opened = await openContainedFile(root, filePathInput);
|
|
17
18
|
if (!opened)
|
|
18
19
|
return null;
|
|
19
20
|
try {
|
|
20
|
-
const body = options.
|
|
21
|
-
? await opened.handle.
|
|
22
|
-
:
|
|
23
|
-
?
|
|
24
|
-
:
|
|
21
|
+
const body = options.length !== undefined
|
|
22
|
+
? await readOpenedPrefix(opened.handle, options.length, options.start)
|
|
23
|
+
: options.maxBytes === undefined
|
|
24
|
+
? await opened.handle.readFile()
|
|
25
|
+
: opened.size > options.maxBytes
|
|
26
|
+
? null
|
|
27
|
+
: await readOpenedAtMost(opened.handle, options.maxBytes);
|
|
25
28
|
if (body === null)
|
|
26
29
|
return null;
|
|
27
30
|
// A write that landed during the read changed the file's mtime and ctime.
|
|
@@ -17,7 +17,7 @@ export declare function servedRunPaths(runRoot: PinnedDirectory): Promise<Prepar
|
|
|
17
17
|
* internal: consumed by src/observer/serve.ts. `notesToken` goes into the run's Observer page so
|
|
18
18
|
* it can add notes; a server passes it only for a run it takes notes for.
|
|
19
19
|
*/
|
|
20
|
-
export declare function serveRunPath(runRoot: PinnedDirectory, relativePath: string, response: ServerResponse, runtimeStreamUrls?: ObserverRuntimeStreamUrl[], request?: Pick<IncomingMessage, "method" | "headers">, notesToken?: string): Promise<void>;
|
|
20
|
+
export declare function serveRunPath(runRoot: PinnedDirectory, relativePath: string, response: ServerResponse, runtimeStreamUrls?: ObserverRuntimeStreamUrl[], request?: Pick<IncomingMessage, "method" | "headers" | "url">, notesToken?: string): Promise<void>;
|
|
21
21
|
/** internal: exported for the stream lifecycle tests. */
|
|
22
22
|
export declare function withRuntimeStreamUrls(data: ObserverData, runtimeStreamUrls: ObserverRuntimeStreamUrl[]): ObserverData;
|
|
23
23
|
/** internal: consumed by src/observer/serve.ts */
|
|
@@ -10,12 +10,13 @@ import { listRuns } from "../run/stored-runs.js";
|
|
|
10
10
|
import { readAnalysisAccounting } from "../run/costs.js";
|
|
11
11
|
import { bindExistingRunArtifactPaths, isPathInside, isSafeRunIdSegment, } from "../run/paths.js";
|
|
12
12
|
import { isRunOutcome } from "../run/bundle-shape.js";
|
|
13
|
-
import { displayedTraces, loadStreamTraces } from "../run/stream-trace.js";
|
|
13
|
+
import { displayedTraces, loadStreamTraces, } from "../run/stream-trace.js";
|
|
14
14
|
import { isNodeError, isRecord } from "../run/type-guards.js";
|
|
15
15
|
import { MAX_NOTE_FILE_BYTES, readRunNotes, RUN_NOTES_DIR, } from "../run/note-files.js";
|
|
16
16
|
import { analysisCostOf, runCost, runCostLabel } from "../run/run-cost.js";
|
|
17
17
|
import { RUN_STATUS_FILE, RUN_STATUS_STALE_MS, isRunStatusRecord, } from "../run/status.js";
|
|
18
18
|
import { renderObserverHtml } from "./artifact.js";
|
|
19
|
+
import { parseFeedCursors, withFeedCursors } from "./feed-cursors.js";
|
|
19
20
|
import { splitObserverData } from "./participant-items.js";
|
|
20
21
|
import { buildObserverData, observerDataFromSaved, recordedStreamEmbed, withObserverEndings, } from "./data.js";
|
|
21
22
|
import { buildArtifactSecurityHeaders } from "./http.js";
|
|
@@ -103,14 +104,14 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
103
104
|
return;
|
|
104
105
|
}
|
|
105
106
|
if (cleanedRelativePath === "observer/index.html") {
|
|
106
|
-
const
|
|
107
|
-
if ("unavailable" in
|
|
108
|
-
writeResponse(response, 404, `Observer data not found: ${
|
|
107
|
+
const observed = await observerDataOrReason(runRoot, runtimeStreamUrls);
|
|
108
|
+
if ("unavailable" in observed) {
|
|
109
|
+
writeResponse(response, 404, `Observer data not found: ${observed.unavailable}`, "text/plain; charset=utf-8");
|
|
109
110
|
return;
|
|
110
111
|
}
|
|
111
112
|
const analysis = await readObserverAnalysis(runRoot);
|
|
112
113
|
const notes = await readRunNotes(servedNoteFiles(runRoot));
|
|
113
|
-
writeResponse(response, 200, renderObserverHtml(
|
|
114
|
+
writeResponse(response, 200, renderObserverHtml(observed.data, {
|
|
114
115
|
analysis,
|
|
115
116
|
notes,
|
|
116
117
|
...(notesToken === undefined ? {} : { notesToken }),
|
|
@@ -121,15 +122,17 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
121
122
|
// polls observer-feed.json, the same data with the items, so one projection serves each poll.
|
|
122
123
|
if (cleanedRelativePath === "observer/observer-data.json" ||
|
|
123
124
|
cleanedRelativePath === "observer/observer-feed.json") {
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
const observed = await observerDataOrReason(runRoot, runtimeStreamUrls, cleanedRelativePath === "observer/observer-feed.json"
|
|
126
|
+
? requestedCursors(runRoot, request)
|
|
127
|
+
: undefined);
|
|
128
|
+
if ("unavailable" in observed) {
|
|
129
|
+
writeResponse(response, 404, `Observer data not found: ${observed.unavailable}`, "text/plain; charset=utf-8");
|
|
127
130
|
return;
|
|
128
131
|
}
|
|
129
|
-
const { shell, items } = splitObserverData(
|
|
132
|
+
const { shell, items } = splitObserverData(observed.data);
|
|
130
133
|
writeResponse(response, 200, cleanedRelativePath === "observer/observer-data.json"
|
|
131
134
|
? JSON.stringify(shell, null, 2)
|
|
132
|
-
: JSON.stringify({ data: shell, items }), "application/json; charset=utf-8");
|
|
135
|
+
: JSON.stringify({ data: shell, items: withFeedCursors(items, observed.live) }), "application/json; charset=utf-8");
|
|
133
136
|
return;
|
|
134
137
|
}
|
|
135
138
|
if (cleanedRelativePath === "observer/study-analysis.json") {
|
|
@@ -163,18 +166,31 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
163
166
|
writeResponse(response, 404, "Not found", "text/plain; charset=utf-8");
|
|
164
167
|
}
|
|
165
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* The live cursors a feed poll sends in `after`. A root that checks reads against admitted files
|
|
171
|
+
* and digests gets none: its digest check covers whole files, and it serves only finished runs.
|
|
172
|
+
*/
|
|
173
|
+
function requestedCursors(runRoot, request) {
|
|
174
|
+
if (runRoot.admitsFile !== undefined ||
|
|
175
|
+
runRoot.admitsContent !== undefined ||
|
|
176
|
+
request?.url === undefined)
|
|
177
|
+
return undefined;
|
|
178
|
+
return parseFeedCursors(new URL(request.url, "http://observer.invalid").searchParams.get("after"));
|
|
179
|
+
}
|
|
166
180
|
/** The run's trace files, read through its pinned folder. */
|
|
167
181
|
function pinnedTraceReader(runRoot) {
|
|
168
|
-
return async (relativePath, maxBytes) => {
|
|
182
|
+
return async (relativePath, maxBytes, length, start) => {
|
|
169
183
|
const bytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, relativePath), {
|
|
170
184
|
maxBytes,
|
|
185
|
+
...(length === undefined ? {} : { length }),
|
|
186
|
+
...(start === undefined ? {} : { start }),
|
|
171
187
|
}).catch(() => null);
|
|
172
188
|
return bytes === null ? { refused: `${relativePath} could not be read` } : { bytes };
|
|
173
189
|
};
|
|
174
190
|
}
|
|
175
191
|
async function readObserverData(runRoot, runtimeStreamUrls = []) {
|
|
176
192
|
const read = await observerDataOrReason(runRoot, runtimeStreamUrls);
|
|
177
|
-
return "unavailable" in read ? null : read;
|
|
193
|
+
return "unavailable" in read ? null : read.data;
|
|
178
194
|
}
|
|
179
195
|
/**
|
|
180
196
|
* The run's Observer data, or why there is none. run.json is the source: each trace is loaded
|
|
@@ -184,7 +200,7 @@ async function readObserverData(runRoot, runtimeStreamUrls = []) {
|
|
|
184
200
|
* cannot be read, parsed or used as a bundle (which a browser poll can meet mid-write) gives the
|
|
185
201
|
* reason, a 404 the poller retries, never the saved projection.
|
|
186
202
|
*/
|
|
187
|
-
async function observerDataOrReason(runRoot, runtimeStreamUrls) {
|
|
203
|
+
async function observerDataOrReason(runRoot, runtimeStreamUrls, cursors) {
|
|
188
204
|
const record = await readLocalRunStatus(runRoot);
|
|
189
205
|
const runJson = path.join(runRoot.physicalPath, "run.json");
|
|
190
206
|
const bundleBytes = await readContainedFile(runRoot, runJson).catch(() => null);
|
|
@@ -193,8 +209,10 @@ async function observerDataOrReason(runRoot, runtimeStreamUrls) {
|
|
|
193
209
|
// A root admitted for safe serving checks every read against the identity and digest admitted
|
|
194
210
|
// with the run's share_ready verification, so a run.json changed since then is refused, and the
|
|
195
211
|
// saved projection it may serve is the one admitted with the verified run.
|
|
196
|
-
if (absent || (bundleBytes === null && runRoot.admitsContent !== undefined))
|
|
197
|
-
|
|
212
|
+
if (absent || (bundleBytes === null && runRoot.admitsContent !== undefined)) {
|
|
213
|
+
const saved = await savedObserverData(runRoot, record, runtimeStreamUrls);
|
|
214
|
+
return "unavailable" in saved ? saved : { data: saved, live: [] };
|
|
215
|
+
}
|
|
198
216
|
if (bundleBytes === null)
|
|
199
217
|
return { unavailable: "run.json is there but could not be read: it is a link or changed." };
|
|
200
218
|
let stored;
|
|
@@ -208,8 +226,12 @@ async function observerDataOrReason(runRoot, runtimeStreamUrls) {
|
|
|
208
226
|
return { unavailable: "run.json is not a run bundle; it may be mid-write, so try again." };
|
|
209
227
|
}
|
|
210
228
|
try {
|
|
211
|
-
const
|
|
212
|
-
|
|
229
|
+
const loaded = await loadStreamTraces(pinnedTraceReader(runRoot), stored, cursors === undefined ? {} : { liveAfter: (index) => cursors.get(index.path) });
|
|
230
|
+
const bundle = displayedTraces(loaded);
|
|
231
|
+
return {
|
|
232
|
+
data: withRuntimeStreamUrls(withLocalRunStatus(record, buildObserverData(bundle, undefined, record), bundle.outcome), runtimeStreamUrls),
|
|
233
|
+
live: loaded.live,
|
|
234
|
+
};
|
|
213
235
|
}
|
|
214
236
|
catch {
|
|
215
237
|
return { unavailable: "run.json could not be shown as a run." };
|