humanish 0.125.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/actors/computer-use/actor.d.ts +2 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/clock.d.ts +9 -8
- package/dist/actors/computer-use/loop/clock.js +27 -17
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -1
- package/dist/actors/computer-use/loop/debrief.js +15 -5
- package/dist/actors/computer-use/loop/ending.d.ts +10 -5
- package/dist/actors/computer-use/loop/ending.js +12 -7
- package/dist/actors/computer-use/loop/observation.d.ts +6 -0
- package/dist/actors/computer-use/loop/observation.js +19 -1
- package/dist/actors/computer-use/loop/session.d.ts +4 -2
- package/dist/actors/computer-use/loop/session.js +4 -1
- package/dist/actors/computer-use/loop/trace.js +4 -2
- package/dist/actors/computer-use/loop/types.d.ts +22 -8
- package/dist/actors/computer-use/loop.js +5 -5
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/contract.d.ts +31 -8
- package/dist/actors/contract.js +3 -0
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/cli/commands/study.js +8 -0
- package/dist/evidence/redaction.d.ts +1 -0
- package/dist/evidence/redaction.js +1 -0
- package/dist/evidence/trace-page.d.ts +6 -0
- package/dist/evidence/trace-page.js +90 -0
- package/dist/observer/data.js +8 -5
- package/dist/observer/feed-cursors.d.ts +14 -0
- package/dist/observer/feed-cursors.js +61 -0
- package/dist/observer/participant-items.d.ts +7 -0
- package/dist/observer/pinned-files.d.ts +2 -1
- package/dist/observer/pinned-files.js +2 -2
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +38 -17
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle.js +4 -0
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/desktop.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +6 -0
- package/dist/routes/computer-use/e2b-desktop/prepare.js +47 -1
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +2 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +15 -3
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -1
- package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +5 -1
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-desktop.d.ts +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-model.js +16 -6
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +54 -84
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +65 -17
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.d.ts +1 -0
- package/dist/routes/computer-use/setup.js +29 -1
- package/dist/routes/computer-use/single-bundle.d.ts +5 -1
- package/dist/routes/computer-use/single-bundle.js +11 -1
- package/dist/routes/computer-use/types.d.ts +18 -13
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +4 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/participant-specs.js +3 -2
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +8 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/contained-output.d.ts +7 -3
- package/dist/run/contained-output.js +11 -7
- package/dist/run/judge.d.ts +14 -4
- package/dist/run/judge.js +17 -12
- package/dist/run/live-items.d.ts +19 -0
- package/dist/run/live-items.js +28 -8
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/run/stream-trace.d.ts +25 -4
- package/dist/run/stream-trace.js +55 -17
- package/dist/run/streams.d.ts +16 -0
- package/dist/study/arrivals.d.ts +105 -34
- package/dist/study/arrivals.js +179 -74
- package/dist/study/bundle-estimate.d.ts +12 -10
- package/dist/study/bundle-estimate.js +138 -72
- package/dist/study/desktop-files.d.ts +37 -0
- package/dist/study/desktop-files.js +174 -0
- package/dist/study/keys.d.ts +8 -0
- package/dist/study/keys.js +3 -0
- package/dist/study/migrate/v2.js +12 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +28 -20
- package/dist/study/parse/clock.d.ts +18 -0
- package/dist/study/parse/clock.js +36 -0
- package/dist/study/plan-participants.d.ts +19 -10
- package/dist/study/plan-participants.js +13 -9
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +28 -15
- package/dist/study/preflight.js +4 -0
- package/dist/study/types.d.ts +35 -5
- package/dist/study/validation.js +3 -0
- package/dist/study/warnings.d.ts +2 -2
- package/dist/study/warnings.js +7 -0
- package/dist/subject/local-tree-archive.d.ts +7 -0
- package/dist/subject/local-tree-archive.js +12 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -1
- package/dist/substrates/e2b/desktop-browser.js +6 -2
- package/dist/substrates/e2b/desktop-clock.d.ts +10 -0
- package/dist/substrates/e2b/desktop-clock.js +29 -0
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/dist/substrates/e2b/sdk.d.ts +6 -0
- package/dist/verify/pages.d.ts +7 -0
- package/dist/verify/pages.js +50 -0
- package/dist/verify/verify.js +10 -3
- package/docs/architecture/actor-contract.md +2 -0
- package/docs/architecture/external-public-shared-world.md +4 -1
- package/docs/architecture/observer.md +10 -1
- package/docs/contracts/policy.md +7 -0
- package/docs/contracts/run-bundle.md +31 -1
- package/docs/contracts/schemas.md +93 -11
- package/package.json +2 -2
|
@@ -37,12 +37,21 @@ export type ParticipantDeclaredOutcome = "reached" | "not_reached" | "blocked";
|
|
|
37
37
|
export declare const PARTICIPANT_OUTCOME_STATUSES: readonly ActorStatus[];
|
|
38
38
|
/** Optional precise interruption cause; completionReason and status retain their original meaning.
|
|
39
39
|
* One list, so the analysis schema and the diagnostics projection cannot drift from the trace. */
|
|
40
|
-
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end"];
|
|
40
|
+
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "session_cap"];
|
|
41
41
|
export type ActorStopCause = (typeof ACTOR_STOP_CAUSES)[number];
|
|
42
42
|
export type ActorCompletionReason = "goal_satisfied" | "turn_completed" | "gave_up" | "blocked_approval" | "timed_out" | "budget_reached" | "actor_error" | "step_failed" | "harness_error";
|
|
43
43
|
type ActorRunKind = "code" | "computer-use" | "scripted-browser" | "terminal";
|
|
44
44
|
type ActorProtocol = "json-rpc" | "json-stream" | "cua-loop" | "scripted-steps" | "terminal-exec";
|
|
45
45
|
export type ActorTraceItemKind = "message" | "reasoning" | "tool_call" | "command" | "file_change" | "approval" | "screenshot" | "ui_action" | "plan" | "notice";
|
|
46
|
+
/**
|
|
47
|
+
* Where a participant's browser was at one observation: the URL's origin and its path, redacted
|
|
48
|
+
* (src/evidence/trace-page.ts). No query string, fragment or user info. A non-web URL keeps only
|
|
49
|
+
* its scheme, as `{ origin: "about:", path: "" }`.
|
|
50
|
+
*/
|
|
51
|
+
export interface TracePage {
|
|
52
|
+
origin: string;
|
|
53
|
+
path: string;
|
|
54
|
+
}
|
|
46
55
|
export interface ActorTraceItem {
|
|
47
56
|
id: string;
|
|
48
57
|
kind: ActorTraceItemKind;
|
|
@@ -74,6 +83,9 @@ export interface ActorTraceItem {
|
|
|
74
83
|
x: number;
|
|
75
84
|
y: number;
|
|
76
85
|
};
|
|
86
|
+
/** On a computer-use `screenshot` item: the page that observation was on. Additive: absent when
|
|
87
|
+
* the observation read no URL, on other producers and in older bundles. */
|
|
88
|
+
page?: TracePage;
|
|
77
89
|
}
|
|
78
90
|
export interface ActorCapabilities {
|
|
79
91
|
headless: boolean;
|
|
@@ -338,19 +350,24 @@ export interface ActorTrace {
|
|
|
338
350
|
*/
|
|
339
351
|
taskFunnel?: TaskFunnel;
|
|
340
352
|
/**
|
|
341
|
-
* Additive + optional: the outcome the participant declared
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
353
|
+
* Additive + optional: the outcome the participant declared, in its provider's outcome field or
|
|
354
|
+
* in the fixed first line its prompt asks for. It comes from its last message when it ended the
|
|
355
|
+
* session itself, and from its accepted closing report at a scheduled end or a session cap.
|
|
356
|
+
* Absent when it declared none, after a stop condition or a dwell window, and on every older
|
|
357
|
+
* bundle. The route reads a declared `reached` or `blocked` before it reads the closing
|
|
358
|
+
* paragraph; three regex patches in one month each fixed a false refusal and each left the next
|
|
359
|
+
* shape unhandled. A declared `not_reached` leaves the paragraph to say whether a blocker
|
|
360
|
+
* stopped the participant.
|
|
345
361
|
*/
|
|
346
362
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
347
363
|
/**
|
|
348
364
|
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
349
|
-
* `dwell`, `scheduled_end`), or impressions only after the participant ended the
|
|
350
|
-
* (`participant_end`, never with `report`). Does not change task outcomes or
|
|
365
|
+
* `dwell`, `scheduled_end`, `session_cap`), or impressions only after the participant ended the
|
|
366
|
+
* session itself (`participant_end`, never with `report`). Does not change task outcomes or
|
|
367
|
+
* permit actions.
|
|
351
368
|
*/
|
|
352
369
|
debrief?: {
|
|
353
|
-
trigger: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
370
|
+
trigger: "stop_when" | "dwell" | "scheduled_end" | "session_cap" | "participant_end";
|
|
354
371
|
status: "completed" | "skipped" | "failed";
|
|
355
372
|
reason: string;
|
|
356
373
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
|
@@ -366,6 +383,12 @@ export interface ActorTrace {
|
|
|
366
383
|
*/
|
|
367
384
|
impressions?: ParticipantImpressions;
|
|
368
385
|
items: ActorTraceItem[];
|
|
386
|
+
/**
|
|
387
|
+
* Additive + optional: the page of the computer-use session's last observation, the closing
|
|
388
|
+
* task observation included. Absent when that observation read no URL, so a failed read never
|
|
389
|
+
* repeats an earlier page, and on other producers and older bundles.
|
|
390
|
+
*/
|
|
391
|
+
finalPage?: TracePage;
|
|
369
392
|
tokenUsage?: ActorTokenUsage;
|
|
370
393
|
/** A stalled or adapter-reported ambiguous interaction may have additional unreported usage.
|
|
371
394
|
* Known tokenUsage remains usable as a partial total. Absence is not proof of completeness. */
|
package/dist/actors/contract.js
CHANGED
|
@@ -56,6 +56,9 @@ export const ACTOR_STOP_CAUSES = [
|
|
|
56
56
|
// The participant's scheduled end (participants[].endAfterMs) came. A planned end: the session
|
|
57
57
|
// is goal_satisfied, and the closing report follows.
|
|
58
58
|
"scheduled_end",
|
|
59
|
+
// The participant was still working at its session cap (participants[].maxSessionMs). A planned
|
|
60
|
+
// end, as scheduled_end is: the session is goal_satisfied, and the closing report follows.
|
|
61
|
+
"session_cap",
|
|
59
62
|
];
|
|
60
63
|
/**
|
|
61
64
|
* The oldest Codex CLI release a recorded account profile may name. Readers accept every stable
|
|
@@ -1,5 +1,35 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export declare const CUA_ENDPOINT_SOURCES: readonly ["participant_report", "condition_matched", "unavailable"];
|
|
6
|
+
export type CuaEndpointSource = (typeof CUA_ENDPOINT_SOURCES)[number];
|
|
7
|
+
/**
|
|
8
|
+
* Presentation provenance for computer-use completion; never reclassifies the stored outcome.
|
|
9
|
+
* `scheduled_end` and `session_cap` name a session that passed at that clock end without the
|
|
10
|
+
* participant declaring it reached the goal (ranUntilClockEnd).
|
|
11
|
+
*/
|
|
12
|
+
export type CuaGoalSource = CuaEndpointSource | ClockEnd;
|
|
13
|
+
/** The two clock ends a computer-use session passes at: the stop causes of the same names. */
|
|
14
|
+
export type ClockEnd = "scheduled_end" | "session_cap";
|
|
15
|
+
export declare function isClockEnd(source: CuaGoalSource | undefined): source is ClockEnd;
|
|
16
|
+
/**
|
|
17
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
18
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
19
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
20
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
21
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
22
|
+
*/
|
|
23
|
+
export declare function ranUntilClockEnd(trace: {
|
|
24
|
+
readonly stopCause?: unknown;
|
|
25
|
+
readonly declaredOutcome?: unknown;
|
|
26
|
+
}): ClockEnd | undefined;
|
|
3
27
|
export declare function isCuaTrace(value: unknown): boolean;
|
|
4
28
|
export declare function cuaGoalSource(value: unknown, recordedStatus?: unknown): CuaGoalSource | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
31
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
32
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
33
|
+
*/
|
|
34
|
+
export declare function cuaEndpointSource(value: unknown, recordedStatus?: unknown): CuaEndpointSource | undefined;
|
|
5
35
|
export declare const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -1,3 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The endpoint a computer-use completion came from, as analysis provenance records it
|
|
3
|
+
* (cuaEndpointSource). Saved analyses hash it, so a new value would make them invalid on read.
|
|
4
|
+
*/
|
|
5
|
+
export const CUA_ENDPOINT_SOURCES = [
|
|
6
|
+
"participant_report",
|
|
7
|
+
"condition_matched",
|
|
8
|
+
"unavailable",
|
|
9
|
+
];
|
|
10
|
+
export function isClockEnd(source) {
|
|
11
|
+
return source === "scheduled_end" || source === "session_cap";
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The clock end a session ran until, when the participant did not declare that it reached the
|
|
15
|
+
* goal: its scheduled end or its session cap. Such a session ends as passed, because ending at
|
|
16
|
+
* the clock is the plan, and its closing report says `DID NOT REACH THE GOAL.` or declares
|
|
17
|
+
* nothing. The tally and every completion label read this, so none counts it as reaching the
|
|
18
|
+
* goal. Undefined for every other ending, including a declared `reached` at a clock end.
|
|
19
|
+
*/
|
|
20
|
+
export function ranUntilClockEnd(trace) {
|
|
21
|
+
if (trace.stopCause !== "scheduled_end" && trace.stopCause !== "session_cap")
|
|
22
|
+
return undefined;
|
|
23
|
+
return trace.declaredOutcome === "reached" ? undefined : trace.stopCause;
|
|
24
|
+
}
|
|
1
25
|
export function isCuaTrace(value) {
|
|
2
26
|
if (typeof value !== "object" || value === null)
|
|
3
27
|
return false;
|
|
@@ -32,7 +56,22 @@ export function cuaGoalSource(value, recordedStatus) {
|
|
|
32
56
|
(item.title.startsWith("stopWhen matched:") || item.title === "dwell window complete"))) {
|
|
33
57
|
return "condition_matched";
|
|
34
58
|
}
|
|
35
|
-
|
|
36
|
-
|
|
59
|
+
return ranUntilClockEnd(trace) ?? endpointOfItems(trace.items);
|
|
60
|
+
}
|
|
61
|
+
// A legacy trace without enough detail cannot establish which endpoint ended the loop.
|
|
62
|
+
function endpointOfItems(items) {
|
|
63
|
+
return items.length > 0 ? "participant_report" : "unavailable";
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* cuaGoalSource for analysis provenance. A clock end reads as it did before it had a source of
|
|
67
|
+
* its own, so an analysis saved then still matches its run when re-read. The provenance's
|
|
68
|
+
* `stopCause` and `declaredOutcome` name the clock end.
|
|
69
|
+
*/
|
|
70
|
+
export function cuaEndpointSource(value, recordedStatus) {
|
|
71
|
+
const source = cuaGoalSource(value, recordedStatus);
|
|
72
|
+
// cuaGoalSource returns a clock end only after it has checked that `items` is an array.
|
|
73
|
+
return isClockEnd(source)
|
|
74
|
+
? endpointOfItems(value.items)
|
|
75
|
+
: source;
|
|
37
76
|
}
|
|
38
77
|
export const CUA_COMPLETION_NOTE = "Participant reports alone do not establish task success. A matched stop condition establishes only its declared condition. Run gate and share-safety results are separate.";
|
|
@@ -11,6 +11,7 @@ const labels = {
|
|
|
11
11
|
provider_refused_prompt: "provider refused the prompt under its usage policy",
|
|
12
12
|
harness_aborted: "stopped by harness",
|
|
13
13
|
scheduled_end: "scheduled end",
|
|
14
|
+
session_cap: "session cap",
|
|
14
15
|
unspecified_limit: "limit reached",
|
|
15
16
|
};
|
|
16
17
|
/** Shared wording for consumers that already hold a validated, finite stop cause. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { receivingAnalysisContext } from "../comms/receiving-evidence.js";
|
|
2
2
|
import { eventRecordIdOf, recordIdOf, streamParticipantIdOf } from "../run/participant-records.js";
|
|
3
|
-
import {
|
|
3
|
+
import { cuaEndpointSource } from "../actors/goal-source.js";
|
|
4
4
|
import { isRecord } from "../run/type-guards.js";
|
|
5
5
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
6
6
|
import { decodesToPlainRelativePath, isEvidencePath } from "../run/evidence-files.js";
|
|
@@ -82,7 +82,7 @@ export function participantSource(stream, captureVersion) {
|
|
|
82
82
|
actorStatus: actor?.status ?? null,
|
|
83
83
|
completionReason: actor?.completionReason ?? null,
|
|
84
84
|
stopCause: actor?.stopCause ?? null,
|
|
85
|
-
goalSource:
|
|
85
|
+
goalSource: cuaEndpointSource(actor, stream.status) ?? null,
|
|
86
86
|
declaredOutcome: actor?.declaredOutcome ?? null,
|
|
87
87
|
taskOutcomes: actor?.taskFunnel === undefined
|
|
88
88
|
? null
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { CuaEndpointSource } from "../actors/goal-source.js";
|
|
3
3
|
import type { AutomaticAnalysisView } from "./job.js";
|
|
4
4
|
import type { RunAnalysisCost } from "../run/run-cost.js";
|
|
5
5
|
/** Independent interpretation of retained evidence; never a participant or harness verdict. */
|
|
@@ -55,7 +55,7 @@ export interface AnalysisParticipantInput {
|
|
|
55
55
|
actorStatus: ActorStatus | null;
|
|
56
56
|
completionReason: ActorCompletionReason | null;
|
|
57
57
|
stopCause: ActorStopCause | null;
|
|
58
|
-
goalSource:
|
|
58
|
+
goalSource: CuaEndpointSource | null;
|
|
59
59
|
declaredOutcome: ParticipantDeclaredOutcome | null;
|
|
60
60
|
taskOutcomes: Array<{
|
|
61
61
|
taskId: string;
|
|
@@ -2,6 +2,7 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
+
import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
|
|
5
6
|
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
6
7
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
7
8
|
const text = (max) => z
|
|
@@ -178,7 +179,7 @@ const analysisParticipantProvenanceSchema = z
|
|
|
178
179
|
])
|
|
179
180
|
.nullable(),
|
|
180
181
|
stopCause: z.enum(ACTOR_STOP_CAUSES).nullable(),
|
|
181
|
-
goalSource: z.enum(
|
|
182
|
+
goalSource: z.enum(CUA_ENDPOINT_SOURCES).nullable(),
|
|
182
183
|
declaredOutcome: z.enum(["reached", "not_reached", "blocked"]).nullable(),
|
|
183
184
|
taskOutcomes: z
|
|
184
185
|
.array(z
|
|
@@ -2,6 +2,8 @@ import { formatAutomaticAnalysisBudget } from "../../analysis/automatic-config.j
|
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
5
|
+
import { describeTimeZones } from "../../study/plan-participants.js";
|
|
6
|
+
import { plannedParticipants } from "../../study/warnings.js";
|
|
5
7
|
import { runStudyPreflight, } from "../../study/preflight.js";
|
|
6
8
|
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, parsePositiveInteger, writeResult, humanError, } from "../io.js";
|
|
7
9
|
export function registerStudyCommands(parent, io) {
|
|
@@ -120,6 +122,7 @@ function formatStudyShowHuman(result) {
|
|
|
120
122
|
...(result.path ? [`path: ${result.path}`] : []),
|
|
121
123
|
...(result.origin ? [`origin: ${result.origin}`] : []),
|
|
122
124
|
...(config.subject.repos?.length ? [`repos: ${config.subject.repos.join(", ")}`] : []),
|
|
125
|
+
...timeZoneLines(config),
|
|
123
126
|
...(result.personas ?? []).map((persona) => `persona ${persona.id}: ${persona.brief ? `authored context (before runtime additions)\n${persona.brief.text}` : "unresolved; id only"}`),
|
|
124
127
|
...result.warnings.map((warning) => `warning: ${warning}`),
|
|
125
128
|
].join("\n") + "\n");
|
|
@@ -145,3 +148,8 @@ function formatStudyCheckHuman(result) {
|
|
|
145
148
|
].join("\n") + "\n";
|
|
146
149
|
return result.error === undefined ? stdout : { stdout, error: result.error };
|
|
147
150
|
}
|
|
151
|
+
/** The `time zones:` line, for a study whose participants declare a time zone. */
|
|
152
|
+
function timeZoneLines(config) {
|
|
153
|
+
const zones = describeTimeZones(plannedParticipants(config));
|
|
154
|
+
return zones === undefined ? [] : [`time zones: ${zones}`];
|
|
155
|
+
}
|
|
@@ -8,6 +8,7 @@ export declare const REDACTION_MARKERS: {
|
|
|
8
8
|
readonly runtimePath: "[REDACTED_RUNTIME_PATH]";
|
|
9
9
|
readonly promptText: "[REDACTED_PROMPT_TEXT]";
|
|
10
10
|
readonly lobbyCode: "[REDACTED_LOBBY_CODE]";
|
|
11
|
+
readonly urlPart: "[REDACTED_URL_PART]";
|
|
11
12
|
};
|
|
12
13
|
/** Every pattern containsSensitive tests. A test pins each to ASCII, which scanEncodedText relies on. */
|
|
13
14
|
export declare function sensitivePatterns(): readonly RegExp[];
|
|
@@ -136,6 +136,7 @@ export const REDACTION_MARKERS = {
|
|
|
136
136
|
runtimePath: "[REDACTED_RUNTIME_PATH]",
|
|
137
137
|
promptText: "[REDACTED_PROMPT_TEXT]",
|
|
138
138
|
lobbyCode: "[REDACTED_LOBBY_CODE]",
|
|
139
|
+
urlPart: "[REDACTED_URL_PART]",
|
|
139
140
|
};
|
|
140
141
|
const LOCAL_PATH_PATTERNS = [
|
|
141
142
|
[/\/private\/var\/folders\/[^\s"'`<>)]*/g, REDACTION_MARKERS.localPath],
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { TracePage } from "../actors/contract.js";
|
|
2
|
+
/**
|
|
3
|
+
* The page of an observed URL, or undefined when there is none or the URL parser rejects it.
|
|
4
|
+
* `redact` is the trace's text redaction (the run's known-value scrub, then redactText).
|
|
5
|
+
*/
|
|
6
|
+
export declare function tracePage(url: string | undefined, redact: (text: string) => string): TracePage | undefined;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { readingsOf, scanEncodedText } from "./encoded-text.js";
|
|
2
|
+
import { REDACTION_MARKERS } from "./redaction.js";
|
|
3
|
+
// The page a computer-use observation was on, as a trace records it: the URL's origin and path,
|
|
4
|
+
// with no query string, fragment or user info, and any path segment that can carry a credential or
|
|
5
|
+
// an address withheld. See docs/contracts/schemas.md#actor-trace.
|
|
6
|
+
const WEB_SCHEMES = new Set(["http:", "https:"]);
|
|
7
|
+
// The longest path a page keeps, its truncation marker included.
|
|
8
|
+
const MAX_PATH = 1024;
|
|
9
|
+
const TRUNCATED = "[TRUNCATED]";
|
|
10
|
+
// A percent-encoded unreserved character (a letter, digit, `-`, `.`, `_` or `~`) or `%` itself.
|
|
11
|
+
// RFC 3986 section 2.3 says producers should not encode the first, and an encoded `%` is an
|
|
12
|
+
// encoding nested in another, so either only hides text from a literal match: the run's
|
|
13
|
+
// sandbox-id scrub, verify's raw-id check, a study secret's scrub.
|
|
14
|
+
const ENCODED_TO_HIDE = /%(?:3[0-9]|4[1-9A-F]|5[0-9AF]|6[1-9A-F]|7[0-9AE]|2[5DE])/i;
|
|
15
|
+
const EMAIL = /[^\s@/]@[^\s@/.]+\.[^\s@/]/;
|
|
16
|
+
// 16 characters is the floor redactText uses for a credential parameter's value. Each open-ended
|
|
17
|
+
// repeat is written `[...]{n}[...]*`, as in redaction.ts, so a megabyte path costs linear time.
|
|
18
|
+
const TOKEN_RUN = /[A-Za-z0-9_-]{16}[A-Za-z0-9_-]*/g;
|
|
19
|
+
const LETTER = /[A-Za-z]/;
|
|
20
|
+
const DIGIT = /[0-9]/;
|
|
21
|
+
/**
|
|
22
|
+
* A run of 16 or more letters, digits, `-` and `_` in which some piece between separators is 8 or
|
|
23
|
+
* more characters long and mixes letters and digits: UUIDs, hex digests, base64url tokens and
|
|
24
|
+
* sandbox ids. A slug's pieces are words or numbers, so `how-to-build-a-thing-2024` stays.
|
|
25
|
+
*/
|
|
26
|
+
function tokenShaped(text) {
|
|
27
|
+
for (const [run] of text.matchAll(TOKEN_RUN))
|
|
28
|
+
for (const piece of run.split(/[-_]/))
|
|
29
|
+
if (piece.length >= 8 && LETTER.test(piece) && DIGIT.test(piece))
|
|
30
|
+
return true;
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
/** A segment is withheld when any reading of it holds an address, a token or redactable text. */
|
|
34
|
+
function withheld(segment, redact) {
|
|
35
|
+
if (ENCODED_TO_HIDE.test(segment))
|
|
36
|
+
return true;
|
|
37
|
+
return readingsOf(segment).some((reading) => EMAIL.test(reading) || tokenShaped(reading) || redact(reading) !== reading);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Each withheld segment as the marker, then the trace redaction over the whole path for patterns
|
|
41
|
+
* that span segments, then the cut. A path verify's trace scan would still read as sensitive or
|
|
42
|
+
* opaque is withheld whole, so a page never changes a run's grade.
|
|
43
|
+
*/
|
|
44
|
+
function redactPath(pathname, redact) {
|
|
45
|
+
const segments = pathname
|
|
46
|
+
.split("/")
|
|
47
|
+
.map((segment) => (withheld(segment, redact) ? REDACTION_MARKERS.urlPart : segment));
|
|
48
|
+
const redacted = redact(segments.join("/"));
|
|
49
|
+
// Cut after redaction: a cut first could shorten a token below the length the rules match.
|
|
50
|
+
const path = redacted.length > MAX_PATH
|
|
51
|
+
? `${redacted.slice(0, MAX_PATH - TRUNCATED.length)}${TRUNCATED}`
|
|
52
|
+
: redacted;
|
|
53
|
+
const scan = scanEncodedText(path);
|
|
54
|
+
return scan.sensitive || scan.opaque ? REDACTION_MARKERS.urlPart : path;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The origin with each token-shaped host label withheld, since a tunnel or preview host can be as
|
|
58
|
+
* unguessable as a shared link, then through the trace redaction, which withholds a sandbox host
|
|
59
|
+
* whole.
|
|
60
|
+
*/
|
|
61
|
+
function redactOrigin(url, redact) {
|
|
62
|
+
if (url.hostname.startsWith("["))
|
|
63
|
+
return redact(url.origin);
|
|
64
|
+
const host = url.hostname
|
|
65
|
+
.split(".")
|
|
66
|
+
.map((label) => (tokenShaped(label) ? REDACTION_MARKERS.urlPart : label))
|
|
67
|
+
.join(".");
|
|
68
|
+
return redact(`${url.protocol}//${host}${url.port === "" ? "" : `:${url.port}`}`);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* The page of an observed URL, or undefined when there is none or the URL parser rejects it.
|
|
72
|
+
* `redact` is the trace's text redaction (the run's known-value scrub, then redactText).
|
|
73
|
+
*/
|
|
74
|
+
export function tracePage(url, redact) {
|
|
75
|
+
if (url === undefined)
|
|
76
|
+
return undefined;
|
|
77
|
+
const parsed = URL.parse(url);
|
|
78
|
+
if (parsed === null)
|
|
79
|
+
return undefined;
|
|
80
|
+
// A non-web URL keeps its scheme only: a file: path or a data: payload never lands.
|
|
81
|
+
if (!WEB_SCHEMES.has(parsed.protocol))
|
|
82
|
+
return { origin: parsed.protocol, path: "" };
|
|
83
|
+
const origin = redactOrigin(parsed, redact);
|
|
84
|
+
const path = redactPath(parsed.pathname, redact);
|
|
85
|
+
// A study secret can span both parts, such as a whole invite link, which neither part holds.
|
|
86
|
+
const joined = `${origin}${path}`;
|
|
87
|
+
if (redact(joined) !== joined)
|
|
88
|
+
return { origin: REDACTION_MARKERS.urlPart, path: REDACTION_MARKERS.urlPart };
|
|
89
|
+
return { origin, path };
|
|
90
|
+
}
|
package/dist/observer/data.js
CHANGED
|
@@ -156,6 +156,13 @@ export function observerDataFromSaved(value) {
|
|
|
156
156
|
? null
|
|
157
157
|
: joinObserverData(saved, null);
|
|
158
158
|
}
|
|
159
|
+
const GOAL_SOURCE_LABELS = {
|
|
160
|
+
participant_report: "Reported complete",
|
|
161
|
+
condition_matched: "Condition matched",
|
|
162
|
+
scheduled_end: "Ran until scheduled end",
|
|
163
|
+
session_cap: "Ran until session cap",
|
|
164
|
+
unavailable: "Completion source unavailable",
|
|
165
|
+
};
|
|
159
166
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
|
160
167
|
export function withObserverEndings(data) {
|
|
161
168
|
const streams = (data.streams ?? []).map(({ ending: _previousEnding, ...stream }) => {
|
|
@@ -166,11 +173,7 @@ export function withObserverEndings(data) {
|
|
|
166
173
|
...(ending === undefined ? {} : { ending }),
|
|
167
174
|
...((stream.status === "passed" || stream.status === "complete") && source !== undefined
|
|
168
175
|
? {
|
|
169
|
-
statusLabel: source
|
|
170
|
-
? "Reported complete"
|
|
171
|
-
: source === "condition_matched"
|
|
172
|
-
? "Condition matched"
|
|
173
|
-
: "Completion source unavailable",
|
|
176
|
+
statusLabel: GOAL_SOURCE_LABELS[source],
|
|
174
177
|
}
|
|
175
178
|
: {}),
|
|
176
179
|
...(stream.status === "incomplete" || (ending !== undefined && stream.status === "abandoned")
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { LiveItemsCursor } from "../run/live-items.js";
|
|
2
|
+
import type { LiveItemsRead } from "../run/stream-trace.js";
|
|
3
|
+
import type { ObserverItems } from "./participant-items.js";
|
|
4
|
+
/**
|
|
5
|
+
* The prefixes a poll's `after` parameter says the page holds, by live file path. A parameter with
|
|
6
|
+
* a cursor that is not spelled as the server spells them, more than 10,000 cursors, or two for one
|
|
7
|
+
* file is ignored whole, so the page gets every list in full.
|
|
8
|
+
*/
|
|
9
|
+
export declare function parseFeedCursors(after: string | null): ReadonlyMap<string, LiveItemsCursor>;
|
|
10
|
+
/**
|
|
11
|
+
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
12
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
13
|
+
*/
|
|
14
|
+
export declare function withFeedCursors(items: ObserverItems, live: readonly LiveItemsRead[]): ObserverItems;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// The cursors of the Observer's feed. A cursor is `<position>.<generation>.<items>.<bytes>`: the
|
|
2
|
+
// live file live/<position>.<generation>.ndjson (src/run/live-items.ts names them) and the prefix of
|
|
3
|
+
// it a page holds. Each feed response gives a running participant's cursor as `live`; the page sends
|
|
4
|
+
// the cursors back in `after` and compares them only as strings, so a cursor has one spelling:
|
|
5
|
+
// decimal integers without leading zeros.
|
|
6
|
+
/** A live file's stem, `<position>.<generation>`, as isLiveItemsIndex accepts its path. */
|
|
7
|
+
const LIVE_PATH = /^live\/([1-9]\d{0,5}\.[1-9]\d{0,8})\.ndjson$/;
|
|
8
|
+
/** A cursor: the stem, then the line and byte counts. */
|
|
9
|
+
const CURSOR = /^([1-9]\d{0,5}\.[1-9]\d{0,8})\.(0|[1-9]\d{0,15})\.(0|[1-9]\d{0,15})$/;
|
|
10
|
+
/** More cursors than any page sends: a run has one live file per participant. */
|
|
11
|
+
const MAX_CURSORS = 10_000;
|
|
12
|
+
/**
|
|
13
|
+
* The prefixes a poll's `after` parameter says the page holds, by live file path. A parameter with
|
|
14
|
+
* a cursor that is not spelled as the server spells them, more than 10,000 cursors, or two for one
|
|
15
|
+
* file is ignored whole, so the page gets every list in full.
|
|
16
|
+
*/
|
|
17
|
+
export function parseFeedCursors(after) {
|
|
18
|
+
const cursors = new Map();
|
|
19
|
+
if (after === null || after === "")
|
|
20
|
+
return cursors;
|
|
21
|
+
const parts = after.split(",", MAX_CURSORS + 1);
|
|
22
|
+
if (parts.length > MAX_CURSORS)
|
|
23
|
+
return new Map();
|
|
24
|
+
for (const part of parts) {
|
|
25
|
+
const match = CURSOR.exec(part);
|
|
26
|
+
const path = `live/${match?.[1]}.ndjson`;
|
|
27
|
+
const cursor = { items: Number(match?.[2]), bytes: Number(match?.[3]) };
|
|
28
|
+
if (match === null ||
|
|
29
|
+
!Number.isSafeInteger(cursor.items) ||
|
|
30
|
+
!Number.isSafeInteger(cursor.bytes) ||
|
|
31
|
+
cursors.has(path))
|
|
32
|
+
return new Map();
|
|
33
|
+
cursors.set(path, cursor);
|
|
34
|
+
}
|
|
35
|
+
return cursors;
|
|
36
|
+
}
|
|
37
|
+
/** The cursor at `items` lines and `bytes` bytes into the live file at `path`. */
|
|
38
|
+
function cursorText(path, position) {
|
|
39
|
+
const stem = LIVE_PATH.exec(path)?.[1];
|
|
40
|
+
if (stem === undefined)
|
|
41
|
+
throw new Error(`${path} is not a live file name.`);
|
|
42
|
+
return `${stem}.${position.items}.${position.bytes}`;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
46
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
47
|
+
*/
|
|
48
|
+
export function withFeedCursors(items, live) {
|
|
49
|
+
const streams = Object.assign(Object.create(null), items.streams);
|
|
50
|
+
for (const read of live) {
|
|
51
|
+
const entry = Object.hasOwn(streams, read.streamId) ? streams[read.streamId] : undefined;
|
|
52
|
+
if (entry === undefined || !Array.isArray(entry.liveActor))
|
|
53
|
+
continue;
|
|
54
|
+
streams[read.streamId] = {
|
|
55
|
+
...entry,
|
|
56
|
+
live: cursorText(read.index.path, read.index),
|
|
57
|
+
...(read.after === undefined ? {} : { liveAfter: cursorText(read.index.path, read.after) }),
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
return { ...items, streams };
|
|
61
|
+
}
|
|
@@ -23,6 +23,13 @@ export interface ObserverItems {
|
|
|
23
23
|
actor?: ActorTraceItem[];
|
|
24
24
|
liveActor?: ActorTraceItem[];
|
|
25
25
|
artifacts?: ObserverStream["artifacts"];
|
|
26
|
+
/**
|
|
27
|
+
* Where `liveActor` ends in the participant's live file, as a cursor the page sends back
|
|
28
|
+
* (src/observer/feed-cursors.ts). Only on a feed response, for a list read from a live file.
|
|
29
|
+
*/
|
|
30
|
+
live?: string;
|
|
31
|
+
/** Set when `liveActor` holds only the items past this cursor, one the page sent. */
|
|
32
|
+
liveAfter?: string;
|
|
26
33
|
}>;
|
|
27
34
|
}
|
|
28
35
|
/** The data as observer-data.json and the page's items carry it. */
|
|
@@ -18,11 +18,12 @@ export interface PinnedDirectory {
|
|
|
18
18
|
* The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
|
|
19
19
|
* file larger than that is refused from its opened size and the read stops there, so a large file
|
|
20
20
|
* is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
|
|
21
|
-
* a shorter file
|
|
21
|
+
* a shorter file; with `start` as well, the bytes from `start` to `length`.
|
|
22
22
|
*/
|
|
23
23
|
export declare function readContainedFile(root: PinnedDirectory, filePathInput: string, options?: {
|
|
24
24
|
maxBytes?: number;
|
|
25
25
|
length?: number;
|
|
26
|
+
start?: number;
|
|
26
27
|
}): Promise<Buffer | null>;
|
|
27
28
|
export declare function openContainedFile(root: PinnedDirectory, filePathInput: string): Promise<{
|
|
28
29
|
handle: FileHandle;
|
|
@@ -11,7 +11,7 @@ import { isPathInside, isSafeRunIdSegment } from "../run/paths.js";
|
|
|
11
11
|
* The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
|
|
12
12
|
* file larger than that is refused from its opened size and the read stops there, so a large file
|
|
13
13
|
* is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
|
|
14
|
-
* a shorter file
|
|
14
|
+
* a shorter file; with `start` as well, the bytes from `start` to `length`.
|
|
15
15
|
*/
|
|
16
16
|
export async function readContainedFile(root, filePathInput, options = {}) {
|
|
17
17
|
const opened = await openContainedFile(root, filePathInput);
|
|
@@ -19,7 +19,7 @@ export async function readContainedFile(root, filePathInput, options = {}) {
|
|
|
19
19
|
return null;
|
|
20
20
|
try {
|
|
21
21
|
const body = options.length !== undefined
|
|
22
|
-
? await readOpenedPrefix(opened.handle, options.length)
|
|
22
|
+
? await readOpenedPrefix(opened.handle, options.length, options.start)
|
|
23
23
|
: options.maxBytes === undefined
|
|
24
24
|
? await opened.handle.readFile()
|
|
25
25
|
: opened.size > options.maxBytes
|
|
@@ -17,7 +17,7 @@ export declare function servedRunPaths(runRoot: PinnedDirectory): Promise<Prepar
|
|
|
17
17
|
* internal: consumed by src/observer/serve.ts. `notesToken` goes into the run's Observer page so
|
|
18
18
|
* it can add notes; a server passes it only for a run it takes notes for.
|
|
19
19
|
*/
|
|
20
|
-
export declare function serveRunPath(runRoot: PinnedDirectory, relativePath: string, response: ServerResponse, runtimeStreamUrls?: ObserverRuntimeStreamUrl[], request?: Pick<IncomingMessage, "method" | "headers">, notesToken?: string): Promise<void>;
|
|
20
|
+
export declare function serveRunPath(runRoot: PinnedDirectory, relativePath: string, response: ServerResponse, runtimeStreamUrls?: ObserverRuntimeStreamUrl[], request?: Pick<IncomingMessage, "method" | "headers" | "url">, notesToken?: string): Promise<void>;
|
|
21
21
|
/** internal: exported for the stream lifecycle tests. */
|
|
22
22
|
export declare function withRuntimeStreamUrls(data: ObserverData, runtimeStreamUrls: ObserverRuntimeStreamUrl[]): ObserverData;
|
|
23
23
|
/** internal: consumed by src/observer/serve.ts */
|