humanish 0.133.1 → 0.135.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/actors/computer-use/loop/actions.d.ts +2 -0
- package/dist/actors/computer-use/loop/actions.js +11 -5
- package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
- package/dist/actors/computer-use/loop/backstop.js +40 -7
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +3 -0
- package/dist/actors/computer-use/loop/streak.d.ts +18 -0
- package/dist/actors/computer-use/loop/streak.js +47 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -1
- package/dist/actors/computer-use/loop/watch.d.ts +22 -4
- package/dist/actors/computer-use/loop/watch.js +83 -27
- package/dist/actors/computer-use/loop.js +18 -6
- package/dist/actors/contract.d.ts +3 -1
- package/dist/analysis/automatic-config.js +5 -3
- package/dist/analysis/execute.js +2 -1
- package/dist/analysis/service.js +5 -3
- package/dist/analysis/source.d.ts +6 -0
- package/dist/analysis/source.js +15 -13
- package/dist/cli/commands/run-signals.js +6 -2
- package/dist/evidence/encoded-text.d.ts +13 -3
- package/dist/evidence/encoded-text.js +88 -25
- package/dist/observer/page-data.d.ts +6 -0
- package/dist/observer/page-data.js +34 -7
- package/dist/observer/participant-items.d.ts +15 -1
- package/dist/observer/participant-items.js +4 -3
- package/dist/observer/render.js +5 -3
- package/dist/observer/run-routes.js +4 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
- package/dist/routes/computer-use/bundle-parts.js +24 -4
- package/dist/routes/computer-use/fanout-bundle.js +8 -6
- package/dist/routes/computer-use/participant-runs.js +2 -2
- package/dist/routes/computer-use/single-bundle.js +5 -5
- package/dist/routes/scripted/setup.js +62 -17
- package/dist/routes/shared-world/bundle.js +3 -3
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/terminal/bundle.js +2 -2
- package/dist/run/locate.d.ts +8 -2
- package/dist/run/locate.js +10 -4
- package/dist/run/run.js +22 -5
- package/dist/run/sandbox-ids.d.ts +4 -3
- package/dist/run/sandbox-ids.js +8 -6
- package/dist/run/stream-trace.d.ts +8 -2
- package/dist/run/stream-trace.js +13 -7
- package/dist/run-study.d.ts +3 -3
- package/dist/run-study.js +3 -3
- package/dist/study/bundle-estimate.d.ts +0 -2
- package/dist/study/bundle-estimate.js +33 -33
- package/dist/study/config.d.ts +4 -3
- package/dist/study/config.js +8 -5
- package/dist/study/parse/actors.d.ts +6 -0
- package/dist/study/parse/actors.js +14 -0
- package/dist/study/parse/subject-state.js +4 -4
- package/dist/study/participant-assignment.js +2 -2
- package/dist/study/persona.d.ts +7 -1
- package/dist/study/persona.js +13 -5
- package/dist/study/plan-base.js +4 -3
- package/dist/verify/artifacts.d.ts +7 -1
- package/dist/verify/artifacts.js +15 -4
- package/dist/verify/verify.js +29 -8
- package/docs/architecture/actor-contract.md +3 -1
- package/docs/architecture/observer.md +24 -7
- package/docs/architecture/state-driven-executor.md +7 -4
- package/docs/contracts/run-bundle.md +16 -2
- package/docs/contracts/study-analysis.md +8 -5
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
- package/package.json +1 -1
|
@@ -1,6 +1,15 @@
|
|
|
1
|
+
import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
|
|
1
2
|
import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
|
|
2
3
|
import { type ParticipantIds } from "../../run/participant-records.js";
|
|
3
4
|
import { type ParticipantRunOutcome } from "./types.js";
|
|
5
|
+
/**
|
|
6
|
+
* The review's gap for the streaks a participant's trace records: turns that repeated a recent
|
|
7
|
+
* action on an unchanged screen past noProgressSteps, which did not end the session of a
|
|
8
|
+
* participant held to its scheduled end. Its session passes, so no other gap names them. One line
|
|
9
|
+
* names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
|
|
10
|
+
* time, records a streak a minute, and a fan-out would list each.
|
|
11
|
+
*/
|
|
12
|
+
export declare function streakGaps(participantId: string, session: CuaLoopResult | undefined): string[];
|
|
4
13
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
5
14
|
export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
|
|
6
15
|
/** Human-readable provenance line for the single-participant subject.provenance event of a clone
|
|
@@ -1,14 +1,34 @@
|
|
|
1
1
|
// The parts both computer-use bundle shapes share: the subject-state story and provenance line,
|
|
2
|
-
// provider-resource records,
|
|
3
|
-
// subject-phase event id suffix. The builders are bundle.ts
|
|
4
|
-
// single-bundle.ts and fanout-bundle.ts. Feedback candidates from
|
|
5
|
-
// participant-feedback.ts, which the shared-world bundle reads too.
|
|
2
|
+
// provider-resource records, the review's gap for a held participant's streak, and the public-safe
|
|
3
|
+
// app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts
|
|
4
|
+
// (the dispatcher and the judge), single-bundle.ts and fanout-bundle.ts. Feedback candidates from
|
|
5
|
+
// participant reports are in participant-feedback.ts, which the shared-world bundle reads too.
|
|
6
|
+
import { STREAK_TITLE } from "../../actors/computer-use/loop/streak.js";
|
|
6
7
|
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
7
8
|
import { digestText } from "../../evidence/redaction.js";
|
|
8
9
|
import {} from "../../run/bundle.js";
|
|
9
10
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
10
11
|
import {} from "./types.js";
|
|
11
12
|
import { plural } from "../../run/text.js";
|
|
13
|
+
/**
|
|
14
|
+
* The review's gap for the streaks a participant's trace records: turns that repeated a recent
|
|
15
|
+
* action on an unchanged screen past noProgressSteps, which did not end the session of a
|
|
16
|
+
* participant held to its scheduled end. Its session passes, so no other gap names them. One line
|
|
17
|
+
* names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
|
|
18
|
+
* time, records a streak a minute, and a fan-out would list each.
|
|
19
|
+
*/
|
|
20
|
+
export function streakGaps(participantId, session) {
|
|
21
|
+
const texts = (session?.trace.items ?? []).flatMap((item) => item.kind === "notice" && item.title === STREAK_TITLE && item.text !== undefined
|
|
22
|
+
? [item.text]
|
|
23
|
+
: []);
|
|
24
|
+
const [first] = texts;
|
|
25
|
+
if (first === undefined)
|
|
26
|
+
return [];
|
|
27
|
+
const more = texts.length > 1
|
|
28
|
+
? ` It had ${plural(texts.length - 1, "more such streak")} after that; its trace records each.`
|
|
29
|
+
: "";
|
|
30
|
+
return [`${participantId}: ${first}${more}`];
|
|
31
|
+
}
|
|
12
32
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
13
33
|
export function describeSubjectState(state, dryRun) {
|
|
14
34
|
switch (state.provenance) {
|
|
@@ -2,10 +2,10 @@ import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
|
2
2
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
3
3
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
4
4
|
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
5
|
-
import {
|
|
5
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
8
|
-
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
|
|
9
9
|
import { subjectName } from "../../run/subject-name.js";
|
|
10
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
11
11
|
import { participantFactsOf } from "./participant-facts.js";
|
|
@@ -70,18 +70,20 @@ function fanoutSummary(args, facts) {
|
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
72
|
* The review's gaps: one line for the participants a host suspension likely failed, then a line
|
|
73
|
-
* for each other participant that did not pass.
|
|
73
|
+
* for each other participant that did not pass, then a line for each held participant's streaks.
|
|
74
74
|
*/
|
|
75
75
|
function fanoutGaps(args, judgment) {
|
|
76
76
|
const suspended = new Set(args.hostSuspension?.participantIds ?? []);
|
|
77
77
|
const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
|
|
78
78
|
? []
|
|
79
79
|
: [participant.gapLine]);
|
|
80
|
+
const streaks = args.specs.flatMap((spec, index) => streakGaps(spec.planned.id, args.outcomes?.[index]?.session));
|
|
80
81
|
if (suspended.size === 0)
|
|
81
|
-
return others;
|
|
82
|
+
return [...others, ...streaks];
|
|
82
83
|
return [
|
|
83
84
|
`${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
|
|
84
85
|
...others,
|
|
86
|
+
...streaks,
|
|
85
87
|
];
|
|
86
88
|
}
|
|
87
89
|
function fanoutReview(args, streams, judgment) {
|
|
@@ -184,7 +186,7 @@ function fanoutFeedbackCandidates(args) {
|
|
|
184
186
|
participantId: spec.planned.id,
|
|
185
187
|
streamId: spec.streamId,
|
|
186
188
|
personaId: spec.persona.id,
|
|
187
|
-
goal:
|
|
189
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
188
190
|
...(outcome?.session === undefined ? {} : { session: outcome.session }),
|
|
189
191
|
...(outcome?.session === undefined
|
|
190
192
|
? {}
|
|
@@ -268,7 +270,7 @@ export function buildCuaFanoutBundle(args) {
|
|
|
268
270
|
// this writer produced. The only adopter-side workaround was scanner evasion.
|
|
269
271
|
//
|
|
270
272
|
// The instructions the model actually receives are untouched; only the persisted copy changes.
|
|
271
|
-
goal:
|
|
273
|
+
goal: redactEncodedText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
272
274
|
source: `study:${plan.studyId}`,
|
|
273
275
|
sourceDigest: specs[0].persona.promptDigest,
|
|
274
276
|
},
|
|
@@ -6,7 +6,7 @@ import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrate
|
|
|
6
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
7
7
|
import {} from "../../study/types.js";
|
|
8
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
9
|
-
import {
|
|
9
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
10
10
|
import {} from "../../run/bundle.js";
|
|
11
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
12
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
@@ -237,7 +237,7 @@ export function sanitizeParticipantRuns(runs, scrub) {
|
|
|
237
237
|
for (const spec of runs) {
|
|
238
238
|
if (spec.evidenceAssignment)
|
|
239
239
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrub);
|
|
240
|
-
spec.evidenceInstructions =
|
|
240
|
+
spec.evidenceInstructions = redactEncodedText(scrub(spec.instructions));
|
|
241
241
|
spec.persona = scrubPersonaBrief(spec.persona, scrub);
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
2
|
-
import {
|
|
2
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
5
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
6
6
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
7
7
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
8
8
|
import { cuaGoalSource, ranUntilClockEnd } from "../../actors/goal-source.js";
|
|
9
|
-
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
9
|
+
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, streakGaps, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
10
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
11
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
12
12
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
@@ -348,7 +348,7 @@ function singleReview(args, view, stream) {
|
|
|
348
348
|
...(credibilityNote === undefined ? {} : { credibilityNote }),
|
|
349
349
|
})}`,
|
|
350
350
|
gaps: args.session || args.sessionError !== undefined
|
|
351
|
-
?
|
|
351
|
+
? streakGaps(args.participantId ?? "lane-01", args.session)
|
|
352
352
|
: args.inProgress === true
|
|
353
353
|
? ["Live desktop session is still running."]
|
|
354
354
|
: ["Live desktop session not yet run (dry run only)."],
|
|
@@ -385,7 +385,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
385
385
|
scenario: {
|
|
386
386
|
id: `cua-${args.studyId}`,
|
|
387
387
|
title: args.studyTitle ?? `Computer-use run: ${args.studyId}`,
|
|
388
|
-
goal:
|
|
388
|
+
goal: redactEncodedText(args.mission),
|
|
389
389
|
source: `study:${args.studyId}`,
|
|
390
390
|
sourceDigest: args.persona.promptDigest,
|
|
391
391
|
},
|
|
@@ -425,7 +425,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
425
425
|
participantId: args.participantId ?? "lane-01",
|
|
426
426
|
streamId: SINGLE.streamId,
|
|
427
427
|
personaId: args.persona.id,
|
|
428
|
-
goal:
|
|
428
|
+
goal: redactEncodedText(args.mission),
|
|
429
429
|
...(args.session === undefined ? {} : { session: args.session }),
|
|
430
430
|
...(args.traceArtifactPath === undefined
|
|
431
431
|
? {}
|
|
@@ -8,6 +8,7 @@ import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-com
|
|
|
8
8
|
import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
|
|
9
9
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
10
10
|
import { RunSecrets } from "../../run/secrets.js";
|
|
11
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
11
12
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
12
13
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
13
14
|
import {} from "./types.js";
|
|
@@ -55,6 +56,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
55
56
|
result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
|
|
56
57
|
};
|
|
57
58
|
}
|
|
59
|
+
const steps = recordedStepIds(scenario.journey.steps, secrets);
|
|
60
|
+
if (!steps.ok)
|
|
61
|
+
return { ok: false, result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", steps.message) };
|
|
58
62
|
// The goal, title and step labels are recorded; a step's path, selector, value and expected text
|
|
59
63
|
// drive the browser and stay as written.
|
|
60
64
|
const journey = {
|
|
@@ -62,9 +66,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
62
66
|
scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
|
|
63
67
|
? "scenario"
|
|
64
68
|
: scenario.journey.scenarioId,
|
|
65
|
-
goal:
|
|
66
|
-
scenarioTitle:
|
|
67
|
-
steps:
|
|
69
|
+
goal: recordedText(scenario.journey.goal, secrets),
|
|
70
|
+
scenarioTitle: recordedText(scenario.journey.scenarioTitle, secrets),
|
|
71
|
+
steps: steps.steps,
|
|
68
72
|
};
|
|
69
73
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
70
74
|
const { requirements } = plan;
|
|
@@ -127,6 +131,13 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
127
131
|
},
|
|
128
132
|
};
|
|
129
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Scenario text the run records, without the run's known values and then without keys as written
|
|
136
|
+
* or encoded, since verify decodes the files that record it.
|
|
137
|
+
*/
|
|
138
|
+
function recordedText(text, secrets) {
|
|
139
|
+
return redactEncodedText(secrets.scrub(text));
|
|
140
|
+
}
|
|
130
141
|
/**
|
|
131
142
|
* Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
|
|
132
143
|
* other characters as `-`, which the scrub would not find.
|
|
@@ -139,22 +150,56 @@ function idHoldsKnownValue(id, secrets) {
|
|
|
139
150
|
}));
|
|
140
151
|
}
|
|
141
152
|
/**
|
|
142
|
-
* The steps with their labels
|
|
143
|
-
* screenshot file and is recorded beside it, and the parser derives an
|
|
144
|
-
* lower-cased and cut to 80 characters, so the scrub may not find a
|
|
145
|
-
* label or id holds a known value is
|
|
153
|
+
* The steps with their labels redacted and their ids safe to record, or why the run is refused. A
|
|
154
|
+
* step id names the step's screenshot file and is recorded beside it, and the parser derives an
|
|
155
|
+
* omitted id from the label, lower-cased and cut to 80 characters, so the scrub may not find a
|
|
156
|
+
* value in it. A step whose label changes under recordedText, or whose id holds a known value, is
|
|
157
|
+
* renamed by its position instead.
|
|
146
158
|
*/
|
|
147
159
|
function recordedStepIds(steps, secrets) {
|
|
148
160
|
const taken = new Set(steps.map((step) => step.id));
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
161
|
+
const recorded = [];
|
|
162
|
+
for (const [index, step] of steps.entries()) {
|
|
163
|
+
const label = recordedText(step.label, secrets);
|
|
164
|
+
if (label === step.label && !idHoldsKnownValue(step.id, secrets)) {
|
|
165
|
+
recorded.push({ ...step, label });
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const id = positionId(index, taken, secrets);
|
|
169
|
+
if (id === undefined)
|
|
170
|
+
return {
|
|
171
|
+
ok: false,
|
|
172
|
+
message: `The label or id of scenario step ${index + 1} holds a key, a local path or a value this run keeps out of its files, so the run renames the step by its position, and each id that position gives holds one of those values too. Remove it from the step's label and id.`,
|
|
173
|
+
};
|
|
157
174
|
taken.add(id);
|
|
158
|
-
|
|
159
|
-
}
|
|
175
|
+
recorded.push({ ...step, id, label });
|
|
176
|
+
}
|
|
177
|
+
return { ok: true, steps: recorded };
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The id a renamed step takes by its position, such as step-03 for the third step, then step-03-2,
|
|
181
|
+
* step-03-3 and on when another step has it. A known value can be one of these ids, so one that
|
|
182
|
+
* holds a known value is passed over. Every id holds the position id, so a position id that holds a
|
|
183
|
+
* known value gives undefined at once. Otherwise taken.size + 1 ids that hold no known value are
|
|
184
|
+
* tried, which includes a free one, as long as known values pass over at most
|
|
185
|
+
* MAX_KNOWN_VALUE_SKIPS of them; a value such as `-01-` that every numbered id holds gives undefined.
|
|
186
|
+
*/
|
|
187
|
+
function positionId(index, taken, secrets) {
|
|
188
|
+
const position = `step-${String(index + 1).padStart(2, "0")}`;
|
|
189
|
+
if (idHoldsKnownValue(position, secrets))
|
|
190
|
+
return undefined;
|
|
191
|
+
let skips = 0;
|
|
192
|
+
for (let suffix = 1; suffix <= taken.size + 1 + skips; suffix += 1) {
|
|
193
|
+
const id = suffix === 1 ? position : `${position}-${suffix}`;
|
|
194
|
+
if (taken.has(id))
|
|
195
|
+
continue;
|
|
196
|
+
if (!idHoldsKnownValue(id, secrets))
|
|
197
|
+
return id;
|
|
198
|
+
if (skips === MAX_KNOWN_VALUE_SKIPS)
|
|
199
|
+
return undefined;
|
|
200
|
+
skips += 1;
|
|
201
|
+
}
|
|
202
|
+
return undefined;
|
|
160
203
|
}
|
|
204
|
+
// Known values are few, and a numbered id holds one only when the value matches its digits.
|
|
205
|
+
const MAX_KNOWN_VALUE_SKIPS = 32;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Builds the run bundle for a concurrent shared-world run: per-participant simulations, streams and
|
|
2
2
|
// events, the sharedWorld evidence block and the review summary.
|
|
3
3
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
4
|
-
import {
|
|
4
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
5
5
|
import { planeStateOf } from "./plan.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } from "../../run/shared-world-evidence.js";
|
|
@@ -362,7 +362,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
362
362
|
scenario: {
|
|
363
363
|
id: `concurrent-shared-world-${plan.studyId}`,
|
|
364
364
|
title: plan.title ?? `Concurrent shared-world: ${plan.studyId}`,
|
|
365
|
-
goal:
|
|
365
|
+
goal: redactEncodedText(actorSpecs[0]?.evidenceInstructions ??
|
|
366
366
|
actorSpecs[0]?.instructions ??
|
|
367
367
|
"Concurrent shared-world interaction."),
|
|
368
368
|
source: `study:${plan.studyId}`,
|
|
@@ -423,7 +423,7 @@ function sharedWorldFeedbackCandidates(args) {
|
|
|
423
423
|
participantId: spec.planned.id,
|
|
424
424
|
streamId: spec.streamId,
|
|
425
425
|
personaId: spec.persona.id,
|
|
426
|
-
goal:
|
|
426
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
427
427
|
...(outcome?.session === undefined
|
|
428
428
|
? {}
|
|
429
429
|
: { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
|
|
@@ -10,7 +10,7 @@ import { startLiveTraceFlush } from "../computer-use/live-flush.js";
|
|
|
10
10
|
import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
|
-
import {
|
|
13
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
14
14
|
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
16
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
@@ -178,7 +178,7 @@ export async function buildParticipantSpecs(participants, cwd, scrubKnownValues)
|
|
|
178
178
|
for (const spec of actorSpecs) {
|
|
179
179
|
if (spec.evidenceAssignment)
|
|
180
180
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrubKnownValues);
|
|
181
|
-
spec.evidenceInstructions =
|
|
181
|
+
spec.evidenceInstructions = redactEncodedText(scrubKnownValues(spec.instructions));
|
|
182
182
|
spec.persona = scrubPersonaBrief(spec.persona, scrubKnownValues);
|
|
183
183
|
}
|
|
184
184
|
return actorSpecs;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/judge.js";
|
|
@@ -250,7 +250,7 @@ function terminalRunBundle(args, parts) {
|
|
|
250
250
|
// The author mission is public-safe committed study text. It is recorded plaintext as the goal,
|
|
251
251
|
// redacted defensively before persisting (it never carries a secret, but the harness never
|
|
252
252
|
// trusts that). The full composed prompt is bound by digest, not text.
|
|
253
|
-
goal:
|
|
253
|
+
goal: redactEncodedText(args.mission),
|
|
254
254
|
source: `study:${args.studyId}`,
|
|
255
255
|
sourceDigest: args.persona.promptDigest,
|
|
256
256
|
},
|
package/dist/run/locate.d.ts
CHANGED
|
@@ -55,8 +55,14 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
|
|
|
55
55
|
bundlePath: string;
|
|
56
56
|
runDir: string;
|
|
57
57
|
} | null>;
|
|
58
|
-
/**
|
|
59
|
-
|
|
58
|
+
/**
|
|
59
|
+
* loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
|
|
60
|
+
* `display`, a participant whose trace the loader refuses, such as one past the read limit, is
|
|
61
|
+
* kept as displayedTraces shows it, for a page that shows the run.
|
|
62
|
+
*/
|
|
63
|
+
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
|
|
64
|
+
display?: boolean;
|
|
65
|
+
}): Promise<{
|
|
60
66
|
bundle: RunBundle;
|
|
61
67
|
bundlePath: string;
|
|
62
68
|
runDir: string;
|
package/dist/run/locate.js
CHANGED
|
@@ -3,7 +3,7 @@ import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDir
|
|
|
3
3
|
import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
|
|
4
4
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
5
5
|
import { isRunBundle, isRunPointer } from "./bundle-shape.js";
|
|
6
|
-
import { loadRunTraces } from "./stream-trace.js";
|
|
6
|
+
import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
|
|
7
7
|
import { readUnchangedFile } from "./evidence-files.js";
|
|
8
8
|
/** The run input that resolves through latest.json rather than naming a run directory. */
|
|
9
9
|
export const LATEST_RUN_ALIAS = "latest";
|
|
@@ -119,8 +119,12 @@ export async function loadRunBundle(cwdInput, runInput) {
|
|
|
119
119
|
}
|
|
120
120
|
return loadRunBundlePrepared(cwd, runPaths);
|
|
121
121
|
}
|
|
122
|
-
/**
|
|
123
|
-
|
|
122
|
+
/**
|
|
123
|
+
* loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
|
|
124
|
+
* `display`, a participant whose trace the loader refuses, such as one past the read limit, is
|
|
125
|
+
* kept as displayedTraces shows it, for a page that shows the run.
|
|
126
|
+
*/
|
|
127
|
+
export async function loadRunBundlePrepared(cwdInput, runPaths, options = {}) {
|
|
124
128
|
const cwd = path.resolve(cwdInput);
|
|
125
129
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
126
130
|
const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
|
|
@@ -134,7 +138,9 @@ export async function loadRunBundlePrepared(cwdInput, runPaths) {
|
|
|
134
138
|
}
|
|
135
139
|
return {
|
|
136
140
|
// A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
|
|
137
|
-
bundle:
|
|
141
|
+
bundle: options.display === true
|
|
142
|
+
? displayedTraces(await loadStreamTraces(runPaths, bundle))
|
|
143
|
+
: await loadRunTraces(runPaths, bundle),
|
|
138
144
|
bundlePath: path.relative(cwd, bundlePath),
|
|
139
145
|
runDir: runPaths.absoluteRunRoot,
|
|
140
146
|
};
|
package/dist/run/run.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// their own evidence files; this module owns when the run starts, how it is published, and how it
|
|
4
4
|
// is closed on every exit.
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
6
7
|
import { redactText } from "../evidence/redaction.js";
|
|
7
8
|
import { buildObserverData } from "../observer/data.js";
|
|
8
9
|
import { splitObserverData } from "../observer/participant-items.js";
|
|
@@ -148,24 +149,35 @@ async function writeRunJson(paths, liveItems, bundle) {
|
|
|
148
149
|
});
|
|
149
150
|
return { ...publicBundle, streams };
|
|
150
151
|
}
|
|
152
|
+
/** The warn event that names the files the final sandbox id sweep could not read. */
|
|
153
|
+
const unsweptEvent = (message, at) => ({
|
|
154
|
+
id: "event-sandbox-ids-unresolved",
|
|
155
|
+
at: new Date(at).toISOString(),
|
|
156
|
+
level: "warn",
|
|
157
|
+
type: "sandbox-ids.unresolved",
|
|
158
|
+
message,
|
|
159
|
+
});
|
|
151
160
|
function runPublisher(args) {
|
|
152
161
|
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
153
162
|
const observer = observerTarget(options);
|
|
154
163
|
// The study's warnings join whatever events the route wrote, once, in every write. A warning can
|
|
155
|
-
// quote the study file, which may name a sandbox URL a participant needs
|
|
156
|
-
// the redaction
|
|
164
|
+
// quote the study file, which may name a sandbox URL a participant needs or hold an encoded key,
|
|
165
|
+
// so each goes through the redaction study text gets.
|
|
157
166
|
const studyWarnings = (options.warnings ?? []).map((message, index) => ({
|
|
158
167
|
id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
|
|
159
168
|
at: createdAt,
|
|
160
169
|
level: "warn",
|
|
161
170
|
type: "study.warning",
|
|
162
|
-
message:
|
|
171
|
+
message: redactEncodedText(message),
|
|
163
172
|
}));
|
|
164
|
-
// So do the host suspensions the heartbeat has seen by the time of the write
|
|
173
|
+
// So do the host suspensions the heartbeat has seen by the time of the write, and the files the
|
|
174
|
+
// final sandbox id sweep could not read.
|
|
175
|
+
let unswept = [];
|
|
165
176
|
const withRunEvents = (events) => {
|
|
166
177
|
const owned = [
|
|
167
178
|
...studyWarnings,
|
|
168
179
|
...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
|
|
180
|
+
...unswept,
|
|
169
181
|
];
|
|
170
182
|
return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
|
|
171
183
|
};
|
|
@@ -231,7 +243,12 @@ function runPublisher(args) {
|
|
|
231
243
|
await writeBundleFiles(bundle, finishedOutcome(outcome), (publicBundle) => runStatus.finish(runStatusOutcome(publicBundle)));
|
|
232
244
|
// The final run.json indexes each finished trace, so the live files are journal no reader needs.
|
|
233
245
|
await liveItems.remove();
|
|
234
|
-
await scrubRunSandboxIds(paths);
|
|
246
|
+
const unread = await scrubRunSandboxIds(paths);
|
|
247
|
+
if (unread !== undefined) {
|
|
248
|
+
unswept = [unsweptEvent(unread, now())];
|
|
249
|
+
// The sweep reads the files the final write left, so run.json is written again to record it.
|
|
250
|
+
await writeBundleFiles(bundle, finishedOutcome(outcome), async () => { });
|
|
251
|
+
}
|
|
235
252
|
await writePointer();
|
|
236
253
|
let current = { ok: outcome.ok, execution: outcome.execution };
|
|
237
254
|
// An Observer that did not render, judged under the route's policy and written to run.json
|
|
@@ -45,10 +45,11 @@ export declare function scrubSandboxIdBytes(file: string, bytes: Buffer, ids: re
|
|
|
45
45
|
* receipts, with the label in place of the id. Writers such as a participant's actor.json can
|
|
46
46
|
* quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
|
|
47
47
|
* Best effort for writes and listings: verify grades a file this misses. A text file it refuses
|
|
48
|
-
* to read,
|
|
49
|
-
*
|
|
48
|
+
* to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
|
|
49
|
+
* returns a warning that names each such file and why, for the caller to record; the run goes
|
|
50
|
+
* on. A journal it refuses throws, since without its ids no file can be swept.
|
|
50
51
|
*/
|
|
51
|
-
export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<
|
|
52
|
+
export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<string | undefined>;
|
|
52
53
|
/**
|
|
53
54
|
* A run's result as it may be returned or printed: the same object when it names no sandbox,
|
|
54
55
|
* otherwise its public view, scrubbed of the ids the run's receipts journal.
|
package/dist/run/sandbox-ids.js
CHANGED
|
@@ -197,16 +197,17 @@ const MAX_SWEPT_ENTRIES = 10_000;
|
|
|
197
197
|
* receipts, with the label in place of the id. Writers such as a participant's actor.json can
|
|
198
198
|
* quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
|
|
199
199
|
* Best effort for writes and listings: verify grades a file this misses. A text file it refuses
|
|
200
|
-
* to read,
|
|
201
|
-
*
|
|
200
|
+
* to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
|
|
201
|
+
* returns a warning that names each such file and why, for the caller to record; the run goes
|
|
202
|
+
* on. A journal it refuses throws, since without its ids no file can be swept.
|
|
202
203
|
*/
|
|
203
204
|
export async function scrubRunSandboxIds(paths) {
|
|
204
205
|
const ids = await readRunSandboxIds(paths);
|
|
205
206
|
if (ids.length === 0)
|
|
206
|
-
return;
|
|
207
|
+
return undefined;
|
|
207
208
|
const valid = await validatePreparedRunRootIdentity(paths).then(() => true, () => false);
|
|
208
209
|
if (!valid)
|
|
209
|
-
return;
|
|
210
|
+
return undefined;
|
|
210
211
|
const refused = [];
|
|
211
212
|
const scrub = async (file) => {
|
|
212
213
|
const read = await readContainedRegularFile(paths, file, RUN_ARTIFACT_MAX_BYTES);
|
|
@@ -242,8 +243,9 @@ export async function scrubRunSandboxIds(paths) {
|
|
|
242
243
|
await walk("").catch(() => {
|
|
243
244
|
// A directory that changed while it was listed ends the sweep; verify names what it missed.
|
|
244
245
|
});
|
|
245
|
-
|
|
246
|
-
|
|
246
|
+
return refused.length === 0
|
|
247
|
+
? undefined
|
|
248
|
+
: `The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`;
|
|
247
249
|
}
|
|
248
250
|
/**
|
|
249
251
|
* A run's result as it may be returned or printed: the same object when it names no sandbox,
|
|
@@ -11,10 +11,16 @@ export declare class StreamTraceRefusedError extends Error {
|
|
|
11
11
|
constructor(streamId: string, detail: string, advice?: string);
|
|
12
12
|
}
|
|
13
13
|
/**
|
|
14
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
15
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
14
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
15
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
16
16
|
*/
|
|
17
17
|
export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
18
|
+
/** The file's run-relative path, as its index records it. */
|
|
19
|
+
readonly file: string;
|
|
20
|
+
constructor(streamId: string, index: {
|
|
21
|
+
path: string;
|
|
22
|
+
bytes: number;
|
|
23
|
+
}, maxBytes: number);
|
|
18
24
|
}
|
|
19
25
|
/** `trace` without the per-request lists; every other field as it is. */
|
|
20
26
|
export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
|
package/dist/run/stream-trace.js
CHANGED
|
@@ -28,13 +28,17 @@ export class StreamTraceRefusedError extends Error {
|
|
|
28
28
|
/** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
|
|
29
29
|
const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
|
|
30
30
|
/**
|
|
31
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
32
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
31
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
32
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
33
33
|
*/
|
|
34
34
|
export class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
35
|
+
/** The file's run-relative path, as its index records it. */
|
|
36
|
+
file;
|
|
37
|
+
constructor(streamId, index, maxBytes) {
|
|
38
|
+
super(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
39
|
+
this.file = index.path;
|
|
40
|
+
}
|
|
35
41
|
}
|
|
36
|
-
/** The refusal of a trace or live file whose index records more than `maxBytes`. */
|
|
37
|
-
const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
38
42
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
39
43
|
const isScreenshot = (artifact) => artifact.kind === "screenshot";
|
|
40
44
|
/**
|
|
@@ -139,7 +143,9 @@ export function splitStreamTraces(bundle) {
|
|
|
139
143
|
throw new Error(`Two participants' traces would both be written to ${filePath}.`);
|
|
140
144
|
claimed.add(traceFileKey(filePath));
|
|
141
145
|
const screenshots = stream.artifacts.filter(isScreenshot);
|
|
142
|
-
|
|
146
|
+
// Compact: every reader refuses a file past RUN_ARTIFACT_MAX_BYTES, and indenting made a
|
|
147
|
+
// trace 1.36 times its compact size on 236 recorded computer-use traces.
|
|
148
|
+
const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots })}\n`);
|
|
143
149
|
files.push({ path: filePath, bytes });
|
|
144
150
|
const { items, ...summary } = actor;
|
|
145
151
|
return {
|
|
@@ -274,7 +280,7 @@ async function readLiveItems(read, stream, maxBytes, liveAfter) {
|
|
|
274
280
|
const after = liveAfter?.(index);
|
|
275
281
|
if (index.bytes > maxBytes) {
|
|
276
282
|
if (after === undefined || !cursorFits(index, after))
|
|
277
|
-
throw
|
|
283
|
+
throw new StreamTraceTooLargeError(stream.id, index, maxBytes);
|
|
278
284
|
// The caller keeps the prefix it holds, read when the file was under the limit.
|
|
279
285
|
return {
|
|
280
286
|
stream: { ...stream, liveActor: { ...summary, items: [] } },
|
|
@@ -343,7 +349,7 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
|
343
349
|
if (stream.trace !== undefined) {
|
|
344
350
|
assertIndex(stream);
|
|
345
351
|
if (stream.trace.bytes > maxBytes)
|
|
346
|
-
throw
|
|
352
|
+
throw new StreamTraceTooLargeError(stream.id, stream.trace, maxBytes);
|
|
347
353
|
const file = await read(stream.trace.path, maxBytes);
|
|
348
354
|
if ("refused" in file)
|
|
349
355
|
throw new StreamTraceRefusedError(stream.id, file.refused);
|
package/dist/run-study.d.ts
CHANGED
|
@@ -21,9 +21,9 @@ export declare function runStudyWith(config: StudyConfig, options: InternalRunSt
|
|
|
21
21
|
/**
|
|
22
22
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
23
23
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
24
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
25
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
26
|
-
* anything runs.
|
|
24
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
25
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
26
|
+
* route's own result envelope before anything runs.
|
|
27
27
|
*/
|
|
28
28
|
export declare function runStudy(config: StudyConfig, options: RunStudyOptions): Promise<StudyOutcome>;
|
|
29
29
|
/** A study planned once: the route's refusal, or the run of its plan. */
|
package/dist/run-study.js
CHANGED
|
@@ -34,9 +34,9 @@ export async function runStudyWith(config, options, deps = {}) {
|
|
|
34
34
|
/**
|
|
35
35
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
36
36
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
37
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
38
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
39
|
-
* anything runs.
|
|
37
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
38
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
39
|
+
* route's own result envelope before anything runs.
|
|
40
40
|
*/
|
|
41
41
|
export async function runStudy(config, options) {
|
|
42
42
|
const rejected = libraryConfigRefusal(config);
|
|
@@ -16,8 +16,6 @@ export interface BundleEstimate {
|
|
|
16
16
|
readonly runFileBytes: number;
|
|
17
17
|
/** The largest participant trace file, which analysis also reads whole. */
|
|
18
18
|
readonly largestTraceBytes: number;
|
|
19
|
-
/** Every participant's trace file together, whose items the Observer page joins in one string. */
|
|
20
|
-
readonly tracesBytes: number;
|
|
21
19
|
/** observer/index.html, which verify reads whole. */
|
|
22
20
|
readonly observerPageBytes: number;
|
|
23
21
|
/** Every file, screenshots included. */
|