humanish 0.134.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/actors/computer-use/loop/session.js +7 -2
- package/dist/actors/scripted-browser/actor.js +3 -3
- package/dist/actors/scripted-browser/steps.d.ts +9 -2
- package/dist/actors/scripted-browser/steps.js +17 -12
- package/dist/analysis/automatic-config.js +5 -3
- package/dist/analysis/evidence-sources.js +4 -7
- package/dist/analysis/evidence.js +4 -4
- package/dist/analysis/execute.js +2 -1
- package/dist/analysis/service.js +2 -1
- package/dist/analysis/source.d.ts +7 -4
- package/dist/analysis/source.js +22 -4
- package/dist/cli/commands/setup.js +7 -10
- package/dist/evidence/encoded-text.d.ts +13 -3
- package/dist/evidence/encoded-text.js +88 -25
- package/dist/evidence/image.d.ts +2 -2
- package/dist/evidence/image.js +47 -32
- package/dist/observer/data.d.ts +7 -3
- package/dist/observer/page-data.d.ts +6 -0
- package/dist/observer/page-data.js +34 -7
- package/dist/observer/participant-items.d.ts +15 -1
- package/dist/observer/participant-items.js +4 -3
- package/dist/observer/render.js +2 -4
- package/dist/observer/run-routes.js +4 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -3
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-runs.js +2 -2
- package/dist/routes/computer-use/single-bundle.js +3 -3
- package/dist/routes/scripted/setup.js +62 -17
- package/dist/routes/shared-world/bundle.js +3 -3
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/terminal/bundle.js +2 -2
- package/dist/run/bundle.d.ts +5 -1
- package/dist/run/locate.d.ts +13 -11
- package/dist/run/locate.js +20 -13
- package/dist/run/paths.js +38 -28
- package/dist/run/run.js +4 -3
- package/dist/run/stream-trace.d.ts +2 -2
- package/dist/run/stream-trace.js +24 -11
- package/dist/run/streams.d.ts +18 -3
- package/dist/run-study.d.ts +3 -3
- package/dist/run-study.js +3 -3
- package/dist/study/bundle-estimate.d.ts +21 -3
- package/dist/study/bundle-estimate.js +65 -45
- package/dist/study/config.d.ts +4 -3
- package/dist/study/config.js +8 -5
- package/dist/study/parse/actors.d.ts +6 -0
- package/dist/study/parse/actors.js +14 -0
- package/dist/study/parse/subject-state.js +4 -4
- package/dist/study/participant-assignment.js +2 -2
- package/dist/study/persona.d.ts +7 -1
- package/dist/study/persona.js +13 -5
- package/dist/study/plan-base.js +4 -3
- package/dist/tui-app.js +23 -23
- package/dist/verify/screenshot-reads.js +1 -1
- package/docs/architecture/observer.md +24 -7
- package/docs/architecture/state-driven-executor.md +4 -3
- package/docs/contracts/run-bundle.md +3 -0
- package/docs/contracts/study-analysis.md +7 -5
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@ import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
|
2
2
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
3
3
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
4
4
|
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
5
|
-
import {
|
|
5
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
8
8
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
|
|
@@ -186,7 +186,7 @@ function fanoutFeedbackCandidates(args) {
|
|
|
186
186
|
participantId: spec.planned.id,
|
|
187
187
|
streamId: spec.streamId,
|
|
188
188
|
personaId: spec.persona.id,
|
|
189
|
-
goal:
|
|
189
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
190
190
|
...(outcome?.session === undefined ? {} : { session: outcome.session }),
|
|
191
191
|
...(outcome?.session === undefined
|
|
192
192
|
? {}
|
|
@@ -270,7 +270,7 @@ export function buildCuaFanoutBundle(args) {
|
|
|
270
270
|
// this writer produced. The only adopter-side workaround was scanner evasion.
|
|
271
271
|
//
|
|
272
272
|
// The instructions the model actually receives are untouched; only the persisted copy changes.
|
|
273
|
-
goal:
|
|
273
|
+
goal: redactEncodedText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
274
274
|
source: `study:${plan.studyId}`,
|
|
275
275
|
sourceDigest: specs[0].persona.promptDigest,
|
|
276
276
|
},
|
|
@@ -26,7 +26,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
26
26
|
const rel = path.posix.join(relPrefix, name);
|
|
27
27
|
// Evidence holds image data only, whatever the executor's frames carried.
|
|
28
28
|
const frame = stripPngMetadataChunks(bytes);
|
|
29
|
-
assertScreenshotEvidence(rel, frame);
|
|
29
|
+
await assertScreenshotEvidence(rel, frame);
|
|
30
30
|
await writeContainedOutputFile(artifactRoot, path.join(...dirParts, name), frame);
|
|
31
31
|
screenshots.push(rel);
|
|
32
32
|
return rel;
|
|
@@ -6,7 +6,7 @@ import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrate
|
|
|
6
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
7
7
|
import {} from "../../study/types.js";
|
|
8
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
9
|
-
import {
|
|
9
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
10
10
|
import {} from "../../run/bundle.js";
|
|
11
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
12
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
@@ -237,7 +237,7 @@ export function sanitizeParticipantRuns(runs, scrub) {
|
|
|
237
237
|
for (const spec of runs) {
|
|
238
238
|
if (spec.evidenceAssignment)
|
|
239
239
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrub);
|
|
240
|
-
spec.evidenceInstructions =
|
|
240
|
+
spec.evidenceInstructions = redactEncodedText(scrub(spec.instructions));
|
|
241
241
|
spec.persona = scrubPersonaBrief(spec.persona, scrub);
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
2
|
-
import {
|
|
2
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
5
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
@@ -385,7 +385,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
385
385
|
scenario: {
|
|
386
386
|
id: `cua-${args.studyId}`,
|
|
387
387
|
title: args.studyTitle ?? `Computer-use run: ${args.studyId}`,
|
|
388
|
-
goal:
|
|
388
|
+
goal: redactEncodedText(args.mission),
|
|
389
389
|
source: `study:${args.studyId}`,
|
|
390
390
|
sourceDigest: args.persona.promptDigest,
|
|
391
391
|
},
|
|
@@ -425,7 +425,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
425
425
|
participantId: args.participantId ?? "lane-01",
|
|
426
426
|
streamId: SINGLE.streamId,
|
|
427
427
|
personaId: args.persona.id,
|
|
428
|
-
goal:
|
|
428
|
+
goal: redactEncodedText(args.mission),
|
|
429
429
|
...(args.session === undefined ? {} : { session: args.session }),
|
|
430
430
|
...(args.traceArtifactPath === undefined
|
|
431
431
|
? {}
|
|
@@ -8,6 +8,7 @@ import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-com
|
|
|
8
8
|
import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
|
|
9
9
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
10
10
|
import { RunSecrets } from "../../run/secrets.js";
|
|
11
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
11
12
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
12
13
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
13
14
|
import {} from "./types.js";
|
|
@@ -55,6 +56,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
55
56
|
result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
|
|
56
57
|
};
|
|
57
58
|
}
|
|
59
|
+
const steps = recordedStepIds(scenario.journey.steps, secrets);
|
|
60
|
+
if (!steps.ok)
|
|
61
|
+
return { ok: false, result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", steps.message) };
|
|
58
62
|
// The goal, title and step labels are recorded; a step's path, selector, value and expected text
|
|
59
63
|
// drive the browser and stay as written.
|
|
60
64
|
const journey = {
|
|
@@ -62,9 +66,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
62
66
|
scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
|
|
63
67
|
? "scenario"
|
|
64
68
|
: scenario.journey.scenarioId,
|
|
65
|
-
goal:
|
|
66
|
-
scenarioTitle:
|
|
67
|
-
steps:
|
|
69
|
+
goal: recordedText(scenario.journey.goal, secrets),
|
|
70
|
+
scenarioTitle: recordedText(scenario.journey.scenarioTitle, secrets),
|
|
71
|
+
steps: steps.steps,
|
|
68
72
|
};
|
|
69
73
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
70
74
|
const { requirements } = plan;
|
|
@@ -127,6 +131,13 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
127
131
|
},
|
|
128
132
|
};
|
|
129
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Scenario text the run records, without the run's known values and then without keys as written
|
|
136
|
+
* or encoded, since verify decodes the files that record it.
|
|
137
|
+
*/
|
|
138
|
+
function recordedText(text, secrets) {
|
|
139
|
+
return redactEncodedText(secrets.scrub(text));
|
|
140
|
+
}
|
|
130
141
|
/**
|
|
131
142
|
* Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
|
|
132
143
|
* other characters as `-`, which the scrub would not find.
|
|
@@ -139,22 +150,56 @@ function idHoldsKnownValue(id, secrets) {
|
|
|
139
150
|
}));
|
|
140
151
|
}
|
|
141
152
|
/**
|
|
142
|
-
* The steps with their labels
|
|
143
|
-
* screenshot file and is recorded beside it, and the parser derives an
|
|
144
|
-
* lower-cased and cut to 80 characters, so the scrub may not find a
|
|
145
|
-
* label or id holds a known value is
|
|
153
|
+
* The steps with their labels redacted and their ids safe to record, or why the run is refused. A
|
|
154
|
+
* step id names the step's screenshot file and is recorded beside it, and the parser derives an
|
|
155
|
+
* omitted id from the label, lower-cased and cut to 80 characters, so the scrub may not find a
|
|
156
|
+
* value in it. A step whose label changes under recordedText, or whose id holds a known value, is
|
|
157
|
+
* renamed by its position instead.
|
|
146
158
|
*/
|
|
147
159
|
function recordedStepIds(steps, secrets) {
|
|
148
160
|
const taken = new Set(steps.map((step) => step.id));
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
161
|
+
const recorded = [];
|
|
162
|
+
for (const [index, step] of steps.entries()) {
|
|
163
|
+
const label = recordedText(step.label, secrets);
|
|
164
|
+
if (label === step.label && !idHoldsKnownValue(step.id, secrets)) {
|
|
165
|
+
recorded.push({ ...step, label });
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const id = positionId(index, taken, secrets);
|
|
169
|
+
if (id === undefined)
|
|
170
|
+
return {
|
|
171
|
+
ok: false,
|
|
172
|
+
message: `The label or id of scenario step ${index + 1} holds a key, a local path or a value this run keeps out of its files, so the run renames the step by its position, and each id that position gives holds one of those values too. Remove it from the step's label and id.`,
|
|
173
|
+
};
|
|
157
174
|
taken.add(id);
|
|
158
|
-
|
|
159
|
-
}
|
|
175
|
+
recorded.push({ ...step, id, label });
|
|
176
|
+
}
|
|
177
|
+
return { ok: true, steps: recorded };
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The id a renamed step takes by its position, such as step-03 for the third step, then step-03-2,
|
|
181
|
+
* step-03-3 and on when another step has it. A known value can be one of these ids, so one that
|
|
182
|
+
* holds a known value is passed over. Every id holds the position id, so a position id that holds a
|
|
183
|
+
* known value gives undefined at once. Otherwise taken.size + 1 ids that hold no known value are
|
|
184
|
+
* tried, which includes a free one, as long as known values pass over at most
|
|
185
|
+
* MAX_KNOWN_VALUE_SKIPS of them; a value such as `-01-` that every numbered id holds gives undefined.
|
|
186
|
+
*/
|
|
187
|
+
function positionId(index, taken, secrets) {
|
|
188
|
+
const position = `step-${String(index + 1).padStart(2, "0")}`;
|
|
189
|
+
if (idHoldsKnownValue(position, secrets))
|
|
190
|
+
return undefined;
|
|
191
|
+
let skips = 0;
|
|
192
|
+
for (let suffix = 1; suffix <= taken.size + 1 + skips; suffix += 1) {
|
|
193
|
+
const id = suffix === 1 ? position : `${position}-${suffix}`;
|
|
194
|
+
if (taken.has(id))
|
|
195
|
+
continue;
|
|
196
|
+
if (!idHoldsKnownValue(id, secrets))
|
|
197
|
+
return id;
|
|
198
|
+
if (skips === MAX_KNOWN_VALUE_SKIPS)
|
|
199
|
+
return undefined;
|
|
200
|
+
skips += 1;
|
|
201
|
+
}
|
|
202
|
+
return undefined;
|
|
160
203
|
}
|
|
204
|
+
// Known values are few, and a numbered id holds one only when the value matches its digits.
|
|
205
|
+
const MAX_KNOWN_VALUE_SKIPS = 32;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Builds the run bundle for a concurrent shared-world run: per-participant simulations, streams and
|
|
2
2
|
// events, the sharedWorld evidence block and the review summary.
|
|
3
3
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
4
|
-
import {
|
|
4
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
5
5
|
import { planeStateOf } from "./plan.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } from "../../run/shared-world-evidence.js";
|
|
@@ -362,7 +362,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
362
362
|
scenario: {
|
|
363
363
|
id: `concurrent-shared-world-${plan.studyId}`,
|
|
364
364
|
title: plan.title ?? `Concurrent shared-world: ${plan.studyId}`,
|
|
365
|
-
goal:
|
|
365
|
+
goal: redactEncodedText(actorSpecs[0]?.evidenceInstructions ??
|
|
366
366
|
actorSpecs[0]?.instructions ??
|
|
367
367
|
"Concurrent shared-world interaction."),
|
|
368
368
|
source: `study:${plan.studyId}`,
|
|
@@ -423,7 +423,7 @@ function sharedWorldFeedbackCandidates(args) {
|
|
|
423
423
|
participantId: spec.planned.id,
|
|
424
424
|
streamId: spec.streamId,
|
|
425
425
|
personaId: spec.persona.id,
|
|
426
|
-
goal:
|
|
426
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
427
427
|
...(outcome?.session === undefined
|
|
428
428
|
? {}
|
|
429
429
|
: { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
|
|
@@ -10,7 +10,7 @@ import { startLiveTraceFlush } from "../computer-use/live-flush.js";
|
|
|
10
10
|
import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
|
-
import {
|
|
13
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
14
14
|
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
16
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
@@ -178,7 +178,7 @@ export async function buildParticipantSpecs(participants, cwd, scrubKnownValues)
|
|
|
178
178
|
for (const spec of actorSpecs) {
|
|
179
179
|
if (spec.evidenceAssignment)
|
|
180
180
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrubKnownValues);
|
|
181
|
-
spec.evidenceInstructions =
|
|
181
|
+
spec.evidenceInstructions = redactEncodedText(scrubKnownValues(spec.instructions));
|
|
182
182
|
spec.persona = scrubPersonaBrief(spec.persona, scrubKnownValues);
|
|
183
183
|
}
|
|
184
184
|
return actorSpecs;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/judge.js";
|
|
@@ -250,7 +250,7 @@ function terminalRunBundle(args, parts) {
|
|
|
250
250
|
// The author mission is public-safe committed study text. It is recorded plaintext as the goal,
|
|
251
251
|
// redacted defensively before persisting (it never carries a secret, but the harness never
|
|
252
252
|
// trusts that). The full composed prompt is bound by digest, not text.
|
|
253
|
-
goal:
|
|
253
|
+
goal: redactEncodedText(args.mission),
|
|
254
254
|
source: `study:${args.studyId}`,
|
|
255
255
|
sourceDigest: args.persona.promptDigest,
|
|
256
256
|
},
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { ExecutionOutcome } from "./judge.js";
|
|
|
4
4
|
import type { SharedWorldEvidence } from "./shared-world-evidence.js";
|
|
5
5
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
6
6
|
import type { RunInterruptSignal } from "./status.js";
|
|
7
|
-
import type { RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
|
|
7
|
+
import type { DisplayedRunStream, RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
|
|
8
8
|
/**
|
|
9
9
|
* The run bundle version this release writes: each stream's trace, and each running participant's
|
|
10
10
|
* items so far, are files run.json indexes, and each participant's per-request lists are only in
|
|
@@ -395,6 +395,10 @@ export interface RunBundle {
|
|
|
395
395
|
export type StoredRunBundle = Omit<RunBundle, "streams"> & {
|
|
396
396
|
streams: StoredRunStream[];
|
|
397
397
|
};
|
|
398
|
+
/** A loaded bundle as a reader that shows the run has it (displayedTraces): DisplayedRunStreams. */
|
|
399
|
+
export type DisplayedRunBundle = Omit<RunBundle, "streams"> & {
|
|
400
|
+
streams: DisplayedRunStream[];
|
|
401
|
+
};
|
|
398
402
|
/**
|
|
399
403
|
* How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
|
|
400
404
|
* and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
|
package/dist/run/locate.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type PreparedRunArtifactPaths } from "./paths.js";
|
|
2
2
|
import { type ContainedRefusal, type PreparedSelectedOutputDirectory } from "./contained-output.js";
|
|
3
3
|
import type { RunPointer } from "./results.js";
|
|
4
|
-
import { type RunBundle } from "./bundle.js";
|
|
4
|
+
import { type DisplayedRunBundle, type RunBundle } from "./bundle.js";
|
|
5
5
|
import { type BoundedFileResult } from "./evidence-files.js";
|
|
6
6
|
/** The run input that resolves through latest.json rather than naming a run directory. */
|
|
7
7
|
export declare const LATEST_RUN_ALIAS = "latest";
|
|
@@ -55,15 +55,17 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
|
|
|
55
55
|
bundlePath: string;
|
|
56
56
|
runDir: string;
|
|
57
57
|
} | null>;
|
|
58
|
-
/**
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
* kept as displayedTraces shows it, for a page that shows the run.
|
|
62
|
-
*/
|
|
63
|
-
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
|
|
64
|
-
display?: boolean;
|
|
65
|
-
}): Promise<{
|
|
66
|
-
bundle: RunBundle;
|
|
58
|
+
/** A run's bundle, run.json's path relative to the working directory, and the run folder. */
|
|
59
|
+
type LoadedRun<T> = {
|
|
60
|
+
bundle: T;
|
|
67
61
|
bundlePath: string;
|
|
68
62
|
runDir: string;
|
|
69
|
-
}
|
|
63
|
+
};
|
|
64
|
+
/** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
|
|
65
|
+
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<RunBundle> | null>;
|
|
66
|
+
/**
|
|
67
|
+
* loadRunBundlePrepared for a page that shows the run: a participant whose trace the loader
|
|
68
|
+
* refuses, such as one past the read limit, is kept as displayedTraces shows it.
|
|
69
|
+
*/
|
|
70
|
+
export declare function loadDisplayedRunBundle(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<DisplayedRunBundle> | null>;
|
|
71
|
+
export {};
|
package/dist/run/locate.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDirectory, resolveLatestRunDirectory, RUNS_RELATIVE_ROOT, validatePreparedRunArtifactPaths, } from "./paths.js";
|
|
3
3
|
import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
|
|
4
|
-
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
4
|
+
import { RUN_BUNDLE_FILE, } from "./bundle.js";
|
|
5
5
|
import { isRunBundle, isRunPointer } from "./bundle-shape.js";
|
|
6
6
|
import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
|
|
7
7
|
import { readUnchangedFile } from "./evidence-files.js";
|
|
@@ -119,29 +119,36 @@ export async function loadRunBundle(cwdInput, runInput) {
|
|
|
119
119
|
}
|
|
120
120
|
return loadRunBundlePrepared(cwd, runPaths);
|
|
121
121
|
}
|
|
122
|
+
/** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
|
|
123
|
+
export async function loadRunBundlePrepared(cwdInput, runPaths) {
|
|
124
|
+
const run = await readPreparedRunJson(cwdInput, runPaths);
|
|
125
|
+
// A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
|
|
126
|
+
return run === null ? null : { ...run, bundle: await loadRunTraces(runPaths, run.bundle) };
|
|
127
|
+
}
|
|
122
128
|
/**
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
* kept as displayedTraces shows it, for a page that shows the run.
|
|
129
|
+
* loadRunBundlePrepared for a page that shows the run: a participant whose trace the loader
|
|
130
|
+
* refuses, such as one past the read limit, is kept as displayedTraces shows it.
|
|
126
131
|
*/
|
|
127
|
-
export async function
|
|
132
|
+
export async function loadDisplayedRunBundle(cwdInput, runPaths) {
|
|
133
|
+
const run = await readPreparedRunJson(cwdInput, runPaths);
|
|
134
|
+
if (run === null)
|
|
135
|
+
return null;
|
|
136
|
+
return { ...run, bundle: displayedTraces(await loadStreamTraces(runPaths, run.bundle)) };
|
|
137
|
+
}
|
|
138
|
+
/** The prepared run's run.json when it is a run bundle, after revalidating the paths; else null. */
|
|
139
|
+
async function readPreparedRunJson(cwdInput, runPaths) {
|
|
128
140
|
const cwd = path.resolve(cwdInput);
|
|
129
141
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
130
|
-
const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
|
|
131
142
|
const read = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
|
|
132
143
|
// A run.json it refuses is no reason to call the run absent: the caller reports why.
|
|
133
144
|
if (read.status === "refused")
|
|
134
145
|
throw new ContainedReadRefusedError(RUN_BUNDLE_FILE, read);
|
|
135
146
|
const bundle = read.status === "read" ? read.value : null;
|
|
136
|
-
if (!isRunBundle(bundle))
|
|
147
|
+
if (!isRunBundle(bundle))
|
|
137
148
|
return null;
|
|
138
|
-
}
|
|
139
149
|
return {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
? displayedTraces(await loadStreamTraces(runPaths, bundle))
|
|
143
|
-
: await loadRunTraces(runPaths, bundle),
|
|
144
|
-
bundlePath: path.relative(cwd, bundlePath),
|
|
150
|
+
bundle,
|
|
151
|
+
bundlePath: path.relative(cwd, path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE)),
|
|
145
152
|
runDir: runPaths.absoluteRunRoot,
|
|
146
153
|
};
|
|
147
154
|
}
|
package/dist/run/paths.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { lstat, mkdir, readdir, realpath, rmdir } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { mapWithConcurrency } from "./concurrency.js";
|
|
3
4
|
import { isNodeError } from "./type-guards.js";
|
|
4
5
|
export const RUNS_RELATIVE_ROOT = path.join(".humanish", "runs");
|
|
5
6
|
const LATEST_RUN_RELATIVE_PATH = path.join(RUNS_RELATIVE_ROOT, "latest.json");
|
|
@@ -262,35 +263,44 @@ export async function assertRegularFileOrMissing(filePath, subject = "humanish s
|
|
|
262
263
|
throw error;
|
|
263
264
|
}
|
|
264
265
|
}
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
266
|
+
// Entries the storage check lists or lstats at once. A run folder holds one entry per screenshot,
|
|
267
|
+
// about 70,000 for 24 participants working 8 hours, and one call at a time left the process idle
|
|
268
|
+
// between calls.
|
|
269
|
+
const STORAGE_CHECK_CONCURRENCY = 32;
|
|
270
|
+
/** Every entry under `root` is a directory or a single-link regular file, checked a level at a time. */
|
|
271
|
+
async function assertNoSymlinkDescendants(root) {
|
|
272
|
+
let directories = [root];
|
|
273
|
+
while (directories.length > 0) {
|
|
274
|
+
const listed = await mapWithConcurrency(directories, STORAGE_CHECK_CONCURRENCY, async (dir) => (await readdir(dir, { withFileTypes: true })).map((entry) => path.join(dir, entry.name)));
|
|
275
|
+
const children = listed.flat();
|
|
276
|
+
const kinds = await mapWithConcurrency(children, STORAGE_CHECK_CONCURRENCY, storageEntryKind);
|
|
277
|
+
directories = children.filter((_, index) => kinds[index] === "directory");
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
async function storageEntryKind(child) {
|
|
281
|
+
// A run directory can be written concurrently with this walk: the atomic writer creates a
|
|
282
|
+
// `.humanish-write-*.tmp` sibling and renames it, and a run also refreshes its own status record
|
|
283
|
+
// on a cadence. An entry that vanished between readdir and lstat therefore proves nothing except
|
|
284
|
+
// that it is gone, and a path that no longer exists cannot be a symlink escape: skip it rather
|
|
285
|
+
// than failing the containment check on a benign race.
|
|
286
|
+
let stats;
|
|
287
|
+
try {
|
|
288
|
+
stats = await lstat(child);
|
|
289
|
+
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
if (isNodeError(error) && error.code === "ENOENT")
|
|
292
|
+
return "gone";
|
|
293
|
+
throw error;
|
|
294
|
+
}
|
|
295
|
+
if (stats.isSymbolicLink()) {
|
|
296
|
+
throw new Error("humanish run directories must not contain symbolic links.");
|
|
297
|
+
}
|
|
298
|
+
if (stats.isDirectory())
|
|
299
|
+
return "directory";
|
|
300
|
+
if (!stats.isFile() || stats.nlink > 1) {
|
|
301
|
+
throw new Error("humanish run leaves must be single-link regular files, not hardlinks or special files.");
|
|
293
302
|
}
|
|
303
|
+
return "file";
|
|
294
304
|
}
|
|
295
305
|
async function capturePreparedRunArtifactPaths(paths, runRoot) {
|
|
296
306
|
const physicalRunRoot = await realpath(runRoot);
|
package/dist/run/run.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// their own evidence files; this module owns when the run starts, how it is published, and how it
|
|
4
4
|
// is closed on every exit.
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
6
7
|
import { redactText } from "../evidence/redaction.js";
|
|
7
8
|
import { buildObserverData } from "../observer/data.js";
|
|
8
9
|
import { splitObserverData } from "../observer/participant-items.js";
|
|
@@ -160,14 +161,14 @@ function runPublisher(args) {
|
|
|
160
161
|
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
161
162
|
const observer = observerTarget(options);
|
|
162
163
|
// The study's warnings join whatever events the route wrote, once, in every write. A warning can
|
|
163
|
-
// quote the study file, which may name a sandbox URL a participant needs
|
|
164
|
-
// the redaction
|
|
164
|
+
// quote the study file, which may name a sandbox URL a participant needs or hold an encoded key,
|
|
165
|
+
// so each goes through the redaction study text gets.
|
|
165
166
|
const studyWarnings = (options.warnings ?? []).map((message, index) => ({
|
|
166
167
|
id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
|
|
167
168
|
at: createdAt,
|
|
168
169
|
level: "warn",
|
|
169
170
|
type: "study.warning",
|
|
170
|
-
message:
|
|
171
|
+
message: redactEncodedText(message),
|
|
171
172
|
}));
|
|
172
173
|
// So do the host suspensions the heartbeat has seen by the time of the write, and the files the
|
|
173
174
|
// final sandbox id sweep could not read.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type RunBundle, type StoredRunBundle } from "./bundle.js";
|
|
1
|
+
import { type DisplayedRunBundle, type RunBundle, type StoredRunBundle } from "./bundle.js";
|
|
2
2
|
import { type PreparedOutputRoot } from "./contained-output.js";
|
|
3
3
|
import { type LiveItemsCursor } from "./live-items.js";
|
|
4
4
|
import type { LiveItemsIndex, RunStream, StoredRunStream, WithoutRequestLists } from "./streams.js";
|
|
@@ -131,7 +131,7 @@ export declare function displayedTraces(loaded: {
|
|
|
131
131
|
bundle: RunBundle;
|
|
132
132
|
refused: readonly StreamTraceRefusedError[];
|
|
133
133
|
live?: readonly LiveItemsRead[];
|
|
134
|
-
}):
|
|
134
|
+
}): DisplayedRunBundle;
|
|
135
135
|
/**
|
|
136
136
|
* What `reduce` returns for each stream of the bundle, loaded as loadRunTraces loads it, in stream
|
|
137
137
|
* order; the first refusal throws. Streams are loaded one at a time and nothing of a stream's files
|
package/dist/run/stream-trace.js
CHANGED
|
@@ -178,7 +178,7 @@ function containedReader(runPaths) {
|
|
|
178
178
|
}
|
|
179
179
|
/**
|
|
180
180
|
* The stream from the bytes of its trace file: its actor with the file's items, and its artifacts
|
|
181
|
-
* followed by the file's screenshot entries, without the index.
|
|
181
|
+
* followed by the file's screenshot entries, without the index. Its live actor is as it was.
|
|
182
182
|
*/
|
|
183
183
|
function joinTrace(stream, bytes) {
|
|
184
184
|
const index = stream.trace;
|
|
@@ -246,12 +246,19 @@ function assertIndex(stream) {
|
|
|
246
246
|
throw new StreamTraceRefusedError(stream.id, "run.json holds items beside its trace index");
|
|
247
247
|
}
|
|
248
248
|
/**
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
* the
|
|
249
|
+
* The joined stream as a loaded one, once its actor's conversation, if any, holds the requests
|
|
250
|
+
* list ActorTrace types as present. The writer writes the list with every conversation, in
|
|
251
|
+
* run.json before v4 and in the trace file from v4, so a stream without it was edited, and a reader
|
|
252
|
+
* that counts the requests would throw. Items a v1 run.json or a humanish.live-actor.v1 live actor
|
|
253
|
+
* holds inline are taken as they are.
|
|
253
254
|
*/
|
|
254
|
-
|
|
255
|
+
function asLoaded(stream) {
|
|
256
|
+
const conversation = stream.actor?.conversation;
|
|
257
|
+
if (conversation !== undefined &&
|
|
258
|
+
!(isRecord(conversation) && Array.isArray(conversation.requests)))
|
|
259
|
+
throw new StreamTraceRefusedError(stream.id, "its actor records a conversation without the requests list");
|
|
260
|
+
return stream;
|
|
261
|
+
}
|
|
255
262
|
/** A live actor whose items run.json holds inline: humanish.live-actor.v1, or one with no schema. */
|
|
256
263
|
const isInlineLiveActor = (live) => live.schema === LIVE_ACTOR_V1_SCHEMA || live.schema === undefined;
|
|
257
264
|
/** A live file that could not be read, which a writer that moved on may have removed. */
|
|
@@ -345,7 +352,7 @@ async function readLiveTail(read, stream, index, after) {
|
|
|
345
352
|
*/
|
|
346
353
|
async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
347
354
|
const files = [];
|
|
348
|
-
let joined =
|
|
355
|
+
let joined = stream;
|
|
349
356
|
if (stream.trace !== undefined) {
|
|
350
357
|
assertIndex(stream);
|
|
351
358
|
if (stream.trace.bytes > maxBytes)
|
|
@@ -359,7 +366,11 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
|
359
366
|
const live = await readLiveItems(read, joined, maxBytes, liveAfter);
|
|
360
367
|
if (live.file !== undefined)
|
|
361
368
|
files.push(live.file);
|
|
362
|
-
return {
|
|
369
|
+
return {
|
|
370
|
+
stream: asLoaded(live.stream),
|
|
371
|
+
files,
|
|
372
|
+
...(live.read === undefined ? {} : { live: live.read }),
|
|
373
|
+
};
|
|
363
374
|
}
|
|
364
375
|
/**
|
|
365
376
|
* The stream as a v1 run.json held it: its actor with every item and every artifact entry. A
|
|
@@ -500,7 +511,9 @@ async function loadEveryStream(read, bundle, maxBytes, liveAfter) {
|
|
|
500
511
|
if (!(error instanceof StreamTraceRefusedError))
|
|
501
512
|
throw error;
|
|
502
513
|
refused.push(error);
|
|
503
|
-
|
|
514
|
+
// Kept as run.json holds it, typed as loaded: from v2 its actor has no items, and from v4
|
|
515
|
+
// none of the lists. The loader's `refused` names it, and displayedTraces shows it so.
|
|
516
|
+
streams.push(stream);
|
|
504
517
|
}
|
|
505
518
|
}
|
|
506
519
|
return { bundle: { ...bundle, streams }, refused, files, live };
|
|
@@ -523,12 +536,12 @@ export function displayedTraces(loaded) {
|
|
|
523
536
|
if (reason === undefined)
|
|
524
537
|
return stream;
|
|
525
538
|
const { liveActor: _live, trace: _index, actor, ...rest } = stream;
|
|
526
|
-
return
|
|
539
|
+
return {
|
|
527
540
|
...rest,
|
|
528
541
|
...(actor === undefined ? {} : { actor: { ...withoutRequestLists(actor), items: [] } }),
|
|
529
542
|
artifacts: stream.artifacts.filter((artifact) => !isScreenshot(artifact)),
|
|
530
543
|
traceUnavailable: reason,
|
|
531
|
-
}
|
|
544
|
+
};
|
|
532
545
|
});
|
|
533
546
|
const events = loaded.bundle.streams
|
|
534
547
|
.filter((stream) => reasons.has(stream.id))
|
package/dist/run/streams.d.ts
CHANGED
|
@@ -327,11 +327,26 @@ export type WithoutRequestLists<T> = Omit<T, "providerRequests" | "tokenUsage" |
|
|
|
327
327
|
[K in "conversation" & keyof T]?: Omit<NonNullable<T[K]>, "requests"> | undefined;
|
|
328
328
|
};
|
|
329
329
|
/**
|
|
330
|
-
*
|
|
331
|
-
*
|
|
330
|
+
* `T` WithoutRequestLists and with `items` optional: a v1 run.json holds the items inline, and from
|
|
331
|
+
* v2 (an actor) or v3 (a live actor) a file holds them.
|
|
332
|
+
*/
|
|
333
|
+
type StoredTrace<T extends {
|
|
334
|
+
items: unknown;
|
|
335
|
+
}> = WithoutRequestLists<Omit<T, "items">> & Partial<Pick<T, "items">>;
|
|
336
|
+
/**
|
|
337
|
+
* A stream as run.json holds it, in any version: its actor and live actor without the per-request
|
|
338
|
+
* lists, and with items only in a v1 run.json. loadStreamTraces joins both from the files.
|
|
332
339
|
*/
|
|
333
340
|
export type StoredRunStream = Omit<RunStream, "actor" | "liveActor"> & {
|
|
341
|
+
actor?: StoredTrace<ActorTrace>;
|
|
342
|
+
liveActor?: StoredTrace<NonNullable<RunStream["liveActor"]>>;
|
|
343
|
+
};
|
|
344
|
+
/**
|
|
345
|
+
* A stream as a reader that shows the run has it (displayedTraces in src/run/stream-trace.ts):
|
|
346
|
+
* loaded, or with `traceUnavailable` when the loader refused its trace. A refused stream's actor
|
|
347
|
+
* keeps run.json's summary, which from v4 has none of the per-request lists, so the type has none.
|
|
348
|
+
*/
|
|
349
|
+
export type DisplayedRunStream = Omit<RunStream, "actor"> & {
|
|
334
350
|
actor?: WithoutRequestLists<ActorTrace>;
|
|
335
|
-
liveActor?: WithoutRequestLists<NonNullable<RunStream["liveActor"]>>;
|
|
336
351
|
};
|
|
337
352
|
export {};
|
package/dist/run-study.d.ts
CHANGED
|
@@ -21,9 +21,9 @@ export declare function runStudyWith(config: StudyConfig, options: InternalRunSt
|
|
|
21
21
|
/**
|
|
22
22
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
23
23
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
24
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
25
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
26
|
-
* anything runs.
|
|
24
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
25
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
26
|
+
* route's own result envelope before anything runs.
|
|
27
27
|
*/
|
|
28
28
|
export declare function runStudy(config: StudyConfig, options: RunStudyOptions): Promise<StudyOutcome>;
|
|
29
29
|
/** A study planned once: the route's refusal, or the run of its plan. */
|