humanish 0.134.0 → 0.135.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/analysis/automatic-config.js +5 -3
- package/dist/analysis/execute.js +2 -1
- package/dist/analysis/service.js +2 -1
- package/dist/evidence/encoded-text.d.ts +13 -3
- package/dist/evidence/encoded-text.js +88 -25
- package/dist/observer/page-data.d.ts +6 -0
- package/dist/observer/page-data.js +34 -7
- package/dist/observer/participant-items.d.ts +15 -1
- package/dist/observer/participant-items.js +4 -3
- package/dist/observer/run-routes.js +4 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -3
- package/dist/routes/computer-use/participant-runs.js +2 -2
- package/dist/routes/computer-use/single-bundle.js +3 -3
- package/dist/routes/scripted/setup.js +62 -17
- package/dist/routes/shared-world/bundle.js +3 -3
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/terminal/bundle.js +2 -2
- package/dist/run/run.js +4 -3
- package/dist/run-study.d.ts +3 -3
- package/dist/run-study.js +3 -3
- package/dist/study/bundle-estimate.d.ts +0 -2
- package/dist/study/bundle-estimate.js +10 -16
- package/dist/study/config.d.ts +4 -3
- package/dist/study/config.js +8 -5
- package/dist/study/parse/actors.d.ts +6 -0
- package/dist/study/parse/actors.js +14 -0
- package/dist/study/parse/subject-state.js +4 -4
- package/dist/study/participant-assignment.js +2 -2
- package/dist/study/persona.d.ts +7 -1
- package/dist/study/persona.js +13 -5
- package/dist/study/plan-base.js +4 -3
- package/docs/architecture/observer.md +24 -7
- package/docs/contracts/run-bundle.md +3 -0
- package/docs/contracts/study-analysis.md +4 -3
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@ import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
|
2
2
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
3
3
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
4
4
|
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
5
|
-
import {
|
|
5
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
8
8
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
|
|
@@ -186,7 +186,7 @@ function fanoutFeedbackCandidates(args) {
|
|
|
186
186
|
participantId: spec.planned.id,
|
|
187
187
|
streamId: spec.streamId,
|
|
188
188
|
personaId: spec.persona.id,
|
|
189
|
-
goal:
|
|
189
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
190
190
|
...(outcome?.session === undefined ? {} : { session: outcome.session }),
|
|
191
191
|
...(outcome?.session === undefined
|
|
192
192
|
? {}
|
|
@@ -270,7 +270,7 @@ export function buildCuaFanoutBundle(args) {
|
|
|
270
270
|
// this writer produced. The only adopter-side workaround was scanner evasion.
|
|
271
271
|
//
|
|
272
272
|
// The instructions the model actually receives are untouched; only the persisted copy changes.
|
|
273
|
-
goal:
|
|
273
|
+
goal: redactEncodedText(specs[0].evidenceInstructions ?? specs[0].instructions),
|
|
274
274
|
source: `study:${plan.studyId}`,
|
|
275
275
|
sourceDigest: specs[0].persona.promptDigest,
|
|
276
276
|
},
|
|
@@ -6,7 +6,7 @@ import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrate
|
|
|
6
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
7
7
|
import {} from "../../study/types.js";
|
|
8
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
9
|
-
import {
|
|
9
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
10
10
|
import {} from "../../run/bundle.js";
|
|
11
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
12
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
@@ -237,7 +237,7 @@ export function sanitizeParticipantRuns(runs, scrub) {
|
|
|
237
237
|
for (const spec of runs) {
|
|
238
238
|
if (spec.evidenceAssignment)
|
|
239
239
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrub);
|
|
240
|
-
spec.evidenceInstructions =
|
|
240
|
+
spec.evidenceInstructions = redactEncodedText(scrub(spec.instructions));
|
|
241
241
|
spec.persona = scrubPersonaBrief(spec.persona, scrub);
|
|
242
242
|
}
|
|
243
243
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
2
|
-
import {
|
|
2
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/streams.js";
|
|
5
5
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
@@ -385,7 +385,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
385
385
|
scenario: {
|
|
386
386
|
id: `cua-${args.studyId}`,
|
|
387
387
|
title: args.studyTitle ?? `Computer-use run: ${args.studyId}`,
|
|
388
|
-
goal:
|
|
388
|
+
goal: redactEncodedText(args.mission),
|
|
389
389
|
source: `study:${args.studyId}`,
|
|
390
390
|
sourceDigest: args.persona.promptDigest,
|
|
391
391
|
},
|
|
@@ -425,7 +425,7 @@ export function buildSingleParticipantBundle(args) {
|
|
|
425
425
|
participantId: args.participantId ?? "lane-01",
|
|
426
426
|
streamId: SINGLE.streamId,
|
|
427
427
|
personaId: args.persona.id,
|
|
428
|
-
goal:
|
|
428
|
+
goal: redactEncodedText(args.mission),
|
|
429
429
|
...(args.session === undefined ? {} : { session: args.session }),
|
|
430
430
|
...(args.traceArtifactPath === undefined
|
|
431
431
|
? {}
|
|
@@ -8,6 +8,7 @@ import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-com
|
|
|
8
8
|
import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
|
|
9
9
|
import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
10
10
|
import { RunSecrets } from "../../run/secrets.js";
|
|
11
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
11
12
|
import { evidenceAppUrlOf } from "./plan.js";
|
|
12
13
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
13
14
|
import {} from "./types.js";
|
|
@@ -55,6 +56,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
55
56
|
result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
|
|
56
57
|
};
|
|
57
58
|
}
|
|
59
|
+
const steps = recordedStepIds(scenario.journey.steps, secrets);
|
|
60
|
+
if (!steps.ok)
|
|
61
|
+
return { ok: false, result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", steps.message) };
|
|
58
62
|
// The goal, title and step labels are recorded; a step's path, selector, value and expected text
|
|
59
63
|
// drive the browser and stay as written.
|
|
60
64
|
const journey = {
|
|
@@ -62,9 +66,9 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
62
66
|
scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
|
|
63
67
|
? "scenario"
|
|
64
68
|
: scenario.journey.scenarioId,
|
|
65
|
-
goal:
|
|
66
|
-
scenarioTitle:
|
|
67
|
-
steps:
|
|
69
|
+
goal: recordedText(scenario.journey.goal, secrets),
|
|
70
|
+
scenarioTitle: recordedText(scenario.journey.scenarioTitle, secrets),
|
|
71
|
+
steps: steps.steps,
|
|
68
72
|
};
|
|
69
73
|
// The plan lists E2B_API_KEY and the subject env only for a live clone.
|
|
70
74
|
const { requirements } = plan;
|
|
@@ -127,6 +131,13 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
127
131
|
},
|
|
128
132
|
};
|
|
129
133
|
}
|
|
134
|
+
/**
|
|
135
|
+
* Scenario text the run records, without the run's known values and then without keys as written
|
|
136
|
+
* or encoded, since verify decodes the files that record it.
|
|
137
|
+
*/
|
|
138
|
+
function recordedText(text, secrets) {
|
|
139
|
+
return redactEncodedText(secrets.scrub(text));
|
|
140
|
+
}
|
|
130
141
|
/**
|
|
131
142
|
* Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
|
|
132
143
|
* other characters as `-`, which the scrub would not find.
|
|
@@ -139,22 +150,56 @@ function idHoldsKnownValue(id, secrets) {
|
|
|
139
150
|
}));
|
|
140
151
|
}
|
|
141
152
|
/**
|
|
142
|
-
* The steps with their labels
|
|
143
|
-
* screenshot file and is recorded beside it, and the parser derives an
|
|
144
|
-
* lower-cased and cut to 80 characters, so the scrub may not find a
|
|
145
|
-
* label or id holds a known value is
|
|
153
|
+
* The steps with their labels redacted and their ids safe to record, or why the run is refused. A
|
|
154
|
+
* step id names the step's screenshot file and is recorded beside it, and the parser derives an
|
|
155
|
+
* omitted id from the label, lower-cased and cut to 80 characters, so the scrub may not find a
|
|
156
|
+
* value in it. A step whose label changes under recordedText, or whose id holds a known value, is
|
|
157
|
+
* renamed by its position instead.
|
|
146
158
|
*/
|
|
147
159
|
function recordedStepIds(steps, secrets) {
|
|
148
160
|
const taken = new Set(steps.map((step) => step.id));
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
161
|
+
const recorded = [];
|
|
162
|
+
for (const [index, step] of steps.entries()) {
|
|
163
|
+
const label = recordedText(step.label, secrets);
|
|
164
|
+
if (label === step.label && !idHoldsKnownValue(step.id, secrets)) {
|
|
165
|
+
recorded.push({ ...step, label });
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
const id = positionId(index, taken, secrets);
|
|
169
|
+
if (id === undefined)
|
|
170
|
+
return {
|
|
171
|
+
ok: false,
|
|
172
|
+
message: `The label or id of scenario step ${index + 1} holds a key, a local path or a value this run keeps out of its files, so the run renames the step by its position, and each id that position gives holds one of those values too. Remove it from the step's label and id.`,
|
|
173
|
+
};
|
|
157
174
|
taken.add(id);
|
|
158
|
-
|
|
159
|
-
}
|
|
175
|
+
recorded.push({ ...step, id, label });
|
|
176
|
+
}
|
|
177
|
+
return { ok: true, steps: recorded };
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The id a renamed step takes by its position, such as step-03 for the third step, then step-03-2,
|
|
181
|
+
* step-03-3 and on when another step has it. A known value can be one of these ids, so one that
|
|
182
|
+
* holds a known value is passed over. Every id holds the position id, so a position id that holds a
|
|
183
|
+
* known value gives undefined at once. Otherwise taken.size + 1 ids that hold no known value are
|
|
184
|
+
* tried, which includes a free one, as long as known values pass over at most
|
|
185
|
+
* MAX_KNOWN_VALUE_SKIPS of them; a value such as `-01-` that every numbered id holds gives undefined.
|
|
186
|
+
*/
|
|
187
|
+
function positionId(index, taken, secrets) {
|
|
188
|
+
const position = `step-${String(index + 1).padStart(2, "0")}`;
|
|
189
|
+
if (idHoldsKnownValue(position, secrets))
|
|
190
|
+
return undefined;
|
|
191
|
+
let skips = 0;
|
|
192
|
+
for (let suffix = 1; suffix <= taken.size + 1 + skips; suffix += 1) {
|
|
193
|
+
const id = suffix === 1 ? position : `${position}-${suffix}`;
|
|
194
|
+
if (taken.has(id))
|
|
195
|
+
continue;
|
|
196
|
+
if (!idHoldsKnownValue(id, secrets))
|
|
197
|
+
return id;
|
|
198
|
+
if (skips === MAX_KNOWN_VALUE_SKIPS)
|
|
199
|
+
return undefined;
|
|
200
|
+
skips += 1;
|
|
201
|
+
}
|
|
202
|
+
return undefined;
|
|
160
203
|
}
|
|
204
|
+
// Known values are few, and a numbered id holds one only when the value matches its digits.
|
|
205
|
+
const MAX_KNOWN_VALUE_SKIPS = 32;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Builds the run bundle for a concurrent shared-world run: per-participant simulations, streams and
|
|
2
2
|
// events, the sharedWorld evidence block and the review summary.
|
|
3
3
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
4
|
-
import {
|
|
4
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
5
5
|
import { planeStateOf } from "./plan.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } from "../../run/shared-world-evidence.js";
|
|
@@ -362,7 +362,7 @@ export function buildConcurrentSharedWorldBundle(args) {
|
|
|
362
362
|
scenario: {
|
|
363
363
|
id: `concurrent-shared-world-${plan.studyId}`,
|
|
364
364
|
title: plan.title ?? `Concurrent shared-world: ${plan.studyId}`,
|
|
365
|
-
goal:
|
|
365
|
+
goal: redactEncodedText(actorSpecs[0]?.evidenceInstructions ??
|
|
366
366
|
actorSpecs[0]?.instructions ??
|
|
367
367
|
"Concurrent shared-world interaction."),
|
|
368
368
|
source: `study:${plan.studyId}`,
|
|
@@ -423,7 +423,7 @@ function sharedWorldFeedbackCandidates(args) {
|
|
|
423
423
|
participantId: spec.planned.id,
|
|
424
424
|
streamId: spec.streamId,
|
|
425
425
|
personaId: spec.persona.id,
|
|
426
|
-
goal:
|
|
426
|
+
goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
|
|
427
427
|
...(outcome?.session === undefined
|
|
428
428
|
? {}
|
|
429
429
|
: { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
|
|
@@ -10,7 +10,7 @@ import { startLiveTraceFlush } from "../computer-use/live-flush.js";
|
|
|
10
10
|
import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
|
-
import {
|
|
13
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
14
14
|
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
16
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
@@ -178,7 +178,7 @@ export async function buildParticipantSpecs(participants, cwd, scrubKnownValues)
|
|
|
178
178
|
for (const spec of actorSpecs) {
|
|
179
179
|
if (spec.evidenceAssignment)
|
|
180
180
|
spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrubKnownValues);
|
|
181
|
-
spec.evidenceInstructions =
|
|
181
|
+
spec.evidenceInstructions = redactEncodedText(scrubKnownValues(spec.instructions));
|
|
182
182
|
spec.persona = scrubPersonaBrief(spec.persona, scrubKnownValues);
|
|
183
183
|
}
|
|
184
184
|
return actorSpecs;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import {} from "../../run/judge.js";
|
|
@@ -250,7 +250,7 @@ function terminalRunBundle(args, parts) {
|
|
|
250
250
|
// The author mission is public-safe committed study text. It is recorded plaintext as the goal,
|
|
251
251
|
// redacted defensively before persisting (it never carries a secret, but the harness never
|
|
252
252
|
// trusts that). The full composed prompt is bound by digest, not text.
|
|
253
|
-
goal:
|
|
253
|
+
goal: redactEncodedText(args.mission),
|
|
254
254
|
source: `study:${args.studyId}`,
|
|
255
255
|
sourceDigest: args.persona.promptDigest,
|
|
256
256
|
},
|
package/dist/run/run.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// their own evidence files; this module owns when the run starts, how it is published, and how it
|
|
4
4
|
// is closed on every exit.
|
|
5
5
|
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
6
7
|
import { redactText } from "../evidence/redaction.js";
|
|
7
8
|
import { buildObserverData } from "../observer/data.js";
|
|
8
9
|
import { splitObserverData } from "../observer/participant-items.js";
|
|
@@ -160,14 +161,14 @@ function runPublisher(args) {
|
|
|
160
161
|
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
161
162
|
const observer = observerTarget(options);
|
|
162
163
|
// The study's warnings join whatever events the route wrote, once, in every write. A warning can
|
|
163
|
-
// quote the study file, which may name a sandbox URL a participant needs
|
|
164
|
-
// the redaction
|
|
164
|
+
// quote the study file, which may name a sandbox URL a participant needs or hold an encoded key,
|
|
165
|
+
// so each goes through the redaction study text gets.
|
|
165
166
|
const studyWarnings = (options.warnings ?? []).map((message, index) => ({
|
|
166
167
|
id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
|
|
167
168
|
at: createdAt,
|
|
168
169
|
level: "warn",
|
|
169
170
|
type: "study.warning",
|
|
170
|
-
message:
|
|
171
|
+
message: redactEncodedText(message),
|
|
171
172
|
}));
|
|
172
173
|
// So do the host suspensions the heartbeat has seen by the time of the write, and the files the
|
|
173
174
|
// final sandbox id sweep could not read.
|
package/dist/run-study.d.ts
CHANGED
|
@@ -21,9 +21,9 @@ export declare function runStudyWith(config: StudyConfig, options: InternalRunSt
|
|
|
21
21
|
/**
|
|
22
22
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
23
23
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
24
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
25
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
26
|
-
* anything runs.
|
|
24
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
25
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
26
|
+
* route's own result envelope before anything runs.
|
|
27
27
|
*/
|
|
28
28
|
export declare function runStudy(config: StudyConfig, options: RunStudyOptions): Promise<StudyOutcome>;
|
|
29
29
|
/** A study planned once: the route's refusal, or the run of its plan. */
|
package/dist/run-study.js
CHANGED
|
@@ -34,9 +34,9 @@ export async function runStudyWith(config, options, deps = {}) {
|
|
|
34
34
|
/**
|
|
35
35
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
36
36
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
37
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
38
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
39
|
-
* anything runs.
|
|
37
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
38
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
39
|
+
* route's own result envelope before anything runs.
|
|
40
40
|
*/
|
|
41
41
|
export async function runStudy(config, options) {
|
|
42
42
|
const rejected = libraryConfigRefusal(config);
|
|
@@ -16,8 +16,6 @@ export interface BundleEstimate {
|
|
|
16
16
|
readonly runFileBytes: number;
|
|
17
17
|
/** The largest participant trace file, which analysis also reads whole. */
|
|
18
18
|
readonly largestTraceBytes: number;
|
|
19
|
-
/** Every participant's trace file together, whose items the Observer page joins in one string. */
|
|
20
|
-
readonly tracesBytes: number;
|
|
21
19
|
/** observer/index.html, which verify reads whole. */
|
|
22
20
|
readonly observerPageBytes: number;
|
|
23
21
|
/** Every file, screenshots included. */
|
|
@@ -16,13 +16,14 @@
|
|
|
16
16
|
// observer-data.json leaves the lists out too and was 0.94 to 1.12 of run.json's size on the live
|
|
17
17
|
// runs measured, so it passes 32 MiB only after run.json passes 16 MiB, and the estimate does not
|
|
18
18
|
// check it.
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
|
|
19
|
+
// A page past 32 MiB grades the run local_only. humanish export then needs --local-only, and a
|
|
20
|
+
// bundle export fails even with it, since its copy must verify as share_ready.
|
|
21
|
+
// The Observer feed, verify, analysis and the run.json writer each hold one participant's items in
|
|
22
|
+
// a string at a time. The page holds all of them gzipped in base64, about 0.18 characters for each
|
|
23
|
+
// trace byte, so it reaches V8's longest string, 536,870,888 characters, near 3 GB of traces.
|
|
24
|
+
// humanish observe on 25 participants with 30 MiB traces wrote a 138 MiB page at a peak of
|
|
25
|
+
// 3.8 GiB, so memory may run out first. Neither limit was measured past that run, and the estimate
|
|
26
|
+
// checks neither.
|
|
26
27
|
import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
|
|
27
28
|
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
28
29
|
import { formatDuration } from "../run/projection.js";
|
|
@@ -169,7 +170,6 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
169
170
|
return {
|
|
170
171
|
runFileBytes,
|
|
171
172
|
largestTraceBytes: largestTrace,
|
|
172
|
-
tracesBytes: traces,
|
|
173
173
|
observerPageBytes,
|
|
174
174
|
// observer-data.json is about run.json's size.
|
|
175
175
|
totalBytes: observerPageBytes + screenshots * RATES.screenshotBytes + traces + 2 * runFileBytes,
|
|
@@ -192,7 +192,6 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
192
192
|
export function bundleLimitReason(estimate) {
|
|
193
193
|
const runFile = estimate.runFileBytes > EVIDENCE_LIMITS.sourceBytes;
|
|
194
194
|
const page = estimate.observerPageBytes > RUN_ARTIFACT_MAX_BYTES;
|
|
195
|
-
const joined = estimate.tracesBytes > constants.MAX_STRING_LENGTH;
|
|
196
195
|
const entries = estimate.entries > estimate.entryLimit;
|
|
197
196
|
const passed = [
|
|
198
197
|
...(runFile
|
|
@@ -207,12 +206,7 @@ export function bundleLimitReason(estimate) {
|
|
|
207
206
|
: []),
|
|
208
207
|
...(page
|
|
209
208
|
? [
|
|
210
|
-
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file.`,
|
|
211
|
-
]
|
|
212
|
-
: []),
|
|
213
|
-
...(joined
|
|
214
|
-
? [
|
|
215
|
-
`All participants' traces together may grow to about ${mib(estimate.tracesBytes)}, past the ${mib(constants.MAX_STRING_LENGTH)} that fits in one string. The Observer page puts all their items in one string, so the page would not build. A bundle export allowed past its 25 MiB default renders that page, so it would fail too.`,
|
|
209
|
+
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file, so verify would grade the run local_only: humanish export would refuse it without --local-only, and a bundle export would fail.`,
|
|
216
210
|
]
|
|
217
211
|
: []),
|
|
218
212
|
...(entries
|
|
@@ -242,7 +236,7 @@ export function bundleLimitReason(estimate) {
|
|
|
242
236
|
...(runFile ? ["shorten the mission (actor.mission)"] : []),
|
|
243
237
|
];
|
|
244
238
|
// A trace is one participant's, so splitting the study helps only the run's own files.
|
|
245
|
-
if (runFile || page ||
|
|
239
|
+
if (runFile || page || entries || changes.length === 0)
|
|
246
240
|
changes.push("split the participants across studies");
|
|
247
241
|
const advice = changes.length < 2
|
|
248
242
|
? changes.join("")
|
package/dist/study/config.d.ts
CHANGED
|
@@ -30,9 +30,10 @@ interface ConfigRefusal {
|
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
32
|
* Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
|
|
33
|
-
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object,
|
|
34
|
-
* whose `route` is not the route its subject and actor take,
|
|
35
|
-
* reads only the v3 fields, so a budget left in
|
|
33
|
+
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
|
|
34
|
+
* whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
|
|
35
|
+
* would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
|
|
36
|
+
* `execution.caps` would otherwise run uncapped, and passes the model on as written.
|
|
36
37
|
*/
|
|
37
38
|
export declare function libraryConfigRefusal(config: StudyConfig): ConfigRefusal | undefined;
|
|
38
39
|
export {};
|
package/dist/study/config.js
CHANGED
|
@@ -30,7 +30,7 @@ import { isLocalBrowserStudy, localBrowserDefaults, localBrowserUnsupportedReaso
|
|
|
30
30
|
import { isRecord } from "../run/type-guards.js";
|
|
31
31
|
import { compositionReason } from "./composition-rules.js";
|
|
32
32
|
import { V2_SCHEMA, V2_UNSUPPORTED_MESSAGE, v2FieldMoves } from "./migrate/v2.js";
|
|
33
|
-
import { parseActorFields, parseActorType, parseParticipantEntries } from "./parse/actors.js";
|
|
33
|
+
import { actorModelReason, parseActorFields, parseActorType, parseParticipantEntries, } from "./parse/actors.js";
|
|
34
34
|
import { parseComms, recipientParticipantId } from "./parse/comms.js";
|
|
35
35
|
import { parseCaps, parseDefaults, parseExecution, parsePolicies, parseReview, } from "./parse/execution.js";
|
|
36
36
|
import { ACTOR_REQUIRED, readStudyFront, routeNameReason, } from "./parse/front.js";
|
|
@@ -96,9 +96,10 @@ function routeMismatchReason(config) {
|
|
|
96
96
|
}
|
|
97
97
|
/**
|
|
98
98
|
* Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
|
|
99
|
-
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object,
|
|
100
|
-
* whose `route` is not the route its subject and actor take,
|
|
101
|
-
* reads only the v3 fields, so a budget left in
|
|
99
|
+
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
|
|
100
|
+
* whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
|
|
101
|
+
* would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
|
|
102
|
+
* `execution.caps` would otherwise run uncapped, and passes the model on as written.
|
|
102
103
|
*/
|
|
103
104
|
export function libraryConfigRefusal(config) {
|
|
104
105
|
const raw = config;
|
|
@@ -114,7 +115,9 @@ export function libraryConfigRefusal(config) {
|
|
|
114
115
|
};
|
|
115
116
|
if (!isRecord(raw.actor))
|
|
116
117
|
return { code: "HUMANISH_STUDY_INVALID", message: ACTOR_REQUIRED };
|
|
117
|
-
const message = routeNameReason(raw.route) ??
|
|
118
|
+
const message = routeNameReason(raw.route) ??
|
|
119
|
+
routeMismatchReason(config) ??
|
|
120
|
+
actorModelReason(raw.actor.model, "actor");
|
|
118
121
|
return message === undefined ? undefined : { code: "HUMANISH_STUDY_INVALID", message };
|
|
119
122
|
}
|
|
120
123
|
// Each section in a fixed order, so a file with more than one error always reports the same one
|
|
@@ -6,6 +6,12 @@ export declare function parseActorType(entry: Record<string, unknown>, path: str
|
|
|
6
6
|
ok: true;
|
|
7
7
|
value: string;
|
|
8
8
|
} | StudyParseFailure;
|
|
9
|
+
/**
|
|
10
|
+
* Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
|
|
11
|
+
* records the name, so redacting it would ask for another model. Text verify would flag, as written
|
|
12
|
+
* or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
|
|
13
|
+
*/
|
|
14
|
+
export declare function actorModelReason(raw: unknown, path: string): string | undefined;
|
|
9
15
|
/** An actor's keys other than `type`. */
|
|
10
16
|
type ActorFields = Omit<StudyActor, "type">;
|
|
11
17
|
/**
|
|
@@ -51,6 +51,17 @@ function parsePersonaId(raw, path) {
|
|
|
51
51
|
}
|
|
52
52
|
return { ok: true, value: persona };
|
|
53
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
|
|
56
|
+
* records the name, so redacting it would ask for another model. Text verify would flag, as written
|
|
57
|
+
* or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
|
|
58
|
+
*/
|
|
59
|
+
export function actorModelReason(raw, path) {
|
|
60
|
+
const model = str(raw);
|
|
61
|
+
if (model === undefined || !scanEncodedText(model).sensitive)
|
|
62
|
+
return undefined;
|
|
63
|
+
return `\`${path}.model\` looks like a secret or a local path, and the run records it. Set the model id your provider lists, or leave \`${path}.model\` out to use the default.`;
|
|
64
|
+
}
|
|
54
65
|
/**
|
|
55
66
|
* Every actor key other than `type`. The parser reads them after the participant entries, so a file
|
|
56
67
|
* with errors in both reports the entry first.
|
|
@@ -65,6 +76,9 @@ export function parseActorFields(entry, path) {
|
|
|
65
76
|
const mission = str(entry.mission);
|
|
66
77
|
if (mission)
|
|
67
78
|
actor.mission = mission;
|
|
79
|
+
const modelReason = actorModelReason(entry.model, path);
|
|
80
|
+
if (modelReason)
|
|
81
|
+
return invalid(modelReason);
|
|
68
82
|
const model = str(entry.model);
|
|
69
83
|
if (model)
|
|
70
84
|
actor.model = model;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// The state and env channels of a served subject (clone or local-tree): `subject.envValues`,
|
|
2
2
|
// the structural parse of `subject.state`, and subjectStateInvalidReason, the semantic check
|
|
3
3
|
// parseStudy and the computer-use route share.
|
|
4
|
-
import {
|
|
4
|
+
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
5
5
|
import { ENV_NAME_PATTERN, invalid, posInt, str, strList } from "./values.js";
|
|
6
6
|
import { isRecord } from "../../run/type-guards.js";
|
|
7
7
|
/**
|
|
@@ -29,9 +29,9 @@ export function parseEnvValues(raw) {
|
|
|
29
29
|
if (value === undefined) {
|
|
30
30
|
return invalid(`\`subject.envValues.${name}\` must be a string, number, or boolean.`);
|
|
31
31
|
}
|
|
32
|
-
//
|
|
33
|
-
// a secret
|
|
34
|
-
if (
|
|
32
|
+
// verify blocks a run whose files scanEncodedText reads as sensitive, so this refuses what that
|
|
33
|
+
// scan reads: a secret or local path as written, or decoded from base64 or an escape.
|
|
34
|
+
if (scanEncodedText(value).sensitive) {
|
|
35
35
|
return invalid(`\`subject.envValues.${name}\` looks like a secret or a local path, and envValues are committed with the study and recorded in evidence. List the name in \`subject.env\` instead: its value then comes from your environment and is never stored.`);
|
|
36
36
|
}
|
|
37
37
|
envValues[name] = value;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
2
2
|
/** Project only authored participant-facing fields. Never pass runtime-composed instructions:
|
|
3
3
|
* they can contain inbox URLs or multiplayer grants. Pick fields explicitly so a task's hidden
|
|
4
4
|
* success criteria cannot cross this evidence boundary, even from an untyped library caller. */
|
|
5
5
|
export function participantAssignment(assignment, scrubKnownValues = (text) => text) {
|
|
6
|
-
const sanitize = (text) =>
|
|
6
|
+
const sanitize = (text) => redactEncodedText(scrubKnownValues(text));
|
|
7
7
|
return {
|
|
8
8
|
mission: sanitize(assignment.mission),
|
|
9
9
|
...(assignment.focus === undefined ? {} : { focus: sanitize(assignment.focus) }),
|
package/dist/study/persona.d.ts
CHANGED
|
@@ -37,7 +37,13 @@ export declare function parseResolvedPersona(raw: unknown, fallback: {
|
|
|
37
37
|
}, warnings?: string[]): ResolvedPersona;
|
|
38
38
|
export declare function personaToDirectives(persona: ResolvedPersona): PersonaDirectives;
|
|
39
39
|
export declare function renderPersonaPromptSection(persona: ResolvedPersona): string;
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
|
|
42
|
+
* `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
|
|
43
|
+
* keys as written come out. A route that learns its known values later passes this brief to
|
|
44
|
+
* scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
|
|
45
|
+
* that scrub could no longer find the value whole.
|
|
46
|
+
*/
|
|
41
47
|
export declare function personaBrief(persona: ResolvedPersona, scrub?: (text: string) => string): NonNullable<ActorPersonaRef["brief"]>;
|
|
42
48
|
export declare function scrubPersonaBrief(persona: ActorPersonaRef, scrub: (text: string) => string): ActorPersonaRef;
|
|
43
49
|
export {};
|
package/dist/study/persona.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
1
2
|
import { digestText, redactText } from "../evidence/redaction.js";
|
|
2
3
|
import { isRecord } from "../run/type-guards.js";
|
|
3
4
|
export const PERSONA_BACKGROUND_MAX_BYTES = 32 * 1024;
|
|
@@ -136,8 +137,9 @@ export function personaToDirectives(persona) {
|
|
|
136
137
|
...(patience ? [`patience:${patience}`] : []),
|
|
137
138
|
...(skill ? [`skill:${skill}`] : []),
|
|
138
139
|
];
|
|
140
|
+
// A live participant's trace records traitsApplied as it is, and the need is free text.
|
|
139
141
|
if (accessibilityNeeds)
|
|
140
|
-
traitsApplied.push(`accessibility:${accessibilityNeeds}`);
|
|
142
|
+
traitsApplied.push(`accessibility:${redactEncodedText(accessibilityNeeds)}`);
|
|
141
143
|
if (persona.constraints.length)
|
|
142
144
|
traitsApplied.push(`constraints:${persona.constraints.length}`);
|
|
143
145
|
return {
|
|
@@ -166,9 +168,15 @@ export function renderPersonaPromptSection(persona) {
|
|
|
166
168
|
.filter(Boolean)
|
|
167
169
|
.join("\n\n");
|
|
168
170
|
}
|
|
169
|
-
/**
|
|
170
|
-
|
|
171
|
-
|
|
171
|
+
/**
|
|
172
|
+
* Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
|
|
173
|
+
* `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
|
|
174
|
+
* keys as written come out. A route that learns its known values later passes this brief to
|
|
175
|
+
* scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
|
|
176
|
+
* that scrub could no longer find the value whole.
|
|
177
|
+
*/
|
|
178
|
+
export function personaBrief(persona, scrub) {
|
|
179
|
+
const original = renderPersonaPromptSection(persona), text = scrub === undefined ? redactText(original) : redactEncodedText(scrub(original));
|
|
172
180
|
return {
|
|
173
181
|
compilerVersion: PERSONA_COMPILER_VERSION,
|
|
174
182
|
text,
|
|
@@ -180,7 +188,7 @@ export function personaBrief(persona, scrub = (text) => text) {
|
|
|
180
188
|
export function scrubPersonaBrief(persona, scrub) {
|
|
181
189
|
if (!persona.brief)
|
|
182
190
|
return persona;
|
|
183
|
-
const text =
|
|
191
|
+
const text = redactEncodedText(scrub(persona.brief.text));
|
|
184
192
|
return {
|
|
185
193
|
...persona,
|
|
186
194
|
brief: {
|
package/dist/study/plan-base.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// What every route planner shares: the plan fields common to all routes, the frozen residual config,
|
|
2
2
|
// and the requirement and subject helpers.
|
|
3
3
|
import { DEFAULT_OPENAI_CU_MODEL } from "../actors/computer-use/openai-provider.js";
|
|
4
|
-
import {
|
|
4
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
5
5
|
// A YAML alias can make a parsed record contain itself (an inline persona that names its own
|
|
6
6
|
// anchor), and structuredClone keeps the cycle, so each object is frozen once.
|
|
7
7
|
function deepFreeze(value, frozen = new WeakSet()) {
|
|
@@ -112,8 +112,9 @@ export function planBase(config, input) {
|
|
|
112
112
|
return {
|
|
113
113
|
studyId: config.id,
|
|
114
114
|
// Every bundle writer takes the title from here, and review.md, run.json and the Observer show
|
|
115
|
-
// it, so a key or sandbox URL in it would make verify grade the run
|
|
116
|
-
|
|
115
|
+
// it, so a key or sandbox URL in it, as written or encoded, would make verify grade the run
|
|
116
|
+
// blocked.
|
|
117
|
+
...(config.title === undefined ? {} : { title: redactEncodedText(config.title) }),
|
|
117
118
|
residual: residualOf(config),
|
|
118
119
|
dryRun: input.dryRun,
|
|
119
120
|
...(analysis === undefined
|