humanish 0.134.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/actors/computer-use/loop/session.js +7 -2
- package/dist/actors/scripted-browser/actor.js +3 -3
- package/dist/actors/scripted-browser/steps.d.ts +9 -2
- package/dist/actors/scripted-browser/steps.js +17 -12
- package/dist/analysis/automatic-config.js +5 -3
- package/dist/analysis/evidence-sources.js +4 -7
- package/dist/analysis/evidence.js +4 -4
- package/dist/analysis/execute.js +2 -1
- package/dist/analysis/service.js +2 -1
- package/dist/analysis/source.d.ts +7 -4
- package/dist/analysis/source.js +22 -4
- package/dist/cli/commands/setup.js +7 -10
- package/dist/evidence/encoded-text.d.ts +13 -3
- package/dist/evidence/encoded-text.js +88 -25
- package/dist/evidence/image.d.ts +2 -2
- package/dist/evidence/image.js +47 -32
- package/dist/observer/data.d.ts +7 -3
- package/dist/observer/page-data.d.ts +6 -0
- package/dist/observer/page-data.js +34 -7
- package/dist/observer/participant-items.d.ts +15 -1
- package/dist/observer/participant-items.js +4 -3
- package/dist/observer/render.js +2 -4
- package/dist/observer/run-routes.js +4 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -3
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-runs.js +2 -2
- package/dist/routes/computer-use/single-bundle.js +3 -3
- package/dist/routes/scripted/setup.js +62 -17
- package/dist/routes/shared-world/bundle.js +3 -3
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/terminal/bundle.js +2 -2
- package/dist/run/bundle.d.ts +5 -1
- package/dist/run/locate.d.ts +13 -11
- package/dist/run/locate.js +20 -13
- package/dist/run/paths.js +38 -28
- package/dist/run/run.js +4 -3
- package/dist/run/stream-trace.d.ts +2 -2
- package/dist/run/stream-trace.js +24 -11
- package/dist/run/streams.d.ts +18 -3
- package/dist/run-study.d.ts +3 -3
- package/dist/run-study.js +3 -3
- package/dist/study/bundle-estimate.d.ts +21 -3
- package/dist/study/bundle-estimate.js +65 -45
- package/dist/study/config.d.ts +4 -3
- package/dist/study/config.js +8 -5
- package/dist/study/parse/actors.d.ts +6 -0
- package/dist/study/parse/actors.js +14 -0
- package/dist/study/parse/subject-state.js +4 -4
- package/dist/study/participant-assignment.js +2 -2
- package/dist/study/persona.d.ts +7 -1
- package/dist/study/persona.js +13 -5
- package/dist/study/plan-base.js +4 -3
- package/dist/tui-app.js +23 -23
- package/dist/verify/screenshot-reads.js +1 -1
- package/docs/architecture/observer.md +24 -7
- package/docs/architecture/state-driven-executor.md +4 -3
- package/docs/contracts/run-bundle.md +3 -0
- package/docs/contracts/study-analysis.md +7 -5
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
- package/package.json +1 -1
package/dist/run-study.js
CHANGED
|
@@ -34,9 +34,9 @@ export async function runStudyWith(config, options, deps = {}) {
|
|
|
34
34
|
/**
|
|
35
35
|
* The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
|
|
36
36
|
* config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
|
|
37
|
-
* or one with a humanish 0.110 field, no `actor`,
|
|
38
|
-
* field RunStudyOptions no longer has, is refused in the
|
|
39
|
-
* anything runs.
|
|
37
|
+
* or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
|
|
38
|
+
* `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
|
|
39
|
+
* route's own result envelope before anything runs.
|
|
40
40
|
*/
|
|
41
41
|
export async function runStudy(config, options) {
|
|
42
42
|
const rejected = libraryConfigRefusal(config);
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { DwellWindow } from "../actors/stop-conditions.js";
|
|
2
2
|
import type { ParticipantSchedule } from "./arrivals.js";
|
|
3
|
-
/**
|
|
3
|
+
/**
|
|
4
|
+
* One participant as the plan has it: its schedule, its dwell window, its mission and instruction,
|
|
5
|
+
* and its task goals.
|
|
6
|
+
*/
|
|
4
7
|
export interface EstimatedParticipant {
|
|
5
8
|
readonly schedule: ParticipantSchedule;
|
|
6
9
|
readonly limits: {
|
|
@@ -8,16 +11,28 @@ export interface EstimatedParticipant {
|
|
|
8
11
|
};
|
|
9
12
|
readonly assignment: {
|
|
10
13
|
readonly mission?: string;
|
|
14
|
+
readonly focus?: string;
|
|
11
15
|
};
|
|
16
|
+
readonly tasks?: readonly {
|
|
17
|
+
readonly goal: string;
|
|
18
|
+
}[];
|
|
12
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* The study fields run.json copies into each participant's stream, and the advice to shorten each.
|
|
22
|
+
* The plan does not keep which form declared the participants, so the advice names both paths.
|
|
23
|
+
*/
|
|
24
|
+
declare const COPIED_FIELDS: Readonly<{
|
|
25
|
+
mission: "shorten the mission (actor.mission)";
|
|
26
|
+
focus: "shorten the participants' instructions (participants.instruction or participants[].instruction)";
|
|
27
|
+
tasks: "shorten the task goals (actor.tasks[].goal)";
|
|
28
|
+
}>;
|
|
29
|
+
type CopiedField = keyof typeof COPIED_FIELDS;
|
|
13
30
|
/** A run's estimated bundle, and the scan's limit for its participants. */
|
|
14
31
|
export interface BundleEstimate {
|
|
15
32
|
/** run.json, which analysis reads whole. */
|
|
16
33
|
readonly runFileBytes: number;
|
|
17
34
|
/** The largest participant trace file, which analysis also reads whole. */
|
|
18
35
|
readonly largestTraceBytes: number;
|
|
19
|
-
/** Every participant's trace file together, whose items the Observer page joins in one string. */
|
|
20
|
-
readonly tracesBytes: number;
|
|
21
36
|
/** observer/index.html, which verify reads whole. */
|
|
22
37
|
readonly observerPageBytes: number;
|
|
23
38
|
/** Every file, screenshots included. */
|
|
@@ -36,6 +51,8 @@ export interface BundleEstimate {
|
|
|
36
51
|
readonly open: number;
|
|
37
52
|
readonly openMs: number;
|
|
38
53
|
readonly dwell: boolean;
|
|
54
|
+
/** The copied field with the most bytes over all participants, when one has any. */
|
|
55
|
+
readonly largestField?: CopiedField;
|
|
39
56
|
};
|
|
40
57
|
}
|
|
41
58
|
/**
|
|
@@ -53,3 +70,4 @@ export declare function estimateBundle(participants: readonly EstimatedParticipa
|
|
|
53
70
|
export declare function bundleLimitReason(estimate: BundleEstimate): string | undefined;
|
|
54
71
|
/** The estimate beside the limits, for `study check`. */
|
|
55
72
|
export declare function describeBundle(estimate: BundleEstimate): string;
|
|
73
|
+
export {};
|
|
@@ -7,22 +7,27 @@
|
|
|
7
7
|
// observer/index.html carries every participant's items gzipped, so with busy participants it
|
|
8
8
|
// passes verify's 32 MiB long before run.json passes analysis's 16 MiB: at these rates, 20
|
|
9
9
|
// participants working 8 hours. From run bundle v4 run.json grows by participant, not by model
|
|
10
|
-
// request: each participant's per-request lists are only in its trace. run.json holds
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
10
|
+
// request: each participant's per-request lists are only in its trace. run.json holds each
|
|
11
|
+
// participant's text, its mission, its instruction and its task goals, in its stream and again in
|
|
12
|
+
// the feedback candidate of a participant that reports friction or stops, and the first
|
|
13
|
+
// participant's once more in the scenario goal. A synthetic live run of 3 participants that all
|
|
14
|
+
// reported friction held their instructions 3, 2 and 2 times, and the mission and each of 2 task
|
|
15
|
+
// goals 7 times. None of the three fields is capped, and the estimate counts the feedback
|
|
16
|
+
// candidate's copy for every participant, so it warns at 100 participants with 75,238 bytes of
|
|
17
|
+
// text each. On 2026-10-13 the longest mission on the live computer-use runs kept on the
|
|
18
|
+
// maintainer's machine was 807 characters. On 2026-10-14 the longest instruction on the 170 live
|
|
19
|
+
// runs kept there was 4,061 characters, and none declared task goals.
|
|
16
20
|
// observer-data.json leaves the lists out too and was 0.94 to 1.12 of run.json's size on the live
|
|
17
21
|
// runs measured, so it passes 32 MiB only after run.json passes 16 MiB, and the estimate does not
|
|
18
22
|
// check it.
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
|
|
23
|
+
// A page past 32 MiB grades the run local_only. humanish export then needs --local-only, and a
|
|
24
|
+
// bundle export fails even with it, since its copy must verify as share_ready.
|
|
25
|
+
// The Observer feed, verify, analysis and the run.json writer each hold one participant's items in
|
|
26
|
+
// a string at a time. The page holds all of them gzipped in base64, about 0.18 characters for each
|
|
27
|
+
// trace byte, so it reaches V8's longest string, 536,870,888 characters, near 3 GB of traces.
|
|
28
|
+
// humanish observe on 25 participants with 30 MiB traces wrote a 138 MiB page at a peak of
|
|
29
|
+
// 3.8 GiB, so memory may run out first. Neither limit was measured past that run, and the estimate
|
|
30
|
+
// checks neither.
|
|
26
31
|
import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
|
|
27
32
|
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
28
33
|
import { formatDuration } from "../run/projection.js";
|
|
@@ -82,10 +87,11 @@ const RATES = Object.freeze({
|
|
|
82
87
|
/**
|
|
83
88
|
* The page's compact copy of observer-data.json, as a share of run.json before the mission: 0.71
|
|
84
89
|
* to 0.84 on the live runs written as v4, where neither carries the per-request lists. The page
|
|
85
|
-
* holds
|
|
86
|
-
* 1 and 4 participants grew the page by 2 and 5 copies
|
|
87
|
-
* participants with 2 feedback candidates held 5 copies in
|
|
88
|
-
*
|
|
90
|
+
* holds each participant's text at its full size once, and the first participant's once more for
|
|
91
|
+
* the run: dry runs of 1 and 4 participants grew the page by 2 and 5 copies of the mission, as
|
|
92
|
+
* run.json grew, and a 0.128.0 run of 2 participants with 2 feedback candidates held 5 copies in
|
|
93
|
+
* run.json and 3 on the page. The synthetic run of 3 held their instructions 2, 1 and 1 times on
|
|
94
|
+
* the page. An analysis adds each participant's assignment, cut to 8,000 bytes.
|
|
89
95
|
*/
|
|
90
96
|
observerPageDataShare: 0.85,
|
|
91
97
|
/**
|
|
@@ -106,6 +112,15 @@ const RATES = Object.freeze({
|
|
|
106
112
|
participantEntries: 3,
|
|
107
113
|
});
|
|
108
114
|
const MIB = 1024 * 1024;
|
|
115
|
+
/**
|
|
116
|
+
* The study fields run.json copies into each participant's stream, and the advice to shorten each.
|
|
117
|
+
* The plan does not keep which form declared the participants, so the advice names both paths.
|
|
118
|
+
*/
|
|
119
|
+
const COPIED_FIELDS = Object.freeze({
|
|
120
|
+
mission: "shorten the mission (actor.mission)",
|
|
121
|
+
focus: "shorten the participants' instructions (participants.instruction or participants[].instruction)",
|
|
122
|
+
tasks: "shorten the task goals (actor.tasks[].goal)",
|
|
123
|
+
});
|
|
109
124
|
/**
|
|
110
125
|
* The bundle a live run of `participants` leaves, each in its session of `sessionMs` unless it
|
|
111
126
|
* declares an end or a cap. A participant with an end works its whole window; one without ends
|
|
@@ -118,13 +133,14 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
118
133
|
let frames = 0;
|
|
119
134
|
let traces = 0;
|
|
120
135
|
let largestTrace = 0;
|
|
121
|
-
|
|
122
|
-
|
|
136
|
+
// The bytes of each copied field over all participants, and the longest participant's text.
|
|
137
|
+
const fieldBytes = { mission: 0, focus: 0, tasks: 0 };
|
|
138
|
+
let longestText = 0;
|
|
123
139
|
let longestWindowMs = 0;
|
|
124
140
|
// The longest any participant without an end is counted, for the messages.
|
|
125
141
|
let openMs = Math.min(sessionMs, RATES.openSessionMs);
|
|
126
142
|
let scheduled = 0;
|
|
127
|
-
for (const { schedule, limits, assignment } of participants) {
|
|
143
|
+
for (const { schedule, limits, assignment, tasks = [] } of participants) {
|
|
128
144
|
const { end } = schedule;
|
|
129
145
|
const budgetMs = end?.windowMs ?? sessionMs;
|
|
130
146
|
let countedMs;
|
|
@@ -145,31 +161,36 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
145
161
|
frames += ownFrames;
|
|
146
162
|
traces += trace;
|
|
147
163
|
largestTrace = Math.max(largestTrace, trace);
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
164
|
+
const mission = storedBytes(assignment.mission);
|
|
165
|
+
const focus = storedBytes(assignment.focus);
|
|
166
|
+
const goals = tasks.reduce((sum, task) => sum + storedBytes(task.goal), 0);
|
|
167
|
+
fieldBytes.mission += mission;
|
|
168
|
+
fieldBytes.focus += focus;
|
|
169
|
+
fieldBytes.tasks += goals;
|
|
170
|
+
longestText = Math.max(longestText, mission + focus + goals);
|
|
154
171
|
}
|
|
172
|
+
const textBytes = fieldBytes.mission + fieldBytes.focus + fieldBytes.tasks;
|
|
173
|
+
// Sorting is stable, so of fields with equal bytes the first declared is named.
|
|
174
|
+
const [largestField] = Object.keys(fieldBytes)
|
|
175
|
+
.filter((field) => fieldBytes[field] > 0)
|
|
176
|
+
.sort((a, b) => fieldBytes[b] - fieldBytes[a]);
|
|
155
177
|
const count = participants.length;
|
|
156
178
|
const dataBytes = RATES.runBytes + count * RATES.participantBytes;
|
|
157
|
-
// run.json holds
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
// twice.
|
|
161
|
-
const runFileBytes = dataBytes +
|
|
179
|
+
// run.json holds each participant's text in its stream and in its feedback candidate if it
|
|
180
|
+
// reports friction or stops, counted for every participant, and the first participant's in the
|
|
181
|
+
// run's scenario goal, counted at the longest. The rates were fit on runs with short texts, so a
|
|
182
|
+
// short text is counted twice.
|
|
183
|
+
const runFileBytes = dataBytes + longestText + 2 * textBytes;
|
|
162
184
|
const screenshots = count + turns + frames;
|
|
163
185
|
// The page holds no feedback candidate's copy.
|
|
164
186
|
const observerPageBytes = Math.round(RATES.observerAppBytes +
|
|
165
187
|
RATES.observerPageDataShare * dataBytes +
|
|
166
|
-
|
|
167
|
-
|
|
188
|
+
longestText +
|
|
189
|
+
textBytes +
|
|
168
190
|
RATES.observerPageItemsShare * traces);
|
|
169
191
|
return {
|
|
170
192
|
runFileBytes,
|
|
171
193
|
largestTraceBytes: largestTrace,
|
|
172
|
-
tracesBytes: traces,
|
|
173
194
|
observerPageBytes,
|
|
174
195
|
// observer-data.json is about run.json's size.
|
|
175
196
|
totalBytes: observerPageBytes + screenshots * RATES.screenshotBytes + traces + 2 * runFileBytes,
|
|
@@ -182,9 +203,14 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
182
203
|
open: count - scheduled,
|
|
183
204
|
openMs,
|
|
184
205
|
dwell: frames > 0,
|
|
206
|
+
...(largestField === undefined ? {} : { largestField }),
|
|
185
207
|
},
|
|
186
208
|
};
|
|
187
209
|
}
|
|
210
|
+
/** A text's bytes as run.json stores it, a JSON string without its quotes. */
|
|
211
|
+
function storedBytes(text) {
|
|
212
|
+
return text === undefined ? 0 : Buffer.byteLength(JSON.stringify(text)) - 2;
|
|
213
|
+
}
|
|
188
214
|
/**
|
|
189
215
|
* The warning for a run whose estimate passes a limit, or undefined: each limit it passes, what the
|
|
190
216
|
* estimate counted, and what to change.
|
|
@@ -192,12 +218,11 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
192
218
|
export function bundleLimitReason(estimate) {
|
|
193
219
|
const runFile = estimate.runFileBytes > EVIDENCE_LIMITS.sourceBytes;
|
|
194
220
|
const page = estimate.observerPageBytes > RUN_ARTIFACT_MAX_BYTES;
|
|
195
|
-
const joined = estimate.tracesBytes > constants.MAX_STRING_LENGTH;
|
|
196
221
|
const entries = estimate.entries > estimate.entryLimit;
|
|
197
222
|
const passed = [
|
|
198
223
|
...(runFile
|
|
199
224
|
? [
|
|
200
|
-
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads. It holds the mission
|
|
225
|
+
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads. It holds the mission, the task goals and each participant's instruction in that participant's stream, and again in the feedback candidate of each participant that reports friction or stops. It holds the first participant's mission, instruction and task goals once more for the run, and the estimate counts a feedback candidate for every participant.`,
|
|
201
226
|
]
|
|
202
227
|
: []),
|
|
203
228
|
...(estimate.largestTraceBytes > RUN_ARTIFACT_MAX_BYTES
|
|
@@ -207,12 +232,7 @@ export function bundleLimitReason(estimate) {
|
|
|
207
232
|
: []),
|
|
208
233
|
...(page
|
|
209
234
|
? [
|
|
210
|
-
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file.`,
|
|
211
|
-
]
|
|
212
|
-
: []),
|
|
213
|
-
...(joined
|
|
214
|
-
? [
|
|
215
|
-
`All participants' traces together may grow to about ${mib(estimate.tracesBytes)}, past the ${mib(constants.MAX_STRING_LENGTH)} that fits in one string. The Observer page puts all their items in one string, so the page would not build. A bundle export allowed past its 25 MiB default renders that page, so it would fail too.`,
|
|
235
|
+
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file, so verify would grade the run local_only: humanish export would refuse it without --local-only, and a bundle export would fail.`,
|
|
216
236
|
]
|
|
217
237
|
: []),
|
|
218
238
|
...(entries
|
|
@@ -223,7 +243,7 @@ export function bundleLimitReason(estimate) {
|
|
|
223
243
|
];
|
|
224
244
|
if (passed.length === 0)
|
|
225
245
|
return undefined;
|
|
226
|
-
const { scheduled, longestWindowMs, open, openMs, dwell } = estimate.basis;
|
|
246
|
+
const { scheduled, longestWindowMs, open, openMs, dwell, largestField } = estimate.basis;
|
|
227
247
|
const counted = [
|
|
228
248
|
...(scheduled > 0
|
|
229
249
|
? [
|
|
@@ -239,10 +259,10 @@ export function bundleLimitReason(estimate) {
|
|
|
239
259
|
const changes = [
|
|
240
260
|
...(scheduled > 0 ? ["shorten the longest windows (participants[].endAfterMs)"] : []),
|
|
241
261
|
...(dwell ? ["shorten the dwell window (dwell.ms) or capture less often (dwell.everyMs)"] : []),
|
|
242
|
-
...(runFile
|
|
262
|
+
...(runFile && largestField !== undefined ? [COPIED_FIELDS[largestField]] : []),
|
|
243
263
|
];
|
|
244
264
|
// A trace is one participant's, so splitting the study helps only the run's own files.
|
|
245
|
-
if (runFile || page ||
|
|
265
|
+
if (runFile || page || entries || changes.length === 0)
|
|
246
266
|
changes.push("split the participants across studies");
|
|
247
267
|
const advice = changes.length < 2
|
|
248
268
|
? changes.join("")
|
package/dist/study/config.d.ts
CHANGED
|
@@ -30,9 +30,10 @@ interface ConfigRefusal {
|
|
|
30
30
|
}
|
|
31
31
|
/**
|
|
32
32
|
* Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
|
|
33
|
-
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object,
|
|
34
|
-
* whose `route` is not the route its subject and actor take,
|
|
35
|
-
* reads only the v3 fields, so a budget left in
|
|
33
|
+
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
|
|
34
|
+
* whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
|
|
35
|
+
* would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
|
|
36
|
+
* `execution.caps` would otherwise run uncapped, and passes the model on as written.
|
|
36
37
|
*/
|
|
37
38
|
export declare function libraryConfigRefusal(config: StudyConfig): ConfigRefusal | undefined;
|
|
38
39
|
export {};
|
package/dist/study/config.js
CHANGED
|
@@ -30,7 +30,7 @@ import { isLocalBrowserStudy, localBrowserDefaults, localBrowserUnsupportedReaso
|
|
|
30
30
|
import { isRecord } from "../run/type-guards.js";
|
|
31
31
|
import { compositionReason } from "./composition-rules.js";
|
|
32
32
|
import { V2_SCHEMA, V2_UNSUPPORTED_MESSAGE, v2FieldMoves } from "./migrate/v2.js";
|
|
33
|
-
import { parseActorFields, parseActorType, parseParticipantEntries } from "./parse/actors.js";
|
|
33
|
+
import { actorModelReason, parseActorFields, parseActorType, parseParticipantEntries, } from "./parse/actors.js";
|
|
34
34
|
import { parseComms, recipientParticipantId } from "./parse/comms.js";
|
|
35
35
|
import { parseCaps, parseDefaults, parseExecution, parsePolicies, parseReview, } from "./parse/execution.js";
|
|
36
36
|
import { ACTOR_REQUIRED, readStudyFront, routeNameReason, } from "./parse/front.js";
|
|
@@ -96,9 +96,10 @@ function routeMismatchReason(config) {
|
|
|
96
96
|
}
|
|
97
97
|
/**
|
|
98
98
|
* Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
|
|
99
|
-
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object,
|
|
100
|
-
* whose `route` is not the route its subject and actor take,
|
|
101
|
-
* reads only the v3 fields, so a budget left in
|
|
99
|
+
* one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
|
|
100
|
+
* whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
|
|
101
|
+
* would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
|
|
102
|
+
* `execution.caps` would otherwise run uncapped, and passes the model on as written.
|
|
102
103
|
*/
|
|
103
104
|
export function libraryConfigRefusal(config) {
|
|
104
105
|
const raw = config;
|
|
@@ -114,7 +115,9 @@ export function libraryConfigRefusal(config) {
|
|
|
114
115
|
};
|
|
115
116
|
if (!isRecord(raw.actor))
|
|
116
117
|
return { code: "HUMANISH_STUDY_INVALID", message: ACTOR_REQUIRED };
|
|
117
|
-
const message = routeNameReason(raw.route) ??
|
|
118
|
+
const message = routeNameReason(raw.route) ??
|
|
119
|
+
routeMismatchReason(config) ??
|
|
120
|
+
actorModelReason(raw.actor.model, "actor");
|
|
118
121
|
return message === undefined ? undefined : { code: "HUMANISH_STUDY_INVALID", message };
|
|
119
122
|
}
|
|
120
123
|
// Each section in a fixed order, so a file with more than one error always reports the same one
|
|
@@ -6,6 +6,12 @@ export declare function parseActorType(entry: Record<string, unknown>, path: str
|
|
|
6
6
|
ok: true;
|
|
7
7
|
value: string;
|
|
8
8
|
} | StudyParseFailure;
|
|
9
|
+
/**
|
|
10
|
+
* Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
|
|
11
|
+
* records the name, so redacting it would ask for another model. Text verify would flag, as written
|
|
12
|
+
* or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
|
|
13
|
+
*/
|
|
14
|
+
export declare function actorModelReason(raw: unknown, path: string): string | undefined;
|
|
9
15
|
/** An actor's keys other than `type`. */
|
|
10
16
|
type ActorFields = Omit<StudyActor, "type">;
|
|
11
17
|
/**
|
|
@@ -51,6 +51,17 @@ function parsePersonaId(raw, path) {
|
|
|
51
51
|
}
|
|
52
52
|
return { ok: true, value: persona };
|
|
53
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
|
|
56
|
+
* records the name, so redacting it would ask for another model. Text verify would flag, as written
|
|
57
|
+
* or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
|
|
58
|
+
*/
|
|
59
|
+
export function actorModelReason(raw, path) {
|
|
60
|
+
const model = str(raw);
|
|
61
|
+
if (model === undefined || !scanEncodedText(model).sensitive)
|
|
62
|
+
return undefined;
|
|
63
|
+
return `\`${path}.model\` looks like a secret or a local path, and the run records it. Set the model id your provider lists, or leave \`${path}.model\` out to use the default.`;
|
|
64
|
+
}
|
|
54
65
|
/**
|
|
55
66
|
* Every actor key other than `type`. The parser reads them after the participant entries, so a file
|
|
56
67
|
* with errors in both reports the entry first.
|
|
@@ -65,6 +76,9 @@ export function parseActorFields(entry, path) {
|
|
|
65
76
|
const mission = str(entry.mission);
|
|
66
77
|
if (mission)
|
|
67
78
|
actor.mission = mission;
|
|
79
|
+
const modelReason = actorModelReason(entry.model, path);
|
|
80
|
+
if (modelReason)
|
|
81
|
+
return invalid(modelReason);
|
|
68
82
|
const model = str(entry.model);
|
|
69
83
|
if (model)
|
|
70
84
|
actor.model = model;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// The state and env channels of a served subject (clone or local-tree): `subject.envValues`,
|
|
2
2
|
// the structural parse of `subject.state`, and subjectStateInvalidReason, the semantic check
|
|
3
3
|
// parseStudy and the computer-use route share.
|
|
4
|
-
import {
|
|
4
|
+
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
5
5
|
import { ENV_NAME_PATTERN, invalid, posInt, str, strList } from "./values.js";
|
|
6
6
|
import { isRecord } from "../../run/type-guards.js";
|
|
7
7
|
/**
|
|
@@ -29,9 +29,9 @@ export function parseEnvValues(raw) {
|
|
|
29
29
|
if (value === undefined) {
|
|
30
30
|
return invalid(`\`subject.envValues.${name}\` must be a string, number, or boolean.`);
|
|
31
31
|
}
|
|
32
|
-
//
|
|
33
|
-
// a secret
|
|
34
|
-
if (
|
|
32
|
+
// verify blocks a run whose files scanEncodedText reads as sensitive, so this refuses what that
|
|
33
|
+
// scan reads: a secret or local path as written, or decoded from base64 or an escape.
|
|
34
|
+
if (scanEncodedText(value).sensitive) {
|
|
35
35
|
return invalid(`\`subject.envValues.${name}\` looks like a secret or a local path, and envValues are committed with the study and recorded in evidence. List the name in \`subject.env\` instead: its value then comes from your environment and is never stored.`);
|
|
36
36
|
}
|
|
37
37
|
envValues[name] = value;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
2
2
|
/** Project only authored participant-facing fields. Never pass runtime-composed instructions:
|
|
3
3
|
* they can contain inbox URLs or multiplayer grants. Pick fields explicitly so a task's hidden
|
|
4
4
|
* success criteria cannot cross this evidence boundary, even from an untyped library caller. */
|
|
5
5
|
export function participantAssignment(assignment, scrubKnownValues = (text) => text) {
|
|
6
|
-
const sanitize = (text) =>
|
|
6
|
+
const sanitize = (text) => redactEncodedText(scrubKnownValues(text));
|
|
7
7
|
return {
|
|
8
8
|
mission: sanitize(assignment.mission),
|
|
9
9
|
...(assignment.focus === undefined ? {} : { focus: sanitize(assignment.focus) }),
|
package/dist/study/persona.d.ts
CHANGED
|
@@ -37,7 +37,13 @@ export declare function parseResolvedPersona(raw: unknown, fallback: {
|
|
|
37
37
|
}, warnings?: string[]): ResolvedPersona;
|
|
38
38
|
export declare function personaToDirectives(persona: ResolvedPersona): PersonaDirectives;
|
|
39
39
|
export declare function renderPersonaPromptSection(persona: ResolvedPersona): string;
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
|
|
42
|
+
* `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
|
|
43
|
+
* keys as written come out. A route that learns its known values later passes this brief to
|
|
44
|
+
* scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
|
|
45
|
+
* that scrub could no longer find the value whole.
|
|
46
|
+
*/
|
|
41
47
|
export declare function personaBrief(persona: ResolvedPersona, scrub?: (text: string) => string): NonNullable<ActorPersonaRef["brief"]>;
|
|
42
48
|
export declare function scrubPersonaBrief(persona: ActorPersonaRef, scrub: (text: string) => string): ActorPersonaRef;
|
|
43
49
|
export {};
|
package/dist/study/persona.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
1
2
|
import { digestText, redactText } from "../evidence/redaction.js";
|
|
2
3
|
import { isRecord } from "../run/type-guards.js";
|
|
3
4
|
export const PERSONA_BACKGROUND_MAX_BYTES = 32 * 1024;
|
|
@@ -136,8 +137,9 @@ export function personaToDirectives(persona) {
|
|
|
136
137
|
...(patience ? [`patience:${patience}`] : []),
|
|
137
138
|
...(skill ? [`skill:${skill}`] : []),
|
|
138
139
|
];
|
|
140
|
+
// A live participant's trace records traitsApplied as it is, and the need is free text.
|
|
139
141
|
if (accessibilityNeeds)
|
|
140
|
-
traitsApplied.push(`accessibility:${accessibilityNeeds}`);
|
|
142
|
+
traitsApplied.push(`accessibility:${redactEncodedText(accessibilityNeeds)}`);
|
|
141
143
|
if (persona.constraints.length)
|
|
142
144
|
traitsApplied.push(`constraints:${persona.constraints.length}`);
|
|
143
145
|
return {
|
|
@@ -166,9 +168,15 @@ export function renderPersonaPromptSection(persona) {
|
|
|
166
168
|
.filter(Boolean)
|
|
167
169
|
.join("\n\n");
|
|
168
170
|
}
|
|
169
|
-
/**
|
|
170
|
-
|
|
171
|
-
|
|
171
|
+
/**
|
|
172
|
+
* Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
|
|
173
|
+
* `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
|
|
174
|
+
* keys as written come out. A route that learns its known values later passes this brief to
|
|
175
|
+
* scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
|
|
176
|
+
* that scrub could no longer find the value whole.
|
|
177
|
+
*/
|
|
178
|
+
export function personaBrief(persona, scrub) {
|
|
179
|
+
const original = renderPersonaPromptSection(persona), text = scrub === undefined ? redactText(original) : redactEncodedText(scrub(original));
|
|
172
180
|
return {
|
|
173
181
|
compilerVersion: PERSONA_COMPILER_VERSION,
|
|
174
182
|
text,
|
|
@@ -180,7 +188,7 @@ export function personaBrief(persona, scrub = (text) => text) {
|
|
|
180
188
|
export function scrubPersonaBrief(persona, scrub) {
|
|
181
189
|
if (!persona.brief)
|
|
182
190
|
return persona;
|
|
183
|
-
const text =
|
|
191
|
+
const text = redactEncodedText(scrub(persona.brief.text));
|
|
184
192
|
return {
|
|
185
193
|
...persona,
|
|
186
194
|
brief: {
|
package/dist/study/plan-base.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// What every route planner shares: the plan fields common to all routes, the frozen residual config,
|
|
2
2
|
// and the requirement and subject helpers.
|
|
3
3
|
import { DEFAULT_OPENAI_CU_MODEL } from "../actors/computer-use/openai-provider.js";
|
|
4
|
-
import {
|
|
4
|
+
import { redactEncodedText } from "../evidence/encoded-text.js";
|
|
5
5
|
// A YAML alias can make a parsed record contain itself (an inline persona that names its own
|
|
6
6
|
// anchor), and structuredClone keeps the cycle, so each object is frozen once.
|
|
7
7
|
function deepFreeze(value, frozen = new WeakSet()) {
|
|
@@ -112,8 +112,9 @@ export function planBase(config, input) {
|
|
|
112
112
|
return {
|
|
113
113
|
studyId: config.id,
|
|
114
114
|
// Every bundle writer takes the title from here, and review.md, run.json and the Observer show
|
|
115
|
-
// it, so a key or sandbox URL in it would make verify grade the run
|
|
116
|
-
|
|
115
|
+
// it, so a key or sandbox URL in it, as written or encoded, would make verify grade the run
|
|
116
|
+
// blocked.
|
|
117
|
+
...(config.title === undefined ? {} : { title: redactEncodedText(config.title) }),
|
|
117
118
|
residual: residualOf(config),
|
|
118
119
|
dryRun: input.dryRun,
|
|
119
120
|
...(analysis === undefined
|