humanish 0.110.4 → 0.111.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/CONTEXT.md +2 -2
- package/dist/actors/local-agent/claude-session.js +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/cli/commands/study.js +2 -4
- package/dist/routes/computer-use/participant-model.js +1 -1
- package/dist/routes/computer-use/plan.js +4 -6
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +1 -1
- package/dist/routes/computer-use/route.js +1 -1
- package/dist/routes/computer-use/types.d.ts +4 -4
- package/dist/routes/scripted/route.js +1 -1
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.js +1 -1
- package/dist/routes/shared-world/plan.js +3 -4
- package/dist/routes/shared-world/types.d.ts +1 -1
- package/dist/routes/terminal/bundle.d.ts +3 -3
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.js +1 -1
- package/dist/routes/terminal/plan.js +1 -1
- package/dist/routes/terminal/route.js +1 -1
- package/dist/routes/terminal/types.d.ts +1 -1
- package/dist/run/results.d.ts +1 -1
- package/dist/run-study.d.ts +5 -3
- package/dist/run-study.js +7 -11
- package/dist/study/adapter-scorer-loader.d.ts +1 -1
- package/dist/study/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.js +21 -11
- package/dist/study/config.d.ts +29 -13
- package/dist/study/config.js +174 -157
- package/dist/study/discover.d.ts +1 -1
- package/dist/study/discover.js +1 -1
- package/dist/study/keys.d.ts +250 -2
- package/dist/study/keys.js +32 -62
- package/dist/study/migrate/convert.js +3 -4
- package/dist/study/migrate/v2.d.ts +17 -7
- package/dist/study/migrate/v2.js +150 -23
- package/dist/study/parse/actors.d.ts +11 -32
- package/dist/study/parse/actors.js +11 -156
- package/dist/study/parse/comms.js +6 -2
- package/dist/study/parse/execution.d.ts +9 -4
- package/dist/study/parse/execution.js +4 -39
- package/dist/study/parse/{study-v3.d.ts → front.d.ts} +6 -17
- package/dist/study/parse/{study-v3.js → front.js} +22 -91
- package/dist/study/parse/subject.d.ts +2 -0
- package/dist/study/parse/subject.js +15 -18
- package/dist/study/persona-resolve.js +7 -9
- package/dist/study/plan-base.d.ts +1 -1
- package/dist/study/plan-base.js +3 -4
- package/dist/study/plan-participants.d.ts +2 -2
- package/dist/study/plan-types.d.ts +5 -5
- package/dist/study/plan.js +1 -1
- package/dist/study/preflight-probes.js +2 -2
- package/dist/study/preflight.d.ts +1 -1
- package/dist/study/preflight.js +1 -1
- package/dist/study/routing.d.ts +5 -5
- package/dist/study/routing.js +5 -5
- package/dist/study/run-study-options.d.ts +1 -1
- package/dist/study/study-deps.d.ts +1 -1
- package/dist/study/study-fields.d.ts +12 -21
- package/dist/study/study-fields.js +29 -34
- package/dist/study/summary.js +9 -4
- package/dist/study/types.d.ts +50 -128
- package/dist/study/types.js +0 -1
- package/dist/study/url-credentials.js +3 -4
- package/dist/study/validation.d.ts +2 -10
- package/dist/study/validation.js +16 -46
- package/dist/study/warnings.d.ts +3 -3
- package/dist/study/warnings.js +40 -109
- package/dist/substrates/local/runtime-config.js +9 -11
- package/docs/contracts/schemas.md +23 -4
- package/package.json +1 -1
- package/dist/study/parse/study.d.ts +0 -38
- package/dist/study/parse/study.js +0 -225
|
@@ -1,51 +1,46 @@
|
|
|
1
|
-
// The study's fields, read in one place.
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
/** The actor that runs. A v3 file declares it as `actor`. */
|
|
1
|
+
// The study's fields, read in one place. Participants are the one read that is not a field: a v3
|
|
2
|
+
// study declares them as a count, as `{ count, instruction }` or as a list, and the accessors below
|
|
3
|
+
// read each form. tests/surface/study-config-reads.test.ts refuses a direct read of a humanish.lab.v2
|
|
4
|
+
// field anywhere outside migrate.
|
|
5
|
+
/** The actor that runs. */
|
|
7
6
|
export function actorOf(config) {
|
|
8
|
-
return config.
|
|
7
|
+
return config.actor;
|
|
9
8
|
}
|
|
10
|
-
/**
|
|
11
|
-
* Every declared actor. parseStudy accepts exactly one; a library config may declare more, and the
|
|
12
|
-
* checks that refuse a second actor's settings read them all.
|
|
13
|
-
*/
|
|
14
|
-
export function actorsOf(config) {
|
|
15
|
-
return config.actors;
|
|
16
|
-
}
|
|
17
|
-
/** The participants listed one by one: a v3 `participants` list. Undefined when none is listed. */
|
|
9
|
+
/** The participants listed one by one: a `participants` list. Undefined when none is listed. */
|
|
18
10
|
export function participantList(config) {
|
|
19
|
-
return config.
|
|
11
|
+
return Array.isArray(config.participants) ? config.participants : undefined;
|
|
20
12
|
}
|
|
21
|
-
/** The declared number of identical participants:
|
|
13
|
+
/** The declared number of identical participants: `participants: <n>` or `participants.count`. */
|
|
22
14
|
export function declaredParticipantCount(config) {
|
|
23
|
-
|
|
15
|
+
const { participants } = config;
|
|
16
|
+
if (typeof participants === "number")
|
|
17
|
+
return participants;
|
|
18
|
+
return isGroup(participants) ? participants.count : undefined;
|
|
24
19
|
}
|
|
25
|
-
/** The steer each of a counted group of participants gets:
|
|
20
|
+
/** The steer each of a counted group of participants gets: `participants.instruction`. */
|
|
26
21
|
export function participantInstruction(config) {
|
|
27
|
-
return config.
|
|
22
|
+
return isGroup(config.participants) ? config.participants.instruction : undefined;
|
|
28
23
|
}
|
|
29
|
-
/** How many scripted surfaces run, 1 for desktop and 2 with mobile.
|
|
24
|
+
/** How many scripted surfaces run, 1 for desktop and 2 with mobile. */
|
|
30
25
|
export function surfaceCount(config) {
|
|
31
|
-
return config.
|
|
26
|
+
return config.surfaces?.length;
|
|
32
27
|
}
|
|
33
|
-
/** The declared run mode.
|
|
28
|
+
/** The declared run mode. */
|
|
34
29
|
export function modeOf(config) {
|
|
35
|
-
return config.
|
|
30
|
+
return config.mode;
|
|
36
31
|
}
|
|
37
|
-
/** The scripted route's scenario id or path.
|
|
32
|
+
/** The scripted route's scenario id or path. */
|
|
38
33
|
export function scenarioRefOf(config) {
|
|
39
|
-
return config.scenario
|
|
34
|
+
return config.scenario;
|
|
40
35
|
}
|
|
41
|
-
/**
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
*/
|
|
45
|
-
export function capsOf(config, route) {
|
|
46
|
-
return route === "terminal" ? config.scenario?.caps : config.execution?.caps;
|
|
36
|
+
/** The spend caps the route enforces. parseStudy refuses a key the route does not read. */
|
|
37
|
+
export function capsOf(config) {
|
|
38
|
+
return config.caps;
|
|
47
39
|
}
|
|
48
|
-
/** True when the participants share one app
|
|
40
|
+
/** True when the participants share one app: `route: shared-world`. */
|
|
49
41
|
export function declaresSharedWorld(config) {
|
|
50
|
-
return config.
|
|
42
|
+
return config.route === "shared-world";
|
|
43
|
+
}
|
|
44
|
+
function isGroup(participants) {
|
|
45
|
+
return typeof participants === "object" && participants !== null && !Array.isArray(participants);
|
|
51
46
|
}
|
package/dist/study/summary.js
CHANGED
|
@@ -18,7 +18,7 @@ import { inspectStudyManifest } from "./discover.js";
|
|
|
18
18
|
import { isComputerUseComposition } from "./routing.js";
|
|
19
19
|
import { probeKeySources } from "../keys/key-resolution.js";
|
|
20
20
|
import { receivingRequiredKey } from "../comms/setup.js";
|
|
21
|
-
import { actorOf, participantList, declaredParticipantCount, capsOf } from "./study-fields.js";
|
|
21
|
+
import { actorOf, participantList, declaredParticipantCount, capsOf, surfaceCount, } from "./study-fields.js";
|
|
22
22
|
export const STUDY_SUMMARY_SCHEMA = "humanish.study-summary.v1";
|
|
23
23
|
/**
|
|
24
24
|
* The effort every participant will run at, or `"per-lane"` when they differ. A participant that declares nothing
|
|
@@ -54,7 +54,12 @@ function participantsOf(config) {
|
|
|
54
54
|
.map((entry) => entry.persona)
|
|
55
55
|
.filter((persona) => typeof persona === "string");
|
|
56
56
|
const personas = rosterPersonas.length > 0 ? rosterPersonas : actor.persona === undefined ? [] : [actor.persona];
|
|
57
|
-
|
|
57
|
+
// A scripted study's participants are its surfaces, one replay on each.
|
|
58
|
+
const count = declaredParticipantCount(config) ??
|
|
59
|
+
surfaceCount(config) ??
|
|
60
|
+
participantList(config)?.length ??
|
|
61
|
+
personas.length ??
|
|
62
|
+
1;
|
|
58
63
|
const unique = [...new Set(personas)];
|
|
59
64
|
if (unique.length === 0)
|
|
60
65
|
return `${count} participant${count === 1 ? "" : "s"}`;
|
|
@@ -62,11 +67,11 @@ function participantsOf(config) {
|
|
|
62
67
|
// are named, because which personas are in a study is the study's design.
|
|
63
68
|
return unique.length === 1 ? `${count} × ${unique[0]}` : unique.join(" · ");
|
|
64
69
|
}
|
|
65
|
-
/** The computer-use caps
|
|
70
|
+
/** The computer-use caps. Other routes draw none. */
|
|
66
71
|
function summaryCapsOf(config) {
|
|
67
72
|
if (!isComputerUseComposition(config))
|
|
68
73
|
return {};
|
|
69
|
-
const caps = capsOf(config
|
|
74
|
+
const caps = capsOf(config);
|
|
70
75
|
return {
|
|
71
76
|
...(typeof caps?.maxUsd === "number" ? { laneUsd: caps.maxUsd } : {}),
|
|
72
77
|
...(typeof caps?.maxTotalUsd === "number" ? { studyUsd: caps.maxTotalUsd } : {}),
|
package/dist/study/types.d.ts
CHANGED
|
@@ -3,7 +3,6 @@ import type { StudyTask } from "./tasks.js";
|
|
|
3
3
|
import type { DwellWindow, StopWhen } from "../actors/stop-conditions.js";
|
|
4
4
|
import { type ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
5
5
|
import type { StudyRoute } from "./routing.js";
|
|
6
|
-
export declare const V2_SCHEMA = "humanish.lab.v2";
|
|
7
6
|
/** The study format: it declares its route, one actor, its participants and one caps block. */
|
|
8
7
|
export declare const STUDY_SCHEMA = "humanish.study.v3";
|
|
9
8
|
export declare const ID_PATTERN: RegExp;
|
|
@@ -18,16 +17,6 @@ export declare const ID_PATTERN: RegExp;
|
|
|
18
17
|
* error, never a desktop attempt. See docs/architecture/state-driven-executor.md.
|
|
19
18
|
*/
|
|
20
19
|
type StudySubjectSource = "this-repo" | "clone" | "app-url" | "local-app" | "terminal-product" | "desktop-cli" | "local-tree";
|
|
21
|
-
/**
|
|
22
|
-
* How a subject's world relates across participants. `per-lane-worlds` (the default; absent ==
|
|
23
|
-
* this) is the only fan-out topology the computer-use route ships: N participants, N independent
|
|
24
|
-
* worlds, isolation + per-participant attribution. `shared-world` is the declared override: one
|
|
25
|
-
* mutable service plane that N participants use at the same time, so their actions interact through
|
|
26
|
-
* shared state. Consumed only on the shared-world routes (a provisioned clone plane, or an
|
|
27
|
-
* external-public app-url plane) with a computer-use actor; inert/warned everywhere else
|
|
28
|
-
* (claims match mechanism).
|
|
29
|
-
*/
|
|
30
|
-
type StudySubjectTopology = "per-lane-worlds" | "shared-world";
|
|
31
20
|
export interface StudySubjectClone {
|
|
32
21
|
/** git clone depth; 1 (shallow) by default. Consumed on the computer-use clone route. */
|
|
33
22
|
depth?: number;
|
|
@@ -189,26 +178,18 @@ export interface StudySubjectProduct {
|
|
|
189
178
|
}
|
|
190
179
|
export interface StudySubject {
|
|
191
180
|
source: StudySubjectSource;
|
|
192
|
-
/**
|
|
193
|
-
* World topology across participants. Absent == `per-lane-worlds` (the isolation default; every
|
|
194
|
-
* existing study is byte-stable). `shared-world` is the declared override: one mutable service
|
|
195
|
-
* plane, N participants at once, each in its role. Consumed only on the shared-world routes (a
|
|
196
|
-
* provisioned clone or an external-public app-url plane, a computer-use actor, and a roster of ≥2
|
|
197
|
-
* participants); inert/warned elsewhere.
|
|
198
|
-
*/
|
|
199
|
-
topology?: StudySubjectTopology;
|
|
200
181
|
/**
|
|
201
182
|
* Concurrent shared-world route only: the author's required attestation that the
|
|
202
183
|
* subject behind the internet-reachable `getHost` URL is synthetic seeded data. The concurrent
|
|
203
184
|
* route exposes the subject on a tokenless public URL for the run's duration, so real/external
|
|
204
185
|
* data must never sit behind it. This is author-trust + a provenance gate (verify also requires
|
|
205
186
|
* `subject.state.provenance == "seeded"`). It does not guarantee the data is synthetic. Required when
|
|
206
|
-
* `
|
|
187
|
+
* `route: shared-world` + `execution.concurrency > 1`; inert/warned elsewhere.
|
|
207
188
|
*/
|
|
208
189
|
exposure?: "synthetic";
|
|
209
190
|
/**
|
|
210
191
|
* External-public shared-world route only: the author's required ownership
|
|
211
|
-
* attestation when a real public deployment (`source: app-url` + `
|
|
192
|
+
* attestation when a real public deployment (`source: app-url` + `route: shared-world` +
|
|
212
193
|
* `concurrency > 1` + `policies.allowPublicTargets: true`) is used directly as the shared plane.
|
|
213
194
|
* The harness neither provisions nor exposes this target (no getHost, no clone, no seed), so it
|
|
214
195
|
* cannot attest the data is synthetic; instead the operator must attest they own/operate it.
|
|
@@ -276,12 +257,6 @@ export interface StudySubject {
|
|
|
276
257
|
*/
|
|
277
258
|
localTree?: StudySubjectLocalTree;
|
|
278
259
|
}
|
|
279
|
-
export interface StudyParticipantFocus {
|
|
280
|
-
id?: string;
|
|
281
|
-
label?: string;
|
|
282
|
-
/** Per-participant steer appended to the actor's mission. Consumed on the app-url route. */
|
|
283
|
-
instruction?: string;
|
|
284
|
-
}
|
|
285
260
|
/**
|
|
286
261
|
* One participant in a differentiated fan-out on the computer-use E2B route (`per-lane-worlds`).
|
|
287
262
|
* Each entry becomes an independent E2B desktop sandbox with its own persona/device/starting-steer. All
|
|
@@ -294,7 +269,7 @@ export interface StudyParticipantEntry {
|
|
|
294
269
|
id?: string;
|
|
295
270
|
/**
|
|
296
271
|
* App-defined actor type label for grouping simulated users ("operator", "viewer",
|
|
297
|
-
* "maintainer", etc.). It is separate from the execution actor dispatch key (`
|
|
272
|
+
* "maintainer", etc.). It is separate from the execution actor dispatch key (`actor.type`);
|
|
298
273
|
* it is adapter-owned taxonomy for roster/readback.
|
|
299
274
|
*/
|
|
300
275
|
actorType?: string;
|
|
@@ -302,7 +277,7 @@ export interface StudyParticipantEntry {
|
|
|
302
277
|
surface?: string;
|
|
303
278
|
/** App-defined correlation id tying participants to one shared case/account/work item. */
|
|
304
279
|
caseGroup?: string;
|
|
305
|
-
/** Persona id/label threaded into this participant's actor prompt. Default:
|
|
280
|
+
/** Persona id/label threaded into this participant's actor prompt. Default: actor.persona. */
|
|
306
281
|
persona?: string;
|
|
307
282
|
/** Named hosted-screen preset for this participant. XOR raw execution.desktop.resolution. */
|
|
308
283
|
device?: string;
|
|
@@ -345,17 +320,6 @@ export interface StudyParticipantEntry {
|
|
|
345
320
|
*/
|
|
346
321
|
host?: boolean;
|
|
347
322
|
}
|
|
348
|
-
/**
|
|
349
|
-
* Compact authoring sugar for repeated participant groups. The parser expands each group into concrete
|
|
350
|
-
* `lanes[]` with deterministic ids (`<group.id>-01`, `<group.id>-02`, ...). The runtime never
|
|
351
|
-
* consumes this shape directly; it always sees ordinary `LabActorLane` entries.
|
|
352
|
-
*/
|
|
353
|
-
export interface StudyActorRosterGroup extends Omit<StudyParticipantEntry, "id"> {
|
|
354
|
-
/** Public-safe group id; prefixes generated participant ids. */
|
|
355
|
-
id: string;
|
|
356
|
-
/** Number of participants to generate for this group. */
|
|
357
|
-
count: number;
|
|
358
|
-
}
|
|
359
323
|
export interface StudyActor {
|
|
360
324
|
/**
|
|
361
325
|
* The actor label. On computer-use (including shared-world), scripted-browser, and
|
|
@@ -365,18 +329,8 @@ export interface StudyActor {
|
|
|
365
329
|
* the descriptor for dispatch and capability enforcement.
|
|
366
330
|
*/
|
|
367
331
|
type: string;
|
|
368
|
-
/** Participant count, route-specific (see the scope header in config.ts): synthetic simCount; scripted
|
|
369
|
-
* surface roster {1 = desktop, 2 = desktop + mobile, default 1}; computer-use E2B route the
|
|
370
|
-
* homogeneous fan-out participant count (cap 16). XOR `lanes`. */
|
|
371
|
-
count?: number;
|
|
372
|
-
/** Computer-use E2B route: a differentiated fan-out roster (`per-lane-worlds`). XOR `count`,
|
|
373
|
-
* `roster`, and `laneFocus`. Cap 16 participants. Consumed only on the cua E2B route
|
|
374
|
-
* (inert/warned elsewhere). */
|
|
375
|
-
lanes?: StudyParticipantEntry[];
|
|
376
332
|
/** Persona id/label threaded into the actor prompt. Consumed on the app-url route. */
|
|
377
333
|
persona?: string;
|
|
378
|
-
/** Consumed on the app-url route (laneFocus.instruction appended to the mission). XOR `lanes`. */
|
|
379
|
-
laneFocus?: StudyParticipantFocus;
|
|
380
334
|
/** Free-form mission threaded into the actor prompt. Consumed on the app-url route. A mission on
|
|
381
335
|
* its own is a complete, valid study; `tasks` is additive, never required. */
|
|
382
336
|
mission?: string;
|
|
@@ -404,7 +358,7 @@ export interface StudyActor {
|
|
|
404
358
|
*/
|
|
405
359
|
localAgent?: "codex" | "claude";
|
|
406
360
|
/**
|
|
407
|
-
* How hard the model is asked to think, per turn. A `
|
|
361
|
+
* How hard the model is asked to think, per turn. A `participants` entry's `reasoningEffort` overrides this.
|
|
408
362
|
*
|
|
409
363
|
* Absent means the provider's default, and absence is recorded as absence: a run that did not
|
|
410
364
|
* declare an effort does not claim one. Support is model-dependent (see src/actors/reasoning-effort.ts),
|
|
@@ -419,12 +373,12 @@ export interface StudyActor {
|
|
|
419
373
|
*/
|
|
420
374
|
reasoningEffort?: ReasoningEffort;
|
|
421
375
|
/**
|
|
422
|
-
* Deterministic completion guard used as the default for CUA participants. A `
|
|
376
|
+
* Deterministic completion guard used as the default for CUA participants. A `participants` entry's stopWhen
|
|
423
377
|
* overrides this value.
|
|
424
378
|
*/
|
|
425
379
|
stopWhen?: StopWhen;
|
|
426
380
|
/**
|
|
427
|
-
* A declared observation window, the default for every participant; a `
|
|
381
|
+
* A declared observation window, the default for every participant; a `participants` entry's dwell overrides
|
|
428
382
|
* it. `when` is a stopWhen-shaped condition (absent: the window opens after the first
|
|
429
383
|
* observation); `ms` is the hold, `everyMs` the frame cadence (default 10 s), `then` whether
|
|
430
384
|
* the participant continues afterwards (default) or the session ends. The harness takes no
|
|
@@ -545,20 +499,6 @@ export interface StudyExecution {
|
|
|
545
499
|
*/
|
|
546
500
|
concurrency?: number;
|
|
547
501
|
desktop?: StudyExecutionDesktop;
|
|
548
|
-
/**
|
|
549
|
-
* Blast-radius budget for each computer-use participant. Consumed on the computer-use route:
|
|
550
|
-
* `caps.maxUsd`, when set, is a fail-closed abort: the session stops the moment its running
|
|
551
|
-
* estimated spend crosses it (the runaway-retry guard), and a cap on a model src/run/pricing.ts
|
|
552
|
-
* cannot price is refused at preflight rather than run uncapped. It is a per-participant cap:
|
|
553
|
-
* enforced inside each participant's loop, so an N-participant fan-out can spend up to N × maxUsd
|
|
554
|
-
* before any participant aborts (the run warns with the true ~N × cap ceiling).
|
|
555
|
-
* `caps.maxTotalUsd` is the shared study budget: one ledger across every participant, the knob a
|
|
556
|
-
* researcher actually reasons with. Absent = uncapped (the historical CUA behavior); maxUsd: 0
|
|
557
|
-
* still permits a request before reported usage trips it. Inert (warned) on non-computer-use
|
|
558
|
-
* routes. Reuses the same StudyScenarioCaps shape as the terminal route's `scenario.caps` (not a
|
|
559
|
-
* fork).
|
|
560
|
-
*/
|
|
561
|
-
caps?: StudyScenarioCaps;
|
|
562
502
|
/** `terminal-product` route: the terminal transport + stdin posture. Consumed on that route. */
|
|
563
503
|
terminal?: StudyExecutionTerminal;
|
|
564
504
|
/** `terminal-product` route: runtime key placement, defaulting to openai-egress, an external
|
|
@@ -584,48 +524,45 @@ export interface StudyExecution {
|
|
|
584
524
|
*/
|
|
585
525
|
egressAllow?: string[];
|
|
586
526
|
}
|
|
587
|
-
type
|
|
527
|
+
type StudyMode = "dry-run" | "live";
|
|
588
528
|
/**
|
|
589
|
-
*
|
|
590
|
-
*
|
|
591
|
-
*
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
529
|
+
* A study's spend, job and time caps, the top-level `caps` block. Each route reads its own keys, and
|
|
530
|
+
* parseStudy refuses a key the route does not read. All values are non-negative numbers.
|
|
531
|
+
*
|
|
532
|
+
* Computer use and shared world read `maxUsd` and `maxTotalUsd`. `maxUsd`, when set, is a
|
|
533
|
+
* fail-closed abort for each participant: the session stops the moment its running estimated spend
|
|
534
|
+
* crosses it (the runaway-retry guard), and a cap on a model src/run/pricing.ts cannot price is
|
|
535
|
+
* refused at preflight rather than run uncapped. It is enforced inside each participant's loop, so
|
|
536
|
+
* an N-participant fan-out can spend up to N × maxUsd before any participant aborts (the run warns
|
|
537
|
+
* with the true ~N × cap ceiling). Absent = uncapped; maxUsd: 0 still permits a request before
|
|
538
|
+
* reported usage trips it.
|
|
539
|
+
*
|
|
540
|
+
* The terminal route passes a live key to an in-sandbox command, and per the safety contract the
|
|
541
|
+
* live key is never exercised without a fail-closed cap in force. It reads `maxUsd`, `maxJobs` and
|
|
542
|
+
* `maxMinutes` (0 is the no-spend default). Live runs require maxUsd and a positive maxMinutes;
|
|
543
|
+
* maxUsd/maxJobs are checked after the session against known ledger signals and maxMinutes is
|
|
544
|
+
* enforced as the command wall clock. Codex tokens are unpriced, so a live run refuses a positive
|
|
545
|
+
* maxUsd unless a costProbe measures spend (HUMANISH_TERMINAL_UNPRICED_CAP).
|
|
595
546
|
*/
|
|
596
|
-
export interface
|
|
547
|
+
export interface StudyCaps {
|
|
597
548
|
/** Max USD the run may spend (provider + product). 0 = no-spend. */
|
|
598
549
|
maxUsd?: number;
|
|
599
550
|
/**
|
|
600
551
|
* study-level model-spend budget, the number a researcher actually reasons with: "this
|
|
601
552
|
* study is N participants, roughly $X", decided once, up front, where recruiting decisions are
|
|
602
|
-
* made.
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
* be set. A positive `scenario.caps.maxTotalUsd` on a computer-use study is a parse error; on the
|
|
609
|
-
* terminal route it is inert (warned), since the single agent's maxUsd already caps the run.
|
|
553
|
+
* made. Computer use and shared world only: every participant's running estimated model spend
|
|
554
|
+
* feeds one shared ledger, and the moment the run total crosses this, each participant stops at
|
|
555
|
+
* its next turn with `budget_reached` (status `incomplete`: the participant ran out of budget;
|
|
556
|
+
* never `gave_up`, because a study-level stop is not the participant's doing). Estimated model
|
|
557
|
+
* spend only; desktop-minutes ride the cost summary but not this ledger. Independent of the
|
|
558
|
+
* per-participant `maxUsd` backstop; either, both, or neither may be set.
|
|
610
559
|
*/
|
|
611
560
|
maxTotalUsd?: number;
|
|
612
|
-
/**
|
|
561
|
+
/** Terminal: max billable product jobs the agent may trigger. 0 = none. */
|
|
613
562
|
maxJobs?: number;
|
|
614
|
-
/**
|
|
563
|
+
/** Terminal: max wall-clock minutes for the agent session. */
|
|
615
564
|
maxMinutes?: number;
|
|
616
565
|
}
|
|
617
|
-
export interface StudyScenario {
|
|
618
|
-
/** Reference a committed scenario by id (humanish/scenarios/<ref>.yaml) or path. Consumed
|
|
619
|
-
* (and required) on the scripted-browser route; forward-declared elsewhere. */
|
|
620
|
-
ref?: string;
|
|
621
|
-
/** Or inline the scenario body. Forward-declared: no route reads it yet, so a set value warns. */
|
|
622
|
-
inline?: Record<string, unknown>;
|
|
623
|
-
/** dry-run = a synthetic bundle with no provider spend; live = real run. Consumed. */
|
|
624
|
-
mode?: StudyScenarioMode;
|
|
625
|
-
/** Spend/job/time caps. Consumed (recorded in the bundle) on the terminal-product route;
|
|
626
|
-
* inert (warned) elsewhere. */
|
|
627
|
-
caps?: StudyScenarioCaps;
|
|
628
|
-
}
|
|
629
566
|
export interface StudyPolicies {
|
|
630
567
|
/**
|
|
631
568
|
* Redact target repo labels in durable artifacts. Consumed on the computer-use,
|
|
@@ -722,7 +659,7 @@ export interface StudyCommsReceivingEmail {
|
|
|
722
659
|
interface StudyCommsCaptureEmail {
|
|
723
660
|
connection?: never;
|
|
724
661
|
allowedOrigins?: never;
|
|
725
|
-
/** Which implementation backs the inbox (a backend discriminator, distinct from `
|
|
662
|
+
/** Which implementation backs the inbox (a backend discriminator, distinct from `mode`):
|
|
726
663
|
* `fake` (default) is an in-harness in-memory inbox in the Fowler test-double sense: an in-sandbox
|
|
727
664
|
* catch captures the app's sends. Provider-backed receiving uses the separate
|
|
728
665
|
* StudyCommsReceivingEmail configuration selected by a saved connection. */
|
|
@@ -789,45 +726,28 @@ export interface StudyCommsRecipient {
|
|
|
789
726
|
* for the persona surface's default address (not drain-matched). */
|
|
790
727
|
address?: string;
|
|
791
728
|
}
|
|
792
|
-
export interface StudyConfig {
|
|
793
|
-
/** The format the file was written in. Both parse into this one shape. */
|
|
794
|
-
schema: typeof V2_SCHEMA | typeof STUDY_SCHEMA;
|
|
795
|
-
id: string;
|
|
796
|
-
title?: string;
|
|
797
|
-
description?: string;
|
|
798
|
-
subject: StudySubject;
|
|
799
|
-
actors: StudyActor[];
|
|
800
|
-
execution?: StudyExecution;
|
|
801
|
-
/** Forward-declared: no route reads it yet, so a set value warns. */
|
|
802
|
-
personas?: Record<string, unknown>[];
|
|
803
|
-
scenario?: StudyScenario;
|
|
804
|
-
policies?: StudyPolicies;
|
|
805
|
-
review?: StudyReview;
|
|
806
|
-
defaults?: StudyDefaults;
|
|
807
|
-
comms?: StudyComms;
|
|
808
|
-
}
|
|
809
729
|
/**
|
|
810
|
-
* A humanish.study.v3
|
|
811
|
-
* participants and one caps block.
|
|
812
|
-
* StudyConfig above.
|
|
730
|
+
* A humanish.study.v3 study as parseStudy returns it, with the keys the file has: the declared
|
|
731
|
+
* route, one actor, its participants and one caps block.
|
|
813
732
|
*/
|
|
814
|
-
export interface
|
|
733
|
+
export interface StudyConfig {
|
|
815
734
|
schema: typeof STUDY_SCHEMA;
|
|
816
735
|
id: string;
|
|
817
736
|
title?: string;
|
|
818
737
|
description?: string;
|
|
738
|
+
/** The route the study declares. parseStudy refuses one its subject and actor do not take. */
|
|
819
739
|
route: StudyRoute;
|
|
820
740
|
/** Absent: a dry run. */
|
|
821
|
-
mode?:
|
|
822
|
-
subject:
|
|
823
|
-
actor:
|
|
741
|
+
mode?: StudyMode;
|
|
742
|
+
subject: StudySubject;
|
|
743
|
+
actor: StudyActor;
|
|
824
744
|
/** Preview, computer-use and shared-world, in the form the file used. */
|
|
825
745
|
participants?: StudyParticipants;
|
|
826
746
|
/** Scripted only. */
|
|
827
747
|
surfaces?: StudySurfaces;
|
|
828
748
|
/** The caps keys the route reads. */
|
|
829
|
-
caps?:
|
|
830
|
-
execution?:
|
|
749
|
+
caps?: StudyCaps;
|
|
750
|
+
execution?: StudyExecution;
|
|
831
751
|
/** Scripted: a committed scenario id or path. */
|
|
832
752
|
scenario?: string;
|
|
833
753
|
policies?: StudyPolicies;
|
|
@@ -836,15 +756,17 @@ export interface StudyV3 {
|
|
|
836
756
|
comms?: StudyComms;
|
|
837
757
|
}
|
|
838
758
|
/**
|
|
839
|
-
* A count, a count with one instruction for every participant, or a list.
|
|
840
|
-
* `count`
|
|
759
|
+
* A count, a count with one instruction for every participant, or a list. parseStudy expands a list
|
|
760
|
+
* entry with a `count` into its participants, `<id>-01` to `<id>-NN`.
|
|
841
761
|
*/
|
|
842
762
|
type StudyParticipants = number | StudyParticipantGroup | StudyParticipantEntry[];
|
|
843
|
-
|
|
763
|
+
/** `participants` as identical participants that share one instruction. */
|
|
764
|
+
export interface StudyParticipantGroup {
|
|
844
765
|
count?: number;
|
|
845
766
|
instruction?: string;
|
|
846
767
|
}
|
|
847
|
-
|
|
768
|
+
/** The scripted surfaces: desktop, or desktop and mobile. */
|
|
769
|
+
export type StudySurfaces = readonly ["desktop"] | readonly ["desktop", "mobile"];
|
|
848
770
|
interface StudyParseSuccess {
|
|
849
771
|
ok: true;
|
|
850
772
|
config: StudyConfig;
|
package/dist/study/types.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {} from "../analysis/automatic-config.js";
|
|
2
2
|
import {} from "../actors/reasoning-effort.js";
|
|
3
|
-
export const V2_SCHEMA = "humanish.lab.v2";
|
|
4
3
|
/** The study format: it declares its route, one actor, its participants and one caps block. */
|
|
5
4
|
export const STUDY_SCHEMA = "humanish.study.v3";
|
|
6
5
|
// Must start alphanumeric so an id never collides with the path-vs-id resolver heuristic
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Every URL a study config declares, checked for a credential. Route planners call it, since a
|
|
2
2
|
// library caller's config skips the parser that refuses these first.
|
|
3
|
-
import { rosterOf } from "./parse/actors.js";
|
|
4
3
|
import { entryCredentialReason, urlCredentialReason } from "./parse/url-credentials.js";
|
|
5
|
-
import {
|
|
4
|
+
import { participantList } from "./study-fields.js";
|
|
6
5
|
/**
|
|
7
6
|
* The first URL in `config` that carries a credential, with why: the subject's app, serve and
|
|
8
7
|
* public surface URLs, and each participant's target and entry.
|
|
@@ -17,14 +16,14 @@ export function studyUrlCredentialReason(config) {
|
|
|
17
16
|
? undefined
|
|
18
17
|
: urlCredentialReason("subject.serve.url", subject.serve.url),
|
|
19
18
|
...(subject.product?.publicSurfaces ?? []).map((surface) => () => urlCredentialReason("subject.product.publicSurfaces", surface)),
|
|
20
|
-
...
|
|
19
|
+
...(participantList(config) ?? []).flatMap((participant, index) => [
|
|
21
20
|
() => participant.target === undefined
|
|
22
21
|
? undefined
|
|
23
22
|
: urlCredentialReason(`participants[${index}].target`, participant.target),
|
|
24
23
|
() => participant.entry === undefined
|
|
25
24
|
? undefined
|
|
26
25
|
: entryCredentialReason(`participants[${index}].entry`, participant.entry),
|
|
27
|
-
])
|
|
26
|
+
]),
|
|
28
27
|
];
|
|
29
28
|
for (const reason of reasons) {
|
|
30
29
|
const found = reason();
|
|
@@ -7,7 +7,7 @@ import type { StudyConfig } from "./types.js";
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function computerUseValidationReason(config: StudyConfig): string | null;
|
|
9
9
|
/**
|
|
10
|
-
* Cross-validate a `
|
|
10
|
+
* Cross-validate a `route: shared-world` declaration. Returns the failure message, or
|
|
11
11
|
* null when the one shared-world route can run it: the external-public checks for an app-url
|
|
12
12
|
* subject, the provisioned checks otherwise. Enforced at parse and again by the route, since
|
|
13
13
|
* runStudyWith takes a config that skipped the parser.
|
|
@@ -34,14 +34,6 @@ export declare function taskProtocolValidationReason(config: StudyConfig, suppor
|
|
|
34
34
|
* at parse and again on the computer-use and scripted-browser routes for library callers.
|
|
35
35
|
*/
|
|
36
36
|
export declare function cloneTargetValidationReason(config: StudyConfig): string | null;
|
|
37
|
-
/**
|
|
38
|
-
* Refuse a positive `scenario.caps.maxUsd` or `maxTotalUsd` on a computer-use study.
|
|
39
|
-
* `scenario.caps` belongs to the terminal route; the computer-use route stops on `execution.caps`,
|
|
40
|
-
* so a dollar figure here would read as a cap while the study ran uncapped. Zero spends nothing
|
|
41
|
-
* either way and stays a warning. Enforced at parse and again on the computer-use routes for
|
|
42
|
-
* library callers.
|
|
43
|
-
*/
|
|
44
|
-
export declare function scenarioCapsValidationReason(config: StudyConfig): string | null;
|
|
45
37
|
/** Refuse a claimed output bound when the route cannot pass it to the first-party provider. */
|
|
46
38
|
export declare function outputTokenLimitValidationReason(config: StudyConfig): string | null;
|
|
47
39
|
/**
|
|
@@ -60,7 +52,7 @@ export declare function concurrentSharedWorldValidationReason(config: StudyConfi
|
|
|
60
52
|
* it rejects every provisioned-subject field (serve/state.seed/state.checkpoint/exposure/clone/repos
|
|
61
53
|
* are inert with no sandbox, so they fail closed), and requires a
|
|
62
54
|
* non-loopback appUrl + allowPublicTargets + the operator-ownership attestation subject.publicTarget +
|
|
63
|
-
* concurrency >= 2 +
|
|
55
|
+
* concurrency >= 2 + a `participants` list of ≥2 with exactly one host participant. The getHost synthetic
|
|
64
56
|
* gate is deliberately unreachable here (there is no internet-reachable harness-owned URL to attest).
|
|
65
57
|
* Enforced at parse and re-enforced in the engine (runStudyWith takes a config that skipped the parser).
|
|
66
58
|
*/
|