humanish 0.111.1 → 0.111.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -1
- package/CHANGELOG.md +29 -0
- package/dist/cli/commands/study-route-computer-use.js +3 -8
- package/dist/cli/commands/study-route-preview.js +4 -6
- package/dist/cli/commands/study-route-shared-world.js +6 -7
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +3 -3
- package/dist/observer-app.html +6 -6
- package/dist/routes/computer-use/bundle-parts.d.ts +1 -32
- package/dist/routes/computer-use/bundle-parts.js +5 -92
- package/dist/routes/computer-use/e2b-desktop/prepare.js +11 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -2
- package/dist/routes/computer-use/live-phase.js +3 -7
- package/dist/routes/computer-use/participant-deps.d.ts +46 -0
- package/dist/routes/computer-use/participant-deps.js +48 -0
- package/dist/routes/computer-use/participant-facts.d.ts +5 -4
- package/dist/routes/computer-use/participant-facts.js +37 -2
- package/dist/routes/computer-use/participant-feedback.d.ts +33 -0
- package/dist/routes/computer-use/participant-feedback.js +92 -0
- package/dist/routes/computer-use/participant-model.d.ts +11 -0
- package/dist/routes/computer-use/participant-model.js +15 -0
- package/dist/routes/computer-use/preflight.js +33 -65
- package/dist/routes/computer-use/result.js +13 -52
- package/dist/routes/computer-use/route.js +6 -7
- package/dist/routes/computer-use/setup.d.ts +2 -6
- package/dist/routes/computer-use/setup.js +47 -93
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +2 -2
- package/dist/routes/preview.js +2 -8
- package/dist/routes/scripted/bundle.d.ts +2 -6
- package/dist/routes/scripted/result.d.ts +2 -1
- package/dist/routes/scripted/result.js +4 -4
- package/dist/routes/scripted/route.js +13 -38
- package/dist/routes/scripted/setup.d.ts +2 -1
- package/dist/routes/scripted/setup.js +21 -28
- package/dist/routes/scripted/subject.d.ts +3 -18
- package/dist/routes/scripted/subject.js +22 -78
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/bundle.js +36 -1
- package/dist/routes/shared-world/comms.d.ts +2 -1
- package/dist/routes/shared-world/comms.js +3 -7
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.d.ts +4 -5
- package/dist/routes/shared-world/participant-specs.js +19 -25
- package/dist/routes/shared-world/plan.js +11 -7
- package/dist/routes/shared-world/provisioned.d.ts +2 -1
- package/dist/routes/shared-world/provisioned.js +37 -80
- package/dist/routes/shared-world/result.js +17 -35
- package/dist/routes/shared-world/route.js +33 -36
- package/dist/routes/shared-world/setup.d.ts +2 -3
- package/dist/routes/shared-world/setup.js +11 -29
- package/dist/routes/shared-world/types.d.ts +6 -15
- package/dist/routes/terminal/dry-run.js +14 -24
- package/dist/routes/terminal/live-finish.d.ts +0 -1
- package/dist/routes/terminal/live-finish.js +2 -2
- package/dist/routes/terminal/route.js +9 -15
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +28 -40
- package/dist/routes/terminal/trace.d.ts +1 -1
- package/dist/routes/terminal/types.d.ts +2 -2
- package/dist/run/run.d.ts +55 -1
- package/dist/run/run.js +38 -2
- package/dist/run/secrets.d.ts +22 -0
- package/dist/run/secrets.js +33 -0
- package/dist/run/study-result.d.ts +34 -0
- package/dist/run/study-result.js +16 -0
- package/dist/study/requirements.d.ts +60 -5
- package/dist/study/requirements.js +88 -2
- package/dist/study/run-study-events.d.ts +4 -3
- package/dist/study/run-study-events.js +9 -14
- package/dist/study/run-study-options.d.ts +4 -1
- package/dist/study/run-study-options.js +22 -32
- package/dist/substrates/e2b/subject-sandbox.d.ts +58 -0
- package/dist/substrates/e2b/subject-sandbox.js +94 -0
- package/docs/contracts/core.md +5 -4
- package/docs/contracts/feedback.md +12 -5
- package/package.json +1 -1
package/dist/run/run.js
CHANGED
|
@@ -2,16 +2,18 @@
|
|
|
2
2
|
// token that lets the Observer and automatic analysis read it. Routes build bundles and write
|
|
3
3
|
// their own evidence files; this module owns when the run starts, how it is published, and how it
|
|
4
4
|
// is closed on every exit.
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
5
6
|
import { redactText } from "../evidence/redaction.js";
|
|
6
7
|
import { buildObserverData } from "../observer/data.js";
|
|
7
8
|
import { renderObserver } from "../observer/render.js";
|
|
8
|
-
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD, } from "./bundle.js";
|
|
9
|
+
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD, buildRunSource, } from "./bundle.js";
|
|
9
10
|
import { judgeExecution, } from "./judge.js";
|
|
10
11
|
import {} from "./results.js";
|
|
11
12
|
import { createRunArtifactPaths, validatePreparedRunArtifactPaths, } from "./paths.js";
|
|
12
13
|
import { registerActiveRun } from "./active-runs.js";
|
|
13
14
|
import { writeContainedOutputFile, writePreparedRunLatestPointer } from "./contained-output.js";
|
|
14
15
|
import { scrubRunSandboxIds, withPublicSandboxIds } from "./sandbox-ids.js";
|
|
16
|
+
import { RunSecrets } from "./secrets.js";
|
|
15
17
|
import { beginRunStatus, runStatusOutcome, } from "./status.js";
|
|
16
18
|
/** An entry point called in a state that does not admit it. */
|
|
17
19
|
class RunLifecycleError extends Error {
|
|
@@ -239,7 +241,7 @@ export async function runScope(fn) {
|
|
|
239
241
|
return operation;
|
|
240
242
|
};
|
|
241
243
|
const refuse = (message) => Promise.reject(new RunLifecycleError(message));
|
|
242
|
-
const
|
|
244
|
+
const startRunWith = (options) => {
|
|
243
245
|
if (closed)
|
|
244
246
|
return refuse("The run scope has closed.");
|
|
245
247
|
if (started)
|
|
@@ -277,6 +279,9 @@ export async function runScope(fn) {
|
|
|
277
279
|
return { ok: true, run: run.run };
|
|
278
280
|
})());
|
|
279
281
|
};
|
|
282
|
+
function startRun(...args) {
|
|
283
|
+
return args.length === 1 ? startRunWith(args[0]) : startRecordedRun(startRunWith, ...args);
|
|
284
|
+
}
|
|
280
285
|
const openRun = (options, runId, paths, runStatus) => {
|
|
281
286
|
const now = options.now ?? Date.now;
|
|
282
287
|
const createdAt = new Date(now()).toISOString();
|
|
@@ -301,6 +306,7 @@ export async function runScope(fn) {
|
|
|
301
306
|
mode: options.mode,
|
|
302
307
|
...(options.study === undefined ? {} : { study: options.study }),
|
|
303
308
|
paths,
|
|
309
|
+
secrets: options.secrets ?? new RunSecrets([]),
|
|
304
310
|
participantStarted() {
|
|
305
311
|
participantsRan = true;
|
|
306
312
|
},
|
|
@@ -348,3 +354,33 @@ export async function runScope(fn) {
|
|
|
348
354
|
}
|
|
349
355
|
return { result: outcome.result, finished };
|
|
350
356
|
}
|
|
357
|
+
/** Start a recorded route's run through `start`, the scope's own form, then capture its source. */
|
|
358
|
+
async function startRecordedRun(start, plan, input, route) {
|
|
359
|
+
const started = await start({
|
|
360
|
+
cwd: route.cwd,
|
|
361
|
+
runId: input.runId,
|
|
362
|
+
mintRunId: () => mintRunId(route.prefix),
|
|
363
|
+
mode: plan.dryRun ? "dry-run" : "live",
|
|
364
|
+
study: plan.study,
|
|
365
|
+
warnings: plan.warnings,
|
|
366
|
+
sandboxes: route.sandboxes,
|
|
367
|
+
renderReview: route.renderReview,
|
|
368
|
+
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
369
|
+
now: route.now,
|
|
370
|
+
secrets: route.secrets,
|
|
371
|
+
});
|
|
372
|
+
if (!started.ok)
|
|
373
|
+
return started;
|
|
374
|
+
const source = await buildRunSource({
|
|
375
|
+
cwd: route.cwd,
|
|
376
|
+
capturedAt: started.run.createdAt,
|
|
377
|
+
humanishSource: "present",
|
|
378
|
+
packageName: "humanish",
|
|
379
|
+
});
|
|
380
|
+
return { ok: true, run: { ...started.run, source } };
|
|
381
|
+
}
|
|
382
|
+
/** `<prefix>-<wall-clock time, with : and . as ->-<8 hex>`. */
|
|
383
|
+
function mintRunId(prefix) {
|
|
384
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
385
|
+
return `${prefix}-${stamp}-${randomBytes(4).toString("hex")}`;
|
|
386
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Each route's seed list, marker and floor are its own; the defaults are what most routes use. */
|
|
2
|
+
interface RunSecretsOptions {
|
|
3
|
+
/** The text that replaces each value. */
|
|
4
|
+
readonly marker?: string;
|
|
5
|
+
/** The shortest value scrubbed. A shorter value is ordinary text often enough to leave alone. */
|
|
6
|
+
readonly minLength?: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class RunSecrets {
|
|
9
|
+
#private;
|
|
10
|
+
/**
|
|
11
|
+
* Replaces every held value with the marker. It reads the values on each call, so a value added
|
|
12
|
+
* after the scrub was handed to a participant or a subject is scrubbed from then on.
|
|
13
|
+
*/
|
|
14
|
+
readonly scrub: (text: string) => string;
|
|
15
|
+
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
16
|
+
constructor(seed: readonly string[], options?: RunSecretsOptions);
|
|
17
|
+
/** Holds each value the run learns while it runs, unless it is under the floor or already held. */
|
|
18
|
+
add(values: readonly string[]): void;
|
|
19
|
+
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
20
|
+
values(): readonly string[];
|
|
21
|
+
}
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// The literal values one run scrubs from what it writes: the provider keys and subject env values
|
|
2
|
+
// its route reads, and the values the run provisions while it runs, such as the addresses email
|
|
3
|
+
// receiving registers. These have no secret shape for pattern redaction to find, so each is
|
|
4
|
+
// replaced as written; redactText runs after the scrub and finds secrets by shape.
|
|
5
|
+
import { scrubLiterals } from "../evidence/redaction.js";
|
|
6
|
+
export class RunSecrets {
|
|
7
|
+
// Private fields stay out of JSON.stringify and object spread, so a Run that reaches a writer
|
|
8
|
+
// whole cannot carry the values with it.
|
|
9
|
+
#held;
|
|
10
|
+
#minLength;
|
|
11
|
+
/**
|
|
12
|
+
* Replaces every held value with the marker. It reads the values on each call, so a value added
|
|
13
|
+
* after the scrub was handed to a participant or a subject is scrubbed from then on.
|
|
14
|
+
*/
|
|
15
|
+
scrub;
|
|
16
|
+
/** `seed` keeps its order; values shorter than the floor are dropped. */
|
|
17
|
+
constructor(seed, options = {}) {
|
|
18
|
+
// An empty value would match between every character.
|
|
19
|
+
this.#minLength = Math.max(1, options.minLength ?? 4);
|
|
20
|
+
this.#held = seed.filter((value) => value.length >= this.#minLength);
|
|
21
|
+
this.scrub = scrubLiterals(this.#held, options.marker);
|
|
22
|
+
}
|
|
23
|
+
/** Holds each value the run learns while it runs, unless it is under the floor or already held. */
|
|
24
|
+
add(values) {
|
|
25
|
+
for (const value of values)
|
|
26
|
+
if (value.length >= this.#minLength && !this.#held.includes(value))
|
|
27
|
+
this.#held.push(value);
|
|
28
|
+
}
|
|
29
|
+
/** The held values, seed first, in one array that later `add` calls extend. */
|
|
30
|
+
values() {
|
|
31
|
+
return this.#held;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -7,5 +7,39 @@ export interface StudyResultIdentity<R extends string> {
|
|
|
7
7
|
/** The study's `id`. */
|
|
8
8
|
studyId: string;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* The error codes every recorded route's result can carry besides its own: a study, option or
|
|
12
|
+
* analysis refusal from before the route's checks, and a run id that is already taken. Each route's
|
|
13
|
+
* error-code union is this set plus the codes named for that route.
|
|
14
|
+
*/
|
|
15
|
+
export type StudyResultErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_RUN_ID_IN_USE";
|
|
10
16
|
/** The identity fields for a result of route `route` from the study `studyId`. */
|
|
11
17
|
export declare function studyResultIdentity<R extends string>(route: R, studyId: string): StudyResultIdentity<R>;
|
|
18
|
+
/** What every refused study's result carries besides its identity and its route's own fields. */
|
|
19
|
+
export interface RefusalBase<C extends string> {
|
|
20
|
+
/** The study's `id`. */
|
|
21
|
+
studyId: string;
|
|
22
|
+
cwd: string;
|
|
23
|
+
/** The caller's warnings array, kept by reference so warnings added later still show. Empty when omitted. */
|
|
24
|
+
warnings?: string[];
|
|
25
|
+
error: {
|
|
26
|
+
code: C;
|
|
27
|
+
message: string;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* A refused study's result on route `route`: its identity, `ok: false` and `cwd`, then `fields`
|
|
32
|
+
* in the caller's order, then `warnings` and `error`. A result is written as JSON, so the key order
|
|
33
|
+
* is part of the output; a site whose order differs spreads its remaining fields after this. The
|
|
34
|
+
* caller keeps runId, actor, dryRun and the route's fields, whose values and order differ by site.
|
|
35
|
+
*/
|
|
36
|
+
export declare function refusedResult<R extends string, C extends string, F extends object>(route: R, base: RefusalBase<C>, fields: F): StudyResultIdentity<R> & {
|
|
37
|
+
ok: false;
|
|
38
|
+
cwd: string;
|
|
39
|
+
} & F & {
|
|
40
|
+
warnings: string[];
|
|
41
|
+
error: {
|
|
42
|
+
code: C;
|
|
43
|
+
message: string;
|
|
44
|
+
};
|
|
45
|
+
};
|
package/dist/run/study-result.js
CHANGED
|
@@ -7,3 +7,19 @@ export const STUDY_RESULT_SCHEMA = "humanish.study-result.v1";
|
|
|
7
7
|
export function studyResultIdentity(route, studyId) {
|
|
8
8
|
return { schema: STUDY_RESULT_SCHEMA, route, studyId };
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* A refused study's result on route `route`: its identity, `ok: false` and `cwd`, then `fields`
|
|
12
|
+
* in the caller's order, then `warnings` and `error`. A result is written as JSON, so the key order
|
|
13
|
+
* is part of the output; a site whose order differs spreads its remaining fields after this. The
|
|
14
|
+
* caller keeps runId, actor, dryRun and the route's fields, whose values and order differ by site.
|
|
15
|
+
*/
|
|
16
|
+
export function refusedResult(route, base, fields) {
|
|
17
|
+
return {
|
|
18
|
+
...studyResultIdentity(route, base.studyId),
|
|
19
|
+
ok: false,
|
|
20
|
+
cwd: base.cwd,
|
|
21
|
+
...fields,
|
|
22
|
+
warnings: base.warnings ?? [],
|
|
23
|
+
error: base.error,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
3
|
-
export declare function missingKeys(requirements: readonly Requirement[], env: Record<string, string | undefined>): string[];
|
|
4
|
-
/** The subject env names the requirements list and `env` lacks, in declaration order. */
|
|
5
|
-
export declare function missingSubjectEnv(requirements: readonly Requirement[], env: Record<string, string | undefined>): string[];
|
|
1
|
+
import { type LocalAgentRefusal } from "../actors/local-agent/readiness.js";
|
|
2
|
+
import type { Brain, StudyPlan, Requirement } from "./plan-types.js";
|
|
6
3
|
/**
|
|
7
4
|
* The provider keys the requirements name, in plan order. A `key-one-of` names the first of its
|
|
8
5
|
* keys that `isPresent` reports, else OPENAI_API_KEY.
|
|
@@ -18,3 +15,61 @@ export declare function requiredSubjectEnv(requirements: readonly Requirement[])
|
|
|
18
15
|
* a declared scorer's host code, are not in the plan.
|
|
19
16
|
*/
|
|
20
17
|
export declare function keyNamesOf(plan: StudyPlan): ReadonlySet<string>;
|
|
18
|
+
/** A live check's refusal, in its route's error code. */
|
|
19
|
+
export interface LiveRefusal<C extends string> {
|
|
20
|
+
readonly code: C;
|
|
21
|
+
readonly message: string;
|
|
22
|
+
}
|
|
23
|
+
/** One live check: its refusal, or undefined when this machine passes it. */
|
|
24
|
+
export type LiveCheck<C extends string> = () => LiveRefusal<C> | undefined | Promise<LiveRefusal<C> | undefined>;
|
|
25
|
+
/**
|
|
26
|
+
* The first refusal of `checks`, run one at a time in the order given. Which check comes first
|
|
27
|
+
* decides which error a run with two problems reports, so each route keeps its own order.
|
|
28
|
+
*/
|
|
29
|
+
export declare function firstLiveRefusal<C extends string>(checks: readonly LiveCheck<C>[]): Promise<LiveRefusal<C> | undefined>;
|
|
30
|
+
/**
|
|
31
|
+
* Refuses a run whose environment lacks a provider key the requirements list. `need` writes the
|
|
32
|
+
* route's sentence for the missing names ("OPENAI_API_KEY and E2B_API_KEY"); the message goes on
|
|
33
|
+
* to say where discovery looked and what fills each key. With `suggestLocalAgent`, a missing
|
|
34
|
+
* OPENAI_API_KEY also names each coding agent signed in on this machine: someone new meets this
|
|
35
|
+
* refusal first, and an agent they already use runs the participants without a key.
|
|
36
|
+
*/
|
|
37
|
+
export declare function keysCheck<C extends string>(args: {
|
|
38
|
+
readonly requirements: readonly Requirement[];
|
|
39
|
+
readonly env: Record<string, string | undefined>;
|
|
40
|
+
readonly code: C;
|
|
41
|
+
readonly need: (names: string) => string;
|
|
42
|
+
readonly suggestLocalAgent?: boolean;
|
|
43
|
+
}): Promise<LiveRefusal<C> | undefined>;
|
|
44
|
+
/**
|
|
45
|
+
* Refuses a local-agent brain whose CLI is missing, signed out, too old, or cannot honor the
|
|
46
|
+
* caps, before any sandbox is paid for. `codes` names the route's code for each kind of refusal.
|
|
47
|
+
*/
|
|
48
|
+
export declare function localAgentCheck<C extends string>(args: {
|
|
49
|
+
readonly brain: Brain;
|
|
50
|
+
readonly env: Record<string, string | undefined>;
|
|
51
|
+
readonly caps: {
|
|
52
|
+
readonly maxUsd?: number;
|
|
53
|
+
readonly maxTotalUsd?: number;
|
|
54
|
+
};
|
|
55
|
+
readonly codes: Readonly<Record<LocalAgentRefusal["kind"], C>>;
|
|
56
|
+
}): Promise<LiveRefusal<C> | undefined>;
|
|
57
|
+
/** Refuses a run whose environment lacks a subject env name the requirements list. */
|
|
58
|
+
export declare function subjectEnvCheck<C extends string>(args: {
|
|
59
|
+
readonly requirements: readonly Requirement[];
|
|
60
|
+
readonly env: Record<string, string | undefined>;
|
|
61
|
+
readonly code: C;
|
|
62
|
+
}): LiveRefusal<C> | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Refuses a maxUsd or maxTotalUsd cap on a model with no rate in src/run/pricing.ts. The loop
|
|
65
|
+
* could not measure spend against the cap, and running uncapped would drop the protection the
|
|
66
|
+
* cap promised; the operator picks a priced model or removes the cap.
|
|
67
|
+
*/
|
|
68
|
+
export declare function unpricedCapCheck<C extends string>(args: {
|
|
69
|
+
readonly caps: {
|
|
70
|
+
readonly maxUsd?: number;
|
|
71
|
+
readonly maxTotalUsd?: number;
|
|
72
|
+
};
|
|
73
|
+
readonly model: string;
|
|
74
|
+
readonly code: C;
|
|
75
|
+
}): LiveRefusal<C> | undefined;
|
|
@@ -1,13 +1,23 @@
|
|
|
1
|
+
// The keys and subject env a live plan's requirements list, and the live checks that read them.
|
|
2
|
+
// Each route lists the checks below (keysCheck, localAgentCheck, subjectEnvCheck,
|
|
3
|
+
// unpricedCapCheck) in its own order, with its own error codes. Doctor and the TUI ask
|
|
4
|
+
// requiredKeys and requiredSubjectEnv which names to report, and `run` asks keyNamesOf which
|
|
5
|
+
// key-source lines to print, so the planner alone decides what a run needs. The terminal runtime
|
|
6
|
+
// key (`key-one-of`) stays with buildRuntimeAuth at run time, which also picks its placement.
|
|
7
|
+
import { detectLocalAgents } from "../actors/local-agent/cli.js";
|
|
8
|
+
import { localAgentRefusal } from "../actors/local-agent/readiness.js";
|
|
9
|
+
import { describeMissingKeys } from "../keys/key-resolution.js";
|
|
10
|
+
import { MODEL_RATES, unpricedCapMessage } from "../run/pricing.js";
|
|
1
11
|
/** Provider keys in the order the routes name them in a refusal. */
|
|
2
12
|
const KEY_ORDER = ["OPENAI_API_KEY", "E2B_API_KEY"];
|
|
3
13
|
const present = (env, name) => (env[name]?.trim() ?? "") !== "";
|
|
4
14
|
/** The `key` requirements whose value `env` lacks, OPENAI_API_KEY before E2B_API_KEY. */
|
|
5
|
-
|
|
15
|
+
function missingKeys(requirements, env) {
|
|
6
16
|
const required = new Set(requirements.flatMap((requirement) => (requirement.kind === "key" ? [requirement.name] : [])));
|
|
7
17
|
return KEY_ORDER.filter((name) => required.has(name) && !present(env, name));
|
|
8
18
|
}
|
|
9
19
|
/** The subject env names the requirements list and `env` lacks, in declaration order. */
|
|
10
|
-
|
|
20
|
+
function missingSubjectEnv(requirements, env) {
|
|
11
21
|
return requiredSubjectEnv(requirements).filter((name) => !present(env, name));
|
|
12
22
|
}
|
|
13
23
|
/**
|
|
@@ -52,3 +62,79 @@ export function keyNamesOf(plan) {
|
|
|
52
62
|
names.add("OPENAI_API_KEY");
|
|
53
63
|
return names;
|
|
54
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* The first refusal of `checks`, run one at a time in the order given. Which check comes first
|
|
67
|
+
* decides which error a run with two problems reports, so each route keeps its own order.
|
|
68
|
+
*/
|
|
69
|
+
export async function firstLiveRefusal(checks) {
|
|
70
|
+
for (const check of checks) {
|
|
71
|
+
const refusal = await check();
|
|
72
|
+
if (refusal !== undefined)
|
|
73
|
+
return refusal;
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Refuses a run whose environment lacks a provider key the requirements list. `need` writes the
|
|
79
|
+
* route's sentence for the missing names ("OPENAI_API_KEY and E2B_API_KEY"); the message goes on
|
|
80
|
+
* to say where discovery looked and what fills each key. With `suggestLocalAgent`, a missing
|
|
81
|
+
* OPENAI_API_KEY also names each coding agent signed in on this machine: someone new meets this
|
|
82
|
+
* refusal first, and an agent they already use runs the participants without a key.
|
|
83
|
+
*/
|
|
84
|
+
export async function keysCheck(args) {
|
|
85
|
+
const { env } = args;
|
|
86
|
+
const missing = missingKeys(args.requirements, env);
|
|
87
|
+
if (missing.length === 0)
|
|
88
|
+
return undefined;
|
|
89
|
+
const suggestion = args.suggestLocalAgent === true && missing.includes("OPENAI_API_KEY")
|
|
90
|
+
? await signedInAgentSuggestion(env)
|
|
91
|
+
: "";
|
|
92
|
+
return {
|
|
93
|
+
code: args.code,
|
|
94
|
+
message: `${args.need(missing.join(" and "))} ${describeMissingKeys(missing, env)}${suggestion}`,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/** The sentence naming the coding agents signed in on this machine, or "" when none is. */
|
|
98
|
+
async function signedInAgentSuggestion(env) {
|
|
99
|
+
const ready = (await detectLocalAgents({ env })).filter((agent) => agent.authStatus === "authenticated");
|
|
100
|
+
if (ready.length === 0)
|
|
101
|
+
return "";
|
|
102
|
+
const labels = ready.map((agent) => agent.label).join(" and ");
|
|
103
|
+
return ` ${labels} reports authenticated on this machine. Set actor.type: local-agent to use ${ready.length === 1 ? "it" : "one"} instead of a key.`;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Refuses a local-agent brain whose CLI is missing, signed out, too old, or cannot honor the
|
|
107
|
+
* caps, before any sandbox is paid for. `codes` names the route's code for each kind of refusal.
|
|
108
|
+
*/
|
|
109
|
+
export async function localAgentCheck(args) {
|
|
110
|
+
const { brain, env, caps } = args;
|
|
111
|
+
if (brain.kind !== "local-agent")
|
|
112
|
+
return undefined;
|
|
113
|
+
const refusal = await localAgentRefusal({ agent: brain.agent, env, caps });
|
|
114
|
+
return refusal === undefined
|
|
115
|
+
? undefined
|
|
116
|
+
: { code: args.codes[refusal.kind], message: refusal.message };
|
|
117
|
+
}
|
|
118
|
+
/** Refuses a run whose environment lacks a subject env name the requirements list. */
|
|
119
|
+
export function subjectEnvCheck(args) {
|
|
120
|
+
const unset = missingSubjectEnv(args.requirements, args.env);
|
|
121
|
+
if (unset.length === 0)
|
|
122
|
+
return undefined;
|
|
123
|
+
return {
|
|
124
|
+
code: args.code,
|
|
125
|
+
message: `subject.env declares ${unset.join(", ")} but the environment does not provide ${unset.length === 1 ? "it" : "them"} (pass via --dotenv; values are never persisted).`,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Refuses a maxUsd or maxTotalUsd cap on a model with no rate in src/run/pricing.ts. The loop
|
|
130
|
+
* could not measure spend against the cap, and running uncapped would drop the protection the
|
|
131
|
+
* cap promised; the operator picks a priced model or removes the cap.
|
|
132
|
+
*/
|
|
133
|
+
export function unpricedCapCheck(args) {
|
|
134
|
+
const { caps, model } = args;
|
|
135
|
+
if (caps.maxUsd === undefined && caps.maxTotalUsd === undefined)
|
|
136
|
+
return undefined;
|
|
137
|
+
if (MODEL_RATES[model.trim().toLowerCase()])
|
|
138
|
+
return undefined;
|
|
139
|
+
return { code: args.code, message: unpricedCapMessage(model) };
|
|
140
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { RunSecrets } from "../run/secrets.js";
|
|
1
2
|
import type { CuaParticipantPlan } from "../routes/computer-use/types.js";
|
|
2
3
|
import type { SubjectPhaseEvent } from "../subject/steps.js";
|
|
3
4
|
import type { InternalRunStudyOptions } from "../run-study.js";
|
|
@@ -50,12 +51,12 @@ export type StudyEvent = {
|
|
|
50
51
|
* analysis API key. A callback can hold any of them, and its warning is appended after the route
|
|
51
52
|
* sanitized its own.
|
|
52
53
|
*/
|
|
53
|
-
export declare function
|
|
54
|
+
export declare function knownSecrets(config: StudyConfig, options: InternalRunStudyOptions, forwardedEnv: Readonly<Record<string, string | undefined>> | undefined): RunSecrets;
|
|
54
55
|
/**
|
|
55
56
|
* The emitter the route hook bags call, or undefined without onEvent. It calls onEvent and never
|
|
56
|
-
* waits for it: a throw or a rejected promise becomes a run warning, scrubbed of `
|
|
57
|
+
* waits for it: a throw or a rejected promise becomes a run warning, scrubbed of `secrets`.
|
|
57
58
|
*/
|
|
58
|
-
export declare function studyEventEmitter(onEvent: ((event: StudyEvent) => void | Promise<void>) | undefined, warnings: string[],
|
|
59
|
+
export declare function studyEventEmitter(onEvent: ((event: StudyEvent) => void | Promise<void>) | undefined, warnings: string[], secrets: () => RunSecrets): ((event: StudyEvent) => void) | undefined;
|
|
59
60
|
/** The computer-use participant plan as a `plan` event: ids, personas, devices and digests. */
|
|
60
61
|
export declare function planEvent(plan: CuaParticipantPlan): StudyEvent;
|
|
61
62
|
export declare function phaseEvent(event: SubjectPhaseEvent, target: SetupTarget): StudyEvent;
|
|
@@ -1,32 +1,28 @@
|
|
|
1
1
|
// What a run reports through RunStudyOptions.onEvent, and how a callback failure becomes a run
|
|
2
2
|
// warning. normalizeRunStudyOptions (run-study-options.ts) builds the emitter and hands it to the
|
|
3
3
|
// route hook bags it maps; the event values are built here.
|
|
4
|
-
import { redactText,
|
|
4
|
+
import { redactText, toErrorMessage } from "../evidence/redaction.js";
|
|
5
|
+
import { RunSecrets } from "../run/secrets.js";
|
|
5
6
|
/**
|
|
6
7
|
* The literal values to scrub from an onEvent warning: the provider keys and the declared subject
|
|
7
8
|
* env from every env the run could read (the `env` option and process.env), and the
|
|
8
9
|
* analysis API key. A callback can hold any of them, and its warning is appended after the route
|
|
9
10
|
* sanitized its own.
|
|
10
11
|
*/
|
|
11
|
-
export function
|
|
12
|
+
export function knownSecrets(config, options, forwardedEnv) {
|
|
12
13
|
const sources = [forwardedEnv, options.env, process.env];
|
|
13
14
|
const names = ["OPENAI_API_KEY", "E2B_API_KEY", "CODEX_API_KEY", ...(config.subject.env ?? [])];
|
|
14
|
-
const
|
|
15
|
-
const add = (value) => {
|
|
16
|
-
const trimmed = value?.trim() ?? "";
|
|
17
|
-
if (trimmed.length >= 4)
|
|
18
|
-
values.add(trimmed);
|
|
19
|
-
};
|
|
15
|
+
const secrets = new RunSecrets([]);
|
|
20
16
|
for (const env of sources)
|
|
21
17
|
for (const name of names)
|
|
22
|
-
add(env?.[name]);
|
|
23
|
-
return
|
|
18
|
+
secrets.add([env?.[name]?.trim() ?? ""]);
|
|
19
|
+
return secrets;
|
|
24
20
|
}
|
|
25
21
|
/**
|
|
26
22
|
* The emitter the route hook bags call, or undefined without onEvent. It calls onEvent and never
|
|
27
|
-
* waits for it: a throw or a rejected promise becomes a run warning, scrubbed of `
|
|
23
|
+
* waits for it: a throw or a rejected promise becomes a run warning, scrubbed of `secrets`.
|
|
28
24
|
*/
|
|
29
|
-
export function studyEventEmitter(onEvent, warnings,
|
|
25
|
+
export function studyEventEmitter(onEvent, warnings, secrets) {
|
|
30
26
|
if (onEvent === undefined)
|
|
31
27
|
return undefined;
|
|
32
28
|
return (event) => {
|
|
@@ -37,8 +33,7 @@ export function studyEventEmitter(onEvent, warnings, secretValues) {
|
|
|
37
33
|
let detail;
|
|
38
34
|
try {
|
|
39
35
|
// Read here, not up front: a run with no failing callback never touches the env.
|
|
40
|
-
|
|
41
|
-
detail = redactText(scrub(toErrorMessage(error)));
|
|
36
|
+
detail = redactText(secrets().scrub(toErrorMessage(error)));
|
|
42
37
|
}
|
|
43
38
|
catch {
|
|
44
39
|
detail = "the thrown value has no message";
|
|
@@ -23,6 +23,9 @@ export declare function removedOptionRefusal(options: RunStudyOptions): Refusal
|
|
|
23
23
|
* Nothing here touches the filesystem, so a refusal leaves no run directory, receipt or sandbox.
|
|
24
24
|
*/
|
|
25
25
|
export declare function normalizeRunStudyOptions(config: StudyConfig, route: StudyRoute, options: InternalRunStudyOptions): Normalized | Refusal;
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* A refusal in the route's own result envelope, before any run exists. Each route's own fields come
|
|
28
|
+
* after `error` here.
|
|
29
|
+
*/
|
|
27
30
|
export declare function optionRefusalOutcome(config: StudyConfig, route: StudyRoute, options: RunStudyOptions, refusal: Refusal): StudyOutcome;
|
|
28
31
|
export {};
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { isLocalBrowserStudy } from "../substrates/local/runtime-config.js";
|
|
7
7
|
import { resolveStudyDryRun } from "./plan.js";
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
8
|
+
import { knownSecrets, studyEventEmitter } from "./run-study-events.js";
|
|
9
|
+
import { refusedResult } from "../run/study-result.js";
|
|
10
10
|
import { participantList } from "./study-fields.js";
|
|
11
11
|
const unsupported = (option, route, reason) => ({
|
|
12
12
|
ok: false,
|
|
@@ -93,7 +93,7 @@ export function normalizeRunStudyOptions(config, route, options) {
|
|
|
93
93
|
// The route gets this copy, so it is the env a warning is scrubbed against, whatever the caller
|
|
94
94
|
// does to its own object afterwards.
|
|
95
95
|
const forwardedEnv = env === undefined ? undefined : { ...env };
|
|
96
|
-
const emit = studyEventEmitter(onEvent, warnings, () =>
|
|
96
|
+
const emit = studyEventEmitter(onEvent, warnings, () => knownSecrets(config, options, forwardedEnv));
|
|
97
97
|
// Every route reads its typed options directly, and onEvent reaches it as `emit`. unsupportedOption
|
|
98
98
|
// already refused an option no route reads; the preview route reads no env.
|
|
99
99
|
const normalized = { ...forwarded };
|
|
@@ -101,33 +101,29 @@ export function normalizeRunStudyOptions(config, route, options) {
|
|
|
101
101
|
normalized.env = forwardedEnv;
|
|
102
102
|
return { ok: true, options: normalized, warnings, emit };
|
|
103
103
|
}
|
|
104
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* A refusal in the route's own result envelope, before any run exists. Each route's own fields come
|
|
106
|
+
* after `error` here.
|
|
107
|
+
*/
|
|
105
108
|
export function optionRefusalOutcome(config, route, options, refusal) {
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const common = {
|
|
109
|
+
const base = {
|
|
110
|
+
studyId: config.id,
|
|
111
|
+
cwd: path.resolve(options.cwd),
|
|
112
|
+
error: { code: refusal.code, message: refusal.message },
|
|
113
|
+
};
|
|
114
|
+
const common = {
|
|
115
|
+
actor: config.actor?.type ?? "",
|
|
116
|
+
dryRun: resolveStudyDryRun(config, options.dryRun, true) ?? true,
|
|
117
|
+
runId: options.runId ?? "not-created",
|
|
118
|
+
};
|
|
112
119
|
switch (route) {
|
|
113
120
|
case "preview":
|
|
114
|
-
return {
|
|
115
|
-
route: "preview",
|
|
116
|
-
result: {
|
|
117
|
-
...studyResultIdentity("preview", config.id),
|
|
118
|
-
ok: false,
|
|
119
|
-
cwd,
|
|
120
|
-
warnings: [],
|
|
121
|
-
error,
|
|
122
|
-
},
|
|
123
|
-
};
|
|
121
|
+
return { route: "preview", result: refusedResult("preview", base, {}) };
|
|
124
122
|
case "computer-use":
|
|
125
123
|
return {
|
|
126
124
|
route: "computer-use",
|
|
127
125
|
result: {
|
|
128
|
-
...
|
|
129
|
-
...common,
|
|
130
|
-
ok: false,
|
|
126
|
+
...refusedResult("computer-use", base, common),
|
|
131
127
|
appUrl: config.subject.appUrl ?? config.subject.serve?.url ?? "",
|
|
132
128
|
lanes: [],
|
|
133
129
|
},
|
|
@@ -136,9 +132,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
136
132
|
return {
|
|
137
133
|
route: "scripted",
|
|
138
134
|
result: {
|
|
139
|
-
...
|
|
140
|
-
...common,
|
|
141
|
-
ok: false,
|
|
135
|
+
...refusedResult("scripted", base, common),
|
|
142
136
|
appUrl: config.subject.appUrl ?? "",
|
|
143
137
|
sessions: [],
|
|
144
138
|
},
|
|
@@ -147,9 +141,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
147
141
|
return {
|
|
148
142
|
route: "terminal",
|
|
149
143
|
result: {
|
|
150
|
-
...
|
|
151
|
-
...common,
|
|
152
|
-
ok: false,
|
|
144
|
+
...refusedResult("terminal", base, common),
|
|
153
145
|
product: config.subject.product?.name ?? "",
|
|
154
146
|
},
|
|
155
147
|
};
|
|
@@ -158,9 +150,7 @@ export function optionRefusalOutcome(config, route, options, refusal) {
|
|
|
158
150
|
return {
|
|
159
151
|
route: "shared-world",
|
|
160
152
|
result: {
|
|
161
|
-
...
|
|
162
|
-
...common,
|
|
163
|
-
ok: false,
|
|
153
|
+
...refusedResult("shared-world", base, common),
|
|
164
154
|
topology: "shared-world",
|
|
165
155
|
topologyMode: "concurrent",
|
|
166
156
|
roleCount: participantCount,
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { PreparedOutputRoot } from "../../run/contained-output.js";
|
|
2
|
+
import { type DesktopResourceObservation } from "./desktop-resources.js";
|
|
3
|
+
import type { E2BDesktopModule, E2BDesktopSandbox } from "./sdk.js";
|
|
4
|
+
/** The subject desktop's billed span and size, for the run's cost estimate. */
|
|
5
|
+
export interface SubjectDesktopUsage {
|
|
6
|
+
durationMs: number | undefined;
|
|
7
|
+
observation: DesktopResourceObservation | undefined;
|
|
8
|
+
killed: boolean;
|
|
9
|
+
}
|
|
10
|
+
/** The run the subject sandbox reports to. */
|
|
11
|
+
export interface SubjectSandboxRun {
|
|
12
|
+
/** The run's warnings. Acquisition and release append to it. */
|
|
13
|
+
warnings: string[];
|
|
14
|
+
scrub: (text: string) => string;
|
|
15
|
+
now: () => number;
|
|
16
|
+
}
|
|
17
|
+
/** How to create the subject sandbox. */
|
|
18
|
+
export interface SubjectSandboxRequest {
|
|
19
|
+
module: E2BDesktopModule;
|
|
20
|
+
apiKey: string;
|
|
21
|
+
requestTimeoutMs: number;
|
|
22
|
+
/**
|
|
23
|
+
* The participants' session budget. The sandbox outlives it by the provision budget, each seed
|
|
24
|
+
* step's budget and the reclamation buffer, so E2B kills it only after the run is done with it.
|
|
25
|
+
*/
|
|
26
|
+
sessionTimeoutMs: number;
|
|
27
|
+
seed: readonly {
|
|
28
|
+
readonly timeoutMs?: number | undefined;
|
|
29
|
+
}[];
|
|
30
|
+
/** The route's labels. Acquisition adds the run's owner tags. */
|
|
31
|
+
metadata: Record<string, string>;
|
|
32
|
+
/** The subject's environment at create; an empty record sets none. */
|
|
33
|
+
envs: Record<string, string>;
|
|
34
|
+
template: string | undefined;
|
|
35
|
+
/** Where the receipt goes, under the participant label `subject`. */
|
|
36
|
+
root: PreparedOutputRoot;
|
|
37
|
+
}
|
|
38
|
+
export declare class E2BSubjectSandbox {
|
|
39
|
+
sandboxId: string | undefined;
|
|
40
|
+
killed: boolean;
|
|
41
|
+
/** The scrubbed release warning when the release is unconfirmed. */
|
|
42
|
+
releaseWarning: string | undefined;
|
|
43
|
+
private readonly run;
|
|
44
|
+
private allocation;
|
|
45
|
+
private createdAtMs;
|
|
46
|
+
private tornDownAtMs;
|
|
47
|
+
private resources;
|
|
48
|
+
constructor(run: SubjectSandboxRun);
|
|
49
|
+
/**
|
|
50
|
+
* Creates the sandbox and reads its size. The receipt is on disk before this returns, so
|
|
51
|
+
* `humanish reclaim` can kill the sandbox by exact id if the process dies during provisioning.
|
|
52
|
+
*/
|
|
53
|
+
acquire(request: SubjectSandboxRequest): Promise<E2BDesktopSandbox>;
|
|
54
|
+
/** Releases the sandbox by exact id, when one was acquired. An unconfirmed release is reported. */
|
|
55
|
+
release(): Promise<void>;
|
|
56
|
+
/** The billed span and size, when a sandbox was acquired. */
|
|
57
|
+
desktopUsage(): SubjectDesktopUsage | undefined;
|
|
58
|
+
}
|