humanish 0.116.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +0 -2
- package/dist/actors/codex/restricted-participant-policy.js +4 -32
- package/dist/actors/codex/restricted-participant.d.ts +5 -0
- package/dist/actors/codex/restricted-participant.js +3 -2
- package/dist/actors/computer-use/actor.d.ts +4 -0
- package/dist/actors/computer-use/actor.js +2 -0
- package/dist/actors/computer-use/loop/actions.d.ts +4 -4
- package/dist/actors/computer-use/loop/actions.js +25 -18
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +9 -1
- package/dist/actors/computer-use/loop/trace.js +1 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -14
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/wait.d.ts +74 -0
- package/dist/actors/computer-use/wait.js +100 -0
- package/dist/actors/contract.d.ts +14 -0
- package/dist/analysis/admission.d.ts +7 -2
- package/dist/analysis/admission.js +12 -3
- package/dist/analysis/automatic-config.d.ts +6 -1
- package/dist/analysis/automatic-config.js +12 -11
- package/dist/analysis/execute.d.ts +19 -7
- package/dist/analysis/execute.js +55 -17
- package/dist/browser-control/protocol.d.ts +86 -6
- package/dist/browser-control/protocol.js +13 -6
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +5 -3
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/doctor.js +11 -1
- package/dist/cli/io.js +3 -2
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/guest/desktop-executor.js +3 -1
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/e2b-desktop/prepare.js +40 -31
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +2 -1
- package/dist/routes/computer-use/local-vm.js +5 -2
- package/dist/routes/computer-use/participant-model.js +9 -0
- package/dist/routes/computer-use/participant-runs.js +3 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +29 -9
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/computer-use/types.d.ts +1 -1
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +3 -1
- package/dist/routes/shared-world/plan.js +39 -7
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/terminal/live-sandbox.d.ts +1 -1
- package/dist/routes/terminal/live-sandbox.js +14 -11
- package/dist/routes/terminal/sandbox.d.ts +4 -3
- package/dist/routes/terminal/sandbox.js +5 -5
- package/dist/run/bundle-shape.js +2 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/projection.js +7 -2
- package/dist/run/route-shell.js +10 -5
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/automatic-analysis-plan.d.ts +28 -0
- package/dist/study/automatic-analysis-plan.js +57 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +10 -21
- package/dist/study/doctor.js +17 -3
- package/dist/study/keys.d.ts +2 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +2 -1
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +9 -1
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +7 -1
- package/dist/study/plan-base.d.ts +2 -0
- package/dist/study/plan-base.js +3 -0
- package/dist/study/plan-participants.d.ts +4 -0
- package/dist/study/plan-participants.js +6 -5
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +12 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +3 -5
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.d.ts +7 -2
- package/dist/study/validation.js +30 -14
- package/dist/study/warnings.js +12 -0
- package/dist/subject/serve.d.ts +3 -0
- package/dist/subject/serve.js +4 -1
- package/dist/substrates/detached.js +14 -7
- package/dist/substrates/e2b/desktop-executor.d.ts +0 -2
- package/dist/substrates/e2b/desktop-executor.js +3 -3
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +119 -118
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/architecture/browser-control.md +10 -4
- package/docs/contracts/run-bundle.md +7 -6
- package/docs/contracts/schemas.md +33 -19
- package/docs/contracts/study-analysis.md +2 -2
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +4 -3
- package/package.json +2 -2
|
@@ -256,6 +256,15 @@ export interface ActorConversation {
|
|
|
256
256
|
estimatedInputTokens?: number;
|
|
257
257
|
}>;
|
|
258
258
|
}
|
|
259
|
+
/** How long a computer-use participant's waits lasted, in milliseconds. */
|
|
260
|
+
export interface ActorWaitSettings {
|
|
261
|
+
/** The longest one wait action lasts (actor.maxWaitMs). A longer wait is shortened to it. */
|
|
262
|
+
maxWaitMs: number;
|
|
263
|
+
/** A wait with no duration in a turn that only waits or takes screenshots (actor.idleWaitMs). */
|
|
264
|
+
idleWaitMs: number;
|
|
265
|
+
/** A wait with no duration after the participant acts in the same turn. */
|
|
266
|
+
settleWaitMs: number;
|
|
267
|
+
}
|
|
259
268
|
export interface ActorTrace {
|
|
260
269
|
schema: typeof ACTOR_TRACE_SCHEMA;
|
|
261
270
|
provider: string;
|
|
@@ -303,6 +312,11 @@ export interface ActorTrace {
|
|
|
303
312
|
reasoningEffort: string;
|
|
304
313
|
maxOutputTokens?: number;
|
|
305
314
|
};
|
|
315
|
+
/**
|
|
316
|
+
* Additive + optional: the wait lengths the computer-use loop applied, defaults included. Absent
|
|
317
|
+
* on other producers and on bundles written before it was recorded.
|
|
318
|
+
*/
|
|
319
|
+
waitSettings?: ActorWaitSettings;
|
|
306
320
|
counts: Record<string, number>;
|
|
307
321
|
/**
|
|
308
322
|
* Additive + optional affordance record (humanish.affordance-use.v1): which kind of route
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { ModelRate } from "../run/pricing.js";
|
|
2
|
-
/**
|
|
3
|
-
|
|
2
|
+
/**
|
|
3
|
+
* When admission refuses a request, in words, for the cap as the sentence names it: `$3`, or `this`
|
|
4
|
+
* in the help of the option that sets it. Admission compares the smaller of the worst case and the
|
|
5
|
+
* expected cost with the margin, so it refuses only when both are over the cap. Every text that
|
|
6
|
+
* states the rule reads it here.
|
|
7
|
+
*/
|
|
8
|
+
export declare function admissionRule(cap: string): string;
|
|
4
9
|
export interface AnalysisRequestSize {
|
|
5
10
|
/** UTF-8 bytes of the instructions, the evidence packet and the result schema. */
|
|
6
11
|
textBytes: number;
|
|
@@ -16,7 +16,17 @@ const FRAMING_TOKENS = 2048;
|
|
|
16
16
|
const EXPECTED_OUTPUT_TOKENS = 12_000;
|
|
17
17
|
const EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT = 1_000;
|
|
18
18
|
/** Admission compares the expected cost times this margin with the cap. */
|
|
19
|
-
|
|
19
|
+
const ADMISSION_MARGIN = 1.1;
|
|
20
|
+
const MARGIN_PERCENT = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
21
|
+
/**
|
|
22
|
+
* When admission refuses a request, in words, for the cap as the sentence names it: `$3`, or `this`
|
|
23
|
+
* in the help of the option that sets it. Admission compares the smaller of the worst case and the
|
|
24
|
+
* expected cost with the margin, so it refuses only when both are over the cap. Every text that
|
|
25
|
+
* states the rule reads it here.
|
|
26
|
+
*/
|
|
27
|
+
export function admissionRule(cap) {
|
|
28
|
+
return `both its worst case and its expected cost plus a ${MARGIN_PERCENT}% margin are over ${cap}`;
|
|
29
|
+
}
|
|
20
30
|
/** Rounded up to the micro-dollar, so a small positive boundary never rounds into admission. */
|
|
21
31
|
const roundUp = (usd) => Math.ceil(usd * 1e6) / 1e6;
|
|
22
32
|
/** One request's cost. Input is priced at the highest input rate the model bills, cache writes
|
|
@@ -47,9 +57,8 @@ export function estimateAnalysisCost(rate, size) {
|
|
|
47
57
|
* This module keeps only type imports, so the Observer can bundle it.
|
|
48
58
|
*/
|
|
49
59
|
export function costRefusal(cost, runFlags, cli) {
|
|
50
|
-
const margin = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
51
60
|
return {
|
|
52
|
-
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${
|
|
61
|
+
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${MARGIN_PERCENT}% margin the expected cost is over the $${cost.maxCostUsd} cap, so no request was sent. To run it, raise the cap:`,
|
|
53
62
|
command: cli(`analyze ${runFlags} --max-cost ${Math.max(1, Math.ceil(cost.worstCaseCostUsd))}`),
|
|
54
63
|
};
|
|
55
64
|
}
|
|
@@ -36,11 +36,16 @@ export interface AutomaticAnalysisBudget {
|
|
|
36
36
|
trigger: "default" | "explicit";
|
|
37
37
|
/** The participants the expected cost range is for. */
|
|
38
38
|
participants?: number;
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* The expected cost of the analysis admission would start, from a run that keeps no evidence to
|
|
41
|
+
* one that keeps the most evidence admission admits, at most the evidence limits.
|
|
42
|
+
*/
|
|
40
43
|
expectedCostUsd?: {
|
|
41
44
|
low: number;
|
|
42
45
|
high: number;
|
|
43
46
|
};
|
|
47
|
+
/** In place of the range when admission refuses even a run that keeps no evidence: its cost. */
|
|
48
|
+
refusedFromUsd?: number;
|
|
44
49
|
}
|
|
45
50
|
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
46
51
|
* participant count it also gives the expected cost range for that many participants. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
2
|
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, isSupportedAnalysisModel, } from "./execute.js";
|
|
3
|
-
import {
|
|
3
|
+
import { admissionRule } from "./admission.js";
|
|
4
4
|
import { plural } from "../run/text.js";
|
|
5
5
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
6
6
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
@@ -123,11 +123,9 @@ export function automaticAnalysisBudget(raw, route, participants) {
|
|
|
123
123
|
if (!resolved.ok || !resolved.config)
|
|
124
124
|
return undefined;
|
|
125
125
|
const { config } = resolved;
|
|
126
|
-
const range = participants === undefined
|
|
126
|
+
const range = participants === undefined
|
|
127
127
|
? undefined
|
|
128
|
-
: analysisCostRange(resolved.preferLargerOutput
|
|
129
|
-
? { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS }
|
|
130
|
-
: config, participants);
|
|
128
|
+
: analysisCostRange(config, participants, resolved.preferLargerOutput === true);
|
|
131
129
|
return {
|
|
132
130
|
...(config.provider === "codex"
|
|
133
131
|
? { provider: "codex", billing: "account-unknown" }
|
|
@@ -135,16 +133,19 @@ export function automaticAnalysisBudget(raw, route, participants) {
|
|
|
135
133
|
model: config.model,
|
|
136
134
|
maxCostUsd: config.maxCostUsd,
|
|
137
135
|
trigger: raw === undefined ? "default" : "explicit",
|
|
138
|
-
...(range === undefined || participants === undefined
|
|
139
|
-
? {}
|
|
140
|
-
: { participants, expectedCostUsd: range }),
|
|
136
|
+
...(range === undefined || participants === undefined ? {} : { participants, ...range }),
|
|
141
137
|
};
|
|
142
138
|
}
|
|
143
139
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
144
140
|
if (budget.provider === "codex")
|
|
145
141
|
return `After live runs: Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
146
|
-
const
|
|
142
|
+
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
143
|
+
const who = budget.participants === undefined ? undefined : plural(budget.participants, "participant");
|
|
144
|
+
const range = budget.expectedCostUsd === undefined || who === undefined
|
|
147
145
|
? ""
|
|
148
|
-
: ` · expected $${budget.expectedCostUsd.low.toFixed(2)} to $${budget.expectedCostUsd.high.toFixed(2)} for ${
|
|
149
|
-
|
|
146
|
+
: ` · expected $${budget.expectedCostUsd.low.toFixed(2)} to $${budget.expectedCostUsd.high.toFixed(2)} for ${who}, depending on how much evidence the run keeps`;
|
|
147
|
+
const refused = budget.refusedFromUsd === undefined || who === undefined
|
|
148
|
+
? `refused before it starts if ${rule}`
|
|
149
|
+
: `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
|
|
150
|
+
return `After live runs: ${budget.trigger} analysis · ${budget.model}${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
150
151
|
}
|
|
@@ -43,14 +43,26 @@ export interface AnalysisProgress {
|
|
|
43
43
|
*/
|
|
44
44
|
export declare function estimateAnalysisAdmission(input: AnalysisInput, config: AnalysisConfig): AnalysisAdmission;
|
|
45
45
|
/**
|
|
46
|
-
* The expected cost of
|
|
47
|
-
* packet with no evidence to
|
|
48
|
-
*
|
|
46
|
+
* The expected cost of the analysis admission would start, before any evidence exists: from a
|
|
47
|
+
* packet with no evidence to the largest packet admission admits. When admission refuses even a
|
|
48
|
+
* packet with no evidence, `refusedFromUsd` is that packet's expected cost.
|
|
49
49
|
*/
|
|
50
|
-
export
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
export type AnalysisCostRange = {
|
|
51
|
+
expectedCostUsd: {
|
|
52
|
+
low: number;
|
|
53
|
+
high: number;
|
|
54
|
+
};
|
|
55
|
+
} | {
|
|
56
|
+
refusedFromUsd: number;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* The expected cost of the analysis admission would start for this many participants, before any
|
|
60
|
+
* evidence exists. Packets grow from no evidence toward the evidence limits with every capture at
|
|
61
|
+
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
62
|
+
* passes the cap. With `preferLargerOutput` each packet gets the output allowance dispatch would
|
|
63
|
+
* give it (preferLargerAnalysisOutput). Undefined for Codex and for a model without rates.
|
|
64
|
+
*/
|
|
65
|
+
export declare function analysisCostRange(config: AnalysisConfig, participants: number, preferLargerOutput?: boolean): AnalysisCostRange | undefined;
|
|
54
66
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
55
67
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
56
68
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
package/dist/analysis/execute.js
CHANGED
|
@@ -217,12 +217,16 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
217
217
|
/** Structure each evidence entry adds to the packet: ids, kind, timestamps and flags. The largest
|
|
218
218
|
* retained packet averaged 200 bytes. */
|
|
219
219
|
const ENTRY_STRUCTURE_BYTES = 256;
|
|
220
|
+
/** How much a range bisection narrows the share of the evidence limits: 2^-40 of them. */
|
|
221
|
+
const RANGE_BISECTIONS = 40;
|
|
220
222
|
/**
|
|
221
|
-
* The expected cost of
|
|
222
|
-
*
|
|
223
|
-
* ceiling
|
|
223
|
+
* The expected cost of the analysis admission would start for this many participants, before any
|
|
224
|
+
* evidence exists. Packets grow from no evidence toward the evidence limits with every capture at
|
|
225
|
+
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
226
|
+
* passes the cap. With `preferLargerOutput` each packet gets the output allowance dispatch would
|
|
227
|
+
* give it (preferLargerAnalysisOutput). Undefined for Codex and for a model without rates.
|
|
224
228
|
*/
|
|
225
|
-
export function analysisCostRange(config, participants) {
|
|
229
|
+
export function analysisCostRange(config, participants, preferLargerOutput = false) {
|
|
226
230
|
if (config.provider === "codex")
|
|
227
231
|
return undefined;
|
|
228
232
|
const rate = MODEL_RATES[config.model];
|
|
@@ -230,26 +234,60 @@ export function analysisCostRange(config, participants) {
|
|
|
230
234
|
return undefined;
|
|
231
235
|
const prompt = Buffer.byteLength(instructions(config)) +
|
|
232
236
|
Buffer.byteLength(JSON.stringify(analysisResultJsonSchema));
|
|
233
|
-
|
|
237
|
+
/** The request for a share of the evidence limits, from none (0) to all of them (1). */
|
|
238
|
+
const request = (share, outputAllowance) => estimateAnalysisCost(rate, {
|
|
234
239
|
participants: Math.min(participants, EVIDENCE_LIMITS.participants),
|
|
235
|
-
outputAllowance
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
...size,
|
|
240
|
-
textBytes: prompt + EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES,
|
|
241
|
-
imageTokens: EVIDENCE_LIMITS.captures * HIGH_DETAIL_IMAGE_TOKEN_CEILING,
|
|
240
|
+
outputAllowance,
|
|
241
|
+
textBytes: prompt +
|
|
242
|
+
share * (EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES),
|
|
243
|
+
imageTokens: share * EVIDENCE_LIMITS.captures * HIGH_DETAIL_IMAGE_TOKEN_CEILING,
|
|
242
244
|
});
|
|
243
|
-
|
|
245
|
+
const admits = (share, allowance) => request(share, allowance).admittedCostUsd <= config.maxCostUsd;
|
|
246
|
+
// Cost grows with the share, so admission admits each allowance up to one share of the limits.
|
|
247
|
+
const largestAdmitted = (allowance) => {
|
|
248
|
+
if (admits(1, allowance))
|
|
249
|
+
return 1;
|
|
250
|
+
let [admitted, refused] = [0, 1];
|
|
251
|
+
for (let step = 0; step < RANGE_BISECTIONS; step += 1) {
|
|
252
|
+
const share = (admitted + refused) / 2;
|
|
253
|
+
if (admits(share, allowance))
|
|
254
|
+
admitted = share;
|
|
255
|
+
else
|
|
256
|
+
refused = share;
|
|
257
|
+
}
|
|
258
|
+
return admitted;
|
|
259
|
+
};
|
|
260
|
+
// Dispatch tries the larger allowance first. A smaller allowance costs no more, so it admits at
|
|
261
|
+
// least the same share, and the packets past the larger one's share get the smaller one.
|
|
262
|
+
const larger = preferLargerOutput ? largerOutput(config) : undefined;
|
|
263
|
+
const allowances = [
|
|
264
|
+
...(larger === undefined ? [] : [larger.maxOutputTokens]),
|
|
265
|
+
config.maxOutputTokens,
|
|
266
|
+
];
|
|
267
|
+
const admitted = allowances.filter((allowance) => admits(0, allowance));
|
|
268
|
+
if (admitted.length === 0)
|
|
269
|
+
return { refusedFromUsd: request(0, config.maxOutputTokens).expectedCostUsd };
|
|
270
|
+
return {
|
|
271
|
+
expectedCostUsd: {
|
|
272
|
+
low: request(0, admitted[0]).expectedCostUsd,
|
|
273
|
+
high: Math.max(...admitted.map((allowance) => request(largestAdmitted(allowance), allowance).expectedCostUsd)),
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
/** The larger output allowance an omitted output limit may get, or undefined when it may not. */
|
|
278
|
+
function largerOutput(config) {
|
|
279
|
+
if (config.provider === "codex" || config.maxOutputTokens !== DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS)
|
|
280
|
+
return undefined;
|
|
281
|
+
return { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS };
|
|
244
282
|
}
|
|
245
283
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
246
284
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
247
285
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
248
286
|
export function preferLargerAnalysisOutput(input, config) {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
287
|
+
const expanded = largerOutput(config);
|
|
288
|
+
return expanded !== undefined && estimateAnalysisAdmission(input, expanded).allowed
|
|
289
|
+
? expanded
|
|
290
|
+
: config;
|
|
253
291
|
}
|
|
254
292
|
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
|
|
255
293
|
* known value is found as written and in its encoded forms. */
|
|
@@ -26,7 +26,7 @@ export interface BrowserControlIdentity {
|
|
|
26
26
|
challenge: string;
|
|
27
27
|
runtimeRevision: string;
|
|
28
28
|
}
|
|
29
|
-
export declare const
|
|
29
|
+
export declare const participantActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<readonly [z.ZodObject<{
|
|
30
30
|
x: z.ZodNumber;
|
|
31
31
|
y: z.ZodNumber;
|
|
32
32
|
kind: z.ZodLiteral<"click">;
|
|
@@ -66,8 +66,88 @@ export declare const browserOnlyControlActionSchema: z.ZodDiscriminatedUnion<rea
|
|
|
66
66
|
ms: z.ZodOptional<z.ZodNumber>;
|
|
67
67
|
}, z.core.$strict>, z.ZodObject<{
|
|
68
68
|
kind: z.ZodLiteral<"screenshot">;
|
|
69
|
-
}, z.core.$strict>], "kind"
|
|
70
|
-
|
|
69
|
+
}, z.core.$strict>], "kind">, z.ZodTransform<{
|
|
70
|
+
kind: "click";
|
|
71
|
+
x: number;
|
|
72
|
+
y: number;
|
|
73
|
+
button?: "left" | "right" | "middle";
|
|
74
|
+
heldKeys?: string[];
|
|
75
|
+
} | {
|
|
76
|
+
kind: "double_click";
|
|
77
|
+
x: number;
|
|
78
|
+
y: number;
|
|
79
|
+
heldKeys?: string[];
|
|
80
|
+
} | {
|
|
81
|
+
kind: "move";
|
|
82
|
+
x: number;
|
|
83
|
+
y: number;
|
|
84
|
+
heldKeys?: string[];
|
|
85
|
+
} | {
|
|
86
|
+
kind: "scroll";
|
|
87
|
+
x: number;
|
|
88
|
+
y: number;
|
|
89
|
+
dx: number;
|
|
90
|
+
dy: number;
|
|
91
|
+
heldKeys?: string[];
|
|
92
|
+
} | {
|
|
93
|
+
kind: "type";
|
|
94
|
+
text: string;
|
|
95
|
+
} | {
|
|
96
|
+
kind: "keypress";
|
|
97
|
+
keys: string[];
|
|
98
|
+
} | {
|
|
99
|
+
kind: "drag";
|
|
100
|
+
path: Array<{
|
|
101
|
+
x: number;
|
|
102
|
+
y: number;
|
|
103
|
+
}>;
|
|
104
|
+
heldKeys?: string[];
|
|
105
|
+
} | {
|
|
106
|
+
kind: "wait";
|
|
107
|
+
ms?: number;
|
|
108
|
+
} | {
|
|
109
|
+
kind: "speak";
|
|
110
|
+
text: string;
|
|
111
|
+
} | {
|
|
112
|
+
kind: "screenshot";
|
|
113
|
+
}, {
|
|
114
|
+
kind: "type";
|
|
115
|
+
text: string;
|
|
116
|
+
} | {
|
|
117
|
+
kind: "keypress";
|
|
118
|
+
keys: string[];
|
|
119
|
+
} | {
|
|
120
|
+
kind: "screenshot";
|
|
121
|
+
} | {
|
|
122
|
+
x: number;
|
|
123
|
+
y: number;
|
|
124
|
+
kind: "click";
|
|
125
|
+
button?: "left" | "middle" | "right" | undefined;
|
|
126
|
+
} | {
|
|
127
|
+
x: number;
|
|
128
|
+
y: number;
|
|
129
|
+
kind: "double_click";
|
|
130
|
+
} | {
|
|
131
|
+
x: number;
|
|
132
|
+
y: number;
|
|
133
|
+
kind: "move";
|
|
134
|
+
} | {
|
|
135
|
+
x: number;
|
|
136
|
+
y: number;
|
|
137
|
+
kind: "scroll";
|
|
138
|
+
dx: number;
|
|
139
|
+
dy: number;
|
|
140
|
+
} | {
|
|
141
|
+
kind: "drag";
|
|
142
|
+
path: {
|
|
143
|
+
x: number;
|
|
144
|
+
y: number;
|
|
145
|
+
}[];
|
|
146
|
+
} | {
|
|
147
|
+
kind: "wait";
|
|
148
|
+
ms?: number | undefined;
|
|
149
|
+
}>>;
|
|
150
|
+
export declare const participantSpeechActionSchema: z.ZodPipe<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
71
151
|
x: z.ZodNumber;
|
|
72
152
|
y: z.ZodNumber;
|
|
73
153
|
kind: z.ZodLiteral<"click">;
|
|
@@ -160,9 +240,6 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
|
|
|
160
240
|
} | {
|
|
161
241
|
kind: "keypress";
|
|
162
242
|
keys: string[];
|
|
163
|
-
} | {
|
|
164
|
-
kind: "wait";
|
|
165
|
-
ms?: number | undefined;
|
|
166
243
|
} | {
|
|
167
244
|
kind: "screenshot";
|
|
168
245
|
} | {
|
|
@@ -193,6 +270,9 @@ export declare const browserControlActionSchema: z.ZodPipe<z.ZodDiscriminatedUni
|
|
|
193
270
|
x: number;
|
|
194
271
|
y: number;
|
|
195
272
|
}[];
|
|
273
|
+
} | {
|
|
274
|
+
kind: "wait";
|
|
275
|
+
ms?: number | undefined;
|
|
196
276
|
}>>;
|
|
197
277
|
declare const observationSchema: z.ZodObject<{
|
|
198
278
|
png: z.ZodString;
|
|
@@ -64,13 +64,18 @@ const dragAction = z.strictObject({
|
|
|
64
64
|
});
|
|
65
65
|
const typeAction = z.strictObject({ kind: z.literal("type"), text });
|
|
66
66
|
const keypressAction = z.strictObject({ kind: z.literal("keypress"), keys: chord });
|
|
67
|
+
const waitMs = z.number().finite().min(0);
|
|
68
|
+
// One request carries one wait, so the wire bounds it by the request deadline.
|
|
67
69
|
const waitAction = z.strictObject({
|
|
68
70
|
kind: z.literal("wait"),
|
|
69
|
-
ms:
|
|
71
|
+
ms: waitMs.max(BROWSER_CONTROL_LIMITS.waitMs).optional(),
|
|
70
72
|
});
|
|
73
|
+
// A participant may ask for any wait: the loop shortens it to the study's longest and sends it in
|
|
74
|
+
// steps the wire accepts.
|
|
75
|
+
const participantWaitAction = z.strictObject({ kind: z.literal("wait"), ms: waitMs.optional() });
|
|
71
76
|
const screenshotAction = z.strictObject({ kind: z.literal("screenshot") });
|
|
72
77
|
const speakAction = z.strictObject({ kind: z.literal("speak"), text: speechText });
|
|
73
|
-
const
|
|
78
|
+
const participantActionSchemas = [
|
|
74
79
|
clickAction,
|
|
75
80
|
doubleClickAction,
|
|
76
81
|
moveAction,
|
|
@@ -78,13 +83,15 @@ const browserActionSchemas = [
|
|
|
78
83
|
typeAction,
|
|
79
84
|
keypressAction,
|
|
80
85
|
dragAction,
|
|
81
|
-
|
|
86
|
+
participantWaitAction,
|
|
82
87
|
screenshotAction,
|
|
83
88
|
];
|
|
84
89
|
// The Codex participant's tool schema: no held keys, so its tool surface stays as published.
|
|
85
|
-
export const
|
|
86
|
-
|
|
87
|
-
.
|
|
90
|
+
export const participantActionSchema = z
|
|
91
|
+
.discriminatedUnion("kind", participantActionSchemas)
|
|
92
|
+
.transform(toCuaAction);
|
|
93
|
+
export const participantSpeechActionSchema = z
|
|
94
|
+
.discriminatedUnion("kind", [...participantActionSchemas, speakAction])
|
|
88
95
|
.transform(toCuaAction);
|
|
89
96
|
// What crosses the host-guest wire: the participant actions plus held keys on pointer actions,
|
|
90
97
|
// which the OpenAI provider maps from its computer tool.
|
|
@@ -2,6 +2,7 @@ import { resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIME
|
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
5
|
+
import { admissionRule } from "../../analysis/admission.js";
|
|
5
6
|
import { listAnalyses } from "../../analysis/store.js";
|
|
6
7
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
7
8
|
import { resolveRunPath } from "../../run/locate.js";
|
|
@@ -30,7 +31,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
30
31
|
.option("--run <id>", RUN_OPTION_DESCRIPTION, "latest")
|
|
31
32
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
32
33
|
.option("--provider <name>", "Analyst: openai (default) or restricted codex account. No provider fallback.")
|
|
33
|
-
.option("--max-cost <usd>",
|
|
34
|
+
.option("--max-cost <usd>", `Required for OpenAI, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
|
|
34
35
|
.option("--model <id>", "Supported vision model. OpenAI uses high effort; qualified Codex account analysis uses low effort.", DEFAULT_ANALYSIS_MODEL)
|
|
35
36
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
36
37
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
3
4
|
/**
|
|
4
5
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
5
6
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
@@ -38,16 +39,12 @@ function sandboxIdsLine(result) {
|
|
|
38
39
|
: [];
|
|
39
40
|
}
|
|
40
41
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* made before the route resolved a URL has none either, so it names the URL the study declares.
|
|
42
|
+
* The `subject:` line's name for the study's declared subject. A refusal made before the route
|
|
43
|
+
* resolved a URL has none, so it names the URL the study declares.
|
|
44
44
|
*/
|
|
45
|
-
function
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (subject.product !== undefined)
|
|
49
|
-
return `${subject.product.name} (${subject.source})`;
|
|
50
|
-
return subject.appUrl ?? subject.serve?.url ?? subject.source;
|
|
45
|
+
function subjectLine(appUrl, subject) {
|
|
46
|
+
const named = subjectName(appUrl, { source: subject.source, product: subject.product?.name });
|
|
47
|
+
return named !== "" ? named : (subject.appUrl ?? subject.serve?.url ?? subject.source);
|
|
51
48
|
}
|
|
52
49
|
/** A participant's status in words: a dry run's placeholder status says that nothing ran live. */
|
|
53
50
|
function participantStatus(status) {
|
|
@@ -107,7 +104,7 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
107
104
|
...runHeader(result, "scripted"),
|
|
108
105
|
`run: ${result.runId}`,
|
|
109
106
|
`actor: ${result.actor}`,
|
|
110
|
-
`subject: ${
|
|
107
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
111
108
|
...(result.scenario
|
|
112
109
|
? [
|
|
113
110
|
`scenario: ${result.scenario.id} @ ${result.scenario.sourceDigest.slice(0, 12)} (${result.scenario.source}, ${result.scenario.steps} step${result.scenario.steps === 1 ? "" : "s"})`,
|
|
@@ -133,7 +130,7 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
133
130
|
...runHeader(result, "computer-use"),
|
|
134
131
|
`run: ${result.runId}`,
|
|
135
132
|
`actor: ${result.actor}`,
|
|
136
|
-
`subject: ${
|
|
133
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
137
134
|
...(result.subject?.source === "clone"
|
|
138
135
|
? [
|
|
139
136
|
`repo: ${result.subject.repo}${result.subject.commit ? `@${result.subject.commit.slice(0, 12)}` : ""}${result.subject.envNames && result.subject.envNames.length > 0 ? ` env=[${result.subject.envNames.join(", ")}]` : ""}`,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import {} from "../../run-study.js";
|
|
4
4
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
@@ -237,7 +237,7 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
writeResult(args.command, args.io, output, (value) => formatCuaStudyHuman(value, args.config.subject));
|
|
240
|
-
args.io.setExitCode(result.ok &&
|
|
240
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
241
241
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
242
242
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
243
243
|
const followResult = output.observer?.ok
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function scriptedRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, (value) => formatScriptedStudyHuman(value, args.config.subject));
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
41
41
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
4
4
|
import { serveObserver } from "../../observer/render.js";
|
|
@@ -39,7 +39,9 @@ export function sharedWorldRouteRun(args) {
|
|
|
39
39
|
topology: "shared-world",
|
|
40
40
|
topologyMode: "concurrent",
|
|
41
41
|
roleCount: participantList(args.config)?.length ?? 0,
|
|
42
|
-
|
|
42
|
+
// Refused before it planned: the declared cap, else every participant.
|
|
43
|
+
concurrency: args.config.execution?.concurrency ??
|
|
44
|
+
Math.max(1, participantList(args.config)?.length ?? 0),
|
|
43
45
|
dryRun,
|
|
44
46
|
runId: runId ?? args.options.runId ?? "not-created",
|
|
45
47
|
roles: [],
|
|
@@ -119,7 +121,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
119
121
|
};
|
|
120
122
|
}
|
|
121
123
|
writeResult(args.command, args.io, output, formatConcurrentSharedWorldStudyHuman);
|
|
122
|
-
args.io.setExitCode(result.ok &&
|
|
124
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
123
125
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
124
126
|
if (server && output.observer?.ok) {
|
|
125
127
|
await followObserver(args.io, output.observer, server);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function terminalRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, formatTerminalStudyHuman);
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
41
41
|
await showObserver({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../analysis/automatic-config.js";
|
|
2
1
|
import { resolve } from "node:path";
|
|
3
2
|
import { Command } from "commander";
|
|
4
3
|
import { deriveRunFacts } from "../telemetry.js";
|
|
5
4
|
import { resolveStudyManifest } from "../../study/discover.js";
|
|
5
|
+
import { formatStudyAnalysisBudget, studyAnalysisBudget, } from "../../study/automatic-analysis-plan.js";
|
|
6
6
|
import { warnAndQueue } from "../deprecations.js";
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
|
@@ -131,9 +131,9 @@ export async function runStudyCommand(args) {
|
|
|
131
131
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
132
132
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
133
133
|
}
|
|
134
|
-
const analysisBudget =
|
|
134
|
+
const analysisBudget = studyAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
135
135
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
136
|
-
args.io.writeErr(`${
|
|
136
|
+
args.io.writeErr(`${formatStudyAnalysisBudget(analysisBudget)}\n`);
|
|
137
137
|
}
|
|
138
138
|
return scorer === undefined ? {} : { scorer };
|
|
139
139
|
},
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { formatStudyAnalysisBudget } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
@@ -136,7 +136,7 @@ function formatStudyCheckHuman(result) {
|
|
|
136
136
|
`targets: ${checkedTargets.length ? `${reachableTargets.length}/${checkedTargets.length} reachable` : `${result.targets.length} declared, not checked`}`,
|
|
137
137
|
...(blockedTargets.length ? [`blocked-targets: ${blockedTargets.length}`] : []),
|
|
138
138
|
`spend: ${result.spend.e2bDesktop ? "one e2b desktop, no model calls" : "none"}`,
|
|
139
|
-
...(result.analysis ? [
|
|
139
|
+
...(result.analysis ? [formatStudyAnalysisBudget(result.analysis)] : []),
|
|
140
140
|
...(result.sandbox.created
|
|
141
141
|
? [`sandbox: created=yes killed=${result.sandbox.killed === true ? "yes" : "no"}`]
|
|
142
142
|
: []),
|