humanish 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/analysis/admission.d.ts +7 -2
- package/dist/analysis/admission.js +12 -3
- package/dist/analysis/automatic-config.d.ts +6 -1
- package/dist/analysis/automatic-config.js +12 -11
- package/dist/analysis/execute.d.ts +19 -7
- package/dist/analysis/execute.js +55 -17
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +5 -3
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/doctor.js +11 -1
- package/dist/cli/io.js +3 -2
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +2 -1
- package/dist/routes/computer-use/participant-runs.js +3 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +25 -7
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +3 -1
- package/dist/routes/shared-world/plan.js +37 -6
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/run/bundle-shape.js +2 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/projection.js +7 -2
- package/dist/run/route-shell.js +10 -5
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/automatic-analysis-plan.d.ts +28 -0
- package/dist/study/automatic-analysis-plan.js +57 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +6 -20
- package/dist/study/doctor.js +17 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +1 -1
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +7 -1
- package/dist/study/plan-base.d.ts +2 -0
- package/dist/study/plan-base.js +3 -0
- package/dist/study/plan-participants.js +2 -5
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +12 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +3 -5
- package/dist/study/validation.d.ts +2 -2
- package/dist/study/validation.js +22 -14
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +112 -112
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/contracts/run-bundle.md +7 -6
- package/docs/contracts/schemas.md +14 -8
- package/docs/contracts/study-analysis.md +2 -2
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +4 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,27 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.118.0: Up to 100 participants, E2B concurrent sandboxes, study check plans like run, analysis range (2026-10-09)
|
|
12
|
+
|
|
13
|
+
humanish 0.118.0 lets a computer-use study have up to 100 participants, up from 16, holds a
|
|
14
|
+
shared-world study to the same 100, and runs them within the number of sandboxes the E2B plan runs
|
|
15
|
+
at once. The new setting `HUMANISH_E2B_MAX_CONCURRENT_SANDBOXES` gives that number (20 when unset,
|
|
16
|
+
E2B Hobby's limit; Pro runs 100). A study with more participants than that runs them in waves, an
|
|
17
|
+
`execution.concurrency` above it is refused before any sandbox is created, and `doctor` shows it. A
|
|
18
|
+
live run of more than 16 participants records its automatic analysis as skipped with
|
|
19
|
+
`AUTOMATIC_ANALYSIS_PARTICIPANT_LIMIT`, because one analysis reads at most 16 participants. `study
|
|
20
|
+
check` now plans a study the way `humanish run` does and fails a study the run refuses, and a
|
|
21
|
+
shared-world or scripted clone study whose sandbox deadline passes the ceiling is refused at plan
|
|
22
|
+
time. The analysis line that `run`, `study check` and the TUI print ends its expected range at the
|
|
23
|
+
largest run admission admits (`$0.75 to $2.83` for one participant at the default $3 cap, where it
|
|
24
|
+
read `$3.79`) and states the rule as admission applies it. An in-process `local-app` run records
|
|
25
|
+
`subject.source: local-app`, a desktop-cli dry run names its product, and `verify` names an unlisted
|
|
26
|
+
folder and the 10,000-entry stop in their own sentences. In `humanish tui`, the key legend says what
|
|
27
|
+
Enter and Esc do while an action is armed, a study's median cost includes its analyses, and Esc
|
|
28
|
+
after entering a key returns to the screen the keys were opened from.
|
|
29
|
+
|
|
30
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.118.0)
|
|
31
|
+
|
|
11
32
|
## 0.117.0: Long participant waits, OpenAI wait length, serve.build fails fast, TUI confirmations (2026-10-09)
|
|
12
33
|
|
|
13
34
|
humanish 0.117.0 lets a participant wait through a long hosted study, gives an OpenAI
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import type { ModelRate } from "../run/pricing.js";
|
|
2
|
-
/**
|
|
3
|
-
|
|
2
|
+
/**
|
|
3
|
+
* When admission refuses a request, in words, for the cap as the sentence names it: `$3`, or `this`
|
|
4
|
+
* in the help of the option that sets it. Admission compares the smaller of the worst case and the
|
|
5
|
+
* expected cost with the margin, so it refuses only when both are over the cap. Every text that
|
|
6
|
+
* states the rule reads it here.
|
|
7
|
+
*/
|
|
8
|
+
export declare function admissionRule(cap: string): string;
|
|
4
9
|
export interface AnalysisRequestSize {
|
|
5
10
|
/** UTF-8 bytes of the instructions, the evidence packet and the result schema. */
|
|
6
11
|
textBytes: number;
|
|
@@ -16,7 +16,17 @@ const FRAMING_TOKENS = 2048;
|
|
|
16
16
|
const EXPECTED_OUTPUT_TOKENS = 12_000;
|
|
17
17
|
const EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT = 1_000;
|
|
18
18
|
/** Admission compares the expected cost times this margin with the cap. */
|
|
19
|
-
|
|
19
|
+
const ADMISSION_MARGIN = 1.1;
|
|
20
|
+
const MARGIN_PERCENT = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
21
|
+
/**
|
|
22
|
+
* When admission refuses a request, in words, for the cap as the sentence names it: `$3`, or `this`
|
|
23
|
+
* in the help of the option that sets it. Admission compares the smaller of the worst case and the
|
|
24
|
+
* expected cost with the margin, so it refuses only when both are over the cap. Every text that
|
|
25
|
+
* states the rule reads it here.
|
|
26
|
+
*/
|
|
27
|
+
export function admissionRule(cap) {
|
|
28
|
+
return `both its worst case and its expected cost plus a ${MARGIN_PERCENT}% margin are over ${cap}`;
|
|
29
|
+
}
|
|
20
30
|
/** Rounded up to the micro-dollar, so a small positive boundary never rounds into admission. */
|
|
21
31
|
const roundUp = (usd) => Math.ceil(usd * 1e6) / 1e6;
|
|
22
32
|
/** One request's cost. Input is priced at the highest input rate the model bills, cache writes
|
|
@@ -47,9 +57,8 @@ export function estimateAnalysisCost(rate, size) {
|
|
|
47
57
|
* This module keeps only type imports, so the Observer can bundle it.
|
|
48
58
|
*/
|
|
49
59
|
export function costRefusal(cost, runFlags, cli) {
|
|
50
|
-
const margin = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
51
60
|
return {
|
|
52
|
-
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${
|
|
61
|
+
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${MARGIN_PERCENT}% margin the expected cost is over the $${cost.maxCostUsd} cap, so no request was sent. To run it, raise the cap:`,
|
|
53
62
|
command: cli(`analyze ${runFlags} --max-cost ${Math.max(1, Math.ceil(cost.worstCaseCostUsd))}`),
|
|
54
63
|
};
|
|
55
64
|
}
|
|
@@ -36,11 +36,16 @@ export interface AutomaticAnalysisBudget {
|
|
|
36
36
|
trigger: "default" | "explicit";
|
|
37
37
|
/** The participants the expected cost range is for. */
|
|
38
38
|
participants?: number;
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* The expected cost of the analysis admission would start, from a run that keeps no evidence to
|
|
41
|
+
* one that keeps the most evidence admission admits, at most the evidence limits.
|
|
42
|
+
*/
|
|
40
43
|
expectedCostUsd?: {
|
|
41
44
|
low: number;
|
|
42
45
|
high: number;
|
|
43
46
|
};
|
|
47
|
+
/** In place of the range when admission refuses even a run that keeps no evidence: its cost. */
|
|
48
|
+
refusedFromUsd?: number;
|
|
44
49
|
}
|
|
45
50
|
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
46
51
|
* participant count it also gives the expected cost range for that many participants. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
2
|
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, isSupportedAnalysisModel, } from "./execute.js";
|
|
3
|
-
import {
|
|
3
|
+
import { admissionRule } from "./admission.js";
|
|
4
4
|
import { plural } from "../run/text.js";
|
|
5
5
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
6
6
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
@@ -123,11 +123,9 @@ export function automaticAnalysisBudget(raw, route, participants) {
|
|
|
123
123
|
if (!resolved.ok || !resolved.config)
|
|
124
124
|
return undefined;
|
|
125
125
|
const { config } = resolved;
|
|
126
|
-
const range = participants === undefined
|
|
126
|
+
const range = participants === undefined
|
|
127
127
|
? undefined
|
|
128
|
-
: analysisCostRange(resolved.preferLargerOutput
|
|
129
|
-
? { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS }
|
|
130
|
-
: config, participants);
|
|
128
|
+
: analysisCostRange(config, participants, resolved.preferLargerOutput === true);
|
|
131
129
|
return {
|
|
132
130
|
...(config.provider === "codex"
|
|
133
131
|
? { provider: "codex", billing: "account-unknown" }
|
|
@@ -135,16 +133,19 @@ export function automaticAnalysisBudget(raw, route, participants) {
|
|
|
135
133
|
model: config.model,
|
|
136
134
|
maxCostUsd: config.maxCostUsd,
|
|
137
135
|
trigger: raw === undefined ? "default" : "explicit",
|
|
138
|
-
...(range === undefined || participants === undefined
|
|
139
|
-
? {}
|
|
140
|
-
: { participants, expectedCostUsd: range }),
|
|
136
|
+
...(range === undefined || participants === undefined ? {} : { participants, ...range }),
|
|
141
137
|
};
|
|
142
138
|
}
|
|
143
139
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
144
140
|
if (budget.provider === "codex")
|
|
145
141
|
return `After live runs: Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
146
|
-
const
|
|
142
|
+
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
143
|
+
const who = budget.participants === undefined ? undefined : plural(budget.participants, "participant");
|
|
144
|
+
const range = budget.expectedCostUsd === undefined || who === undefined
|
|
147
145
|
? ""
|
|
148
|
-
: ` · expected $${budget.expectedCostUsd.low.toFixed(2)} to $${budget.expectedCostUsd.high.toFixed(2)} for ${
|
|
149
|
-
|
|
146
|
+
: ` · expected $${budget.expectedCostUsd.low.toFixed(2)} to $${budget.expectedCostUsd.high.toFixed(2)} for ${who}, depending on how much evidence the run keeps`;
|
|
147
|
+
const refused = budget.refusedFromUsd === undefined || who === undefined
|
|
148
|
+
? `refused before it starts if ${rule}`
|
|
149
|
+
: `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
|
|
150
|
+
return `After live runs: ${budget.trigger} analysis · ${budget.model}${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
150
151
|
}
|
|
@@ -43,14 +43,26 @@ export interface AnalysisProgress {
|
|
|
43
43
|
*/
|
|
44
44
|
export declare function estimateAnalysisAdmission(input: AnalysisInput, config: AnalysisConfig): AnalysisAdmission;
|
|
45
45
|
/**
|
|
46
|
-
* The expected cost of
|
|
47
|
-
* packet with no evidence to
|
|
48
|
-
*
|
|
46
|
+
* The expected cost of the analysis admission would start, before any evidence exists: from a
|
|
47
|
+
* packet with no evidence to the largest packet admission admits. When admission refuses even a
|
|
48
|
+
* packet with no evidence, `refusedFromUsd` is that packet's expected cost.
|
|
49
49
|
*/
|
|
50
|
-
export
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
export type AnalysisCostRange = {
|
|
51
|
+
expectedCostUsd: {
|
|
52
|
+
low: number;
|
|
53
|
+
high: number;
|
|
54
|
+
};
|
|
55
|
+
} | {
|
|
56
|
+
refusedFromUsd: number;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* The expected cost of the analysis admission would start for this many participants, before any
|
|
60
|
+
* evidence exists. Packets grow from no evidence toward the evidence limits with every capture at
|
|
61
|
+
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
62
|
+
* passes the cap. With `preferLargerOutput` each packet gets the output allowance dispatch would
|
|
63
|
+
* give it (preferLargerAnalysisOutput). Undefined for Codex and for a model without rates.
|
|
64
|
+
*/
|
|
65
|
+
export declare function analysisCostRange(config: AnalysisConfig, participants: number, preferLargerOutput?: boolean): AnalysisCostRange | undefined;
|
|
54
66
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
55
67
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
56
68
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
package/dist/analysis/execute.js
CHANGED
|
@@ -217,12 +217,16 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
217
217
|
/** Structure each evidence entry adds to the packet: ids, kind, timestamps and flags. The largest
|
|
218
218
|
* retained packet averaged 200 bytes. */
|
|
219
219
|
const ENTRY_STRUCTURE_BYTES = 256;
|
|
220
|
+
/** How much a range bisection narrows the share of the evidence limits: 2^-40 of them. */
|
|
221
|
+
const RANGE_BISECTIONS = 40;
|
|
220
222
|
/**
|
|
221
|
-
* The expected cost of
|
|
222
|
-
*
|
|
223
|
-
* ceiling
|
|
223
|
+
* The expected cost of the analysis admission would start for this many participants, before any
|
|
224
|
+
* evidence exists. Packets grow from no evidence toward the evidence limits with every capture at
|
|
225
|
+
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
226
|
+
* passes the cap. With `preferLargerOutput` each packet gets the output allowance dispatch would
|
|
227
|
+
* give it (preferLargerAnalysisOutput). Undefined for Codex and for a model without rates.
|
|
224
228
|
*/
|
|
225
|
-
export function analysisCostRange(config, participants) {
|
|
229
|
+
export function analysisCostRange(config, participants, preferLargerOutput = false) {
|
|
226
230
|
if (config.provider === "codex")
|
|
227
231
|
return undefined;
|
|
228
232
|
const rate = MODEL_RATES[config.model];
|
|
@@ -230,26 +234,60 @@ export function analysisCostRange(config, participants) {
|
|
|
230
234
|
return undefined;
|
|
231
235
|
const prompt = Buffer.byteLength(instructions(config)) +
|
|
232
236
|
Buffer.byteLength(JSON.stringify(analysisResultJsonSchema));
|
|
233
|
-
|
|
237
|
+
/** The request for a share of the evidence limits, from none (0) to all of them (1). */
|
|
238
|
+
const request = (share, outputAllowance) => estimateAnalysisCost(rate, {
|
|
234
239
|
participants: Math.min(participants, EVIDENCE_LIMITS.participants),
|
|
235
|
-
outputAllowance
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
...size,
|
|
240
|
-
textBytes: prompt + EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES,
|
|
241
|
-
imageTokens: EVIDENCE_LIMITS.captures * HIGH_DETAIL_IMAGE_TOKEN_CEILING,
|
|
240
|
+
outputAllowance,
|
|
241
|
+
textBytes: prompt +
|
|
242
|
+
share * (EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES),
|
|
243
|
+
imageTokens: share * EVIDENCE_LIMITS.captures * HIGH_DETAIL_IMAGE_TOKEN_CEILING,
|
|
242
244
|
});
|
|
243
|
-
|
|
245
|
+
const admits = (share, allowance) => request(share, allowance).admittedCostUsd <= config.maxCostUsd;
|
|
246
|
+
// Cost grows with the share, so admission admits each allowance up to one share of the limits.
|
|
247
|
+
const largestAdmitted = (allowance) => {
|
|
248
|
+
if (admits(1, allowance))
|
|
249
|
+
return 1;
|
|
250
|
+
let [admitted, refused] = [0, 1];
|
|
251
|
+
for (let step = 0; step < RANGE_BISECTIONS; step += 1) {
|
|
252
|
+
const share = (admitted + refused) / 2;
|
|
253
|
+
if (admits(share, allowance))
|
|
254
|
+
admitted = share;
|
|
255
|
+
else
|
|
256
|
+
refused = share;
|
|
257
|
+
}
|
|
258
|
+
return admitted;
|
|
259
|
+
};
|
|
260
|
+
// Dispatch tries the larger allowance first. A smaller allowance costs no more, so it admits at
|
|
261
|
+
// least the same share, and the packets past the larger one's share get the smaller one.
|
|
262
|
+
const larger = preferLargerOutput ? largerOutput(config) : undefined;
|
|
263
|
+
const allowances = [
|
|
264
|
+
...(larger === undefined ? [] : [larger.maxOutputTokens]),
|
|
265
|
+
config.maxOutputTokens,
|
|
266
|
+
];
|
|
267
|
+
const admitted = allowances.filter((allowance) => admits(0, allowance));
|
|
268
|
+
if (admitted.length === 0)
|
|
269
|
+
return { refusedFromUsd: request(0, config.maxOutputTokens).expectedCostUsd };
|
|
270
|
+
return {
|
|
271
|
+
expectedCostUsd: {
|
|
272
|
+
low: request(0, admitted[0]).expectedCostUsd,
|
|
273
|
+
high: Math.max(...admitted.map((allowance) => request(largestAdmitted(allowance), allowance).expectedCostUsd)),
|
|
274
|
+
},
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
/** The larger output allowance an omitted output limit may get, or undefined when it may not. */
|
|
278
|
+
function largerOutput(config) {
|
|
279
|
+
if (config.provider === "codex" || config.maxOutputTokens !== DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS)
|
|
280
|
+
return undefined;
|
|
281
|
+
return { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS };
|
|
244
282
|
}
|
|
245
283
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
246
284
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
247
285
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
248
286
|
export function preferLargerAnalysisOutput(input, config) {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
287
|
+
const expanded = largerOutput(config);
|
|
288
|
+
return expanded !== undefined && estimateAnalysisAdmission(input, expanded).allowed
|
|
289
|
+
? expanded
|
|
290
|
+
: config;
|
|
253
291
|
}
|
|
254
292
|
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
|
|
255
293
|
* known value is found as written and in its encoded forms. */
|
|
@@ -2,6 +2,7 @@ import { resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIME
|
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
5
|
+
import { admissionRule } from "../../analysis/admission.js";
|
|
5
6
|
import { listAnalyses } from "../../analysis/store.js";
|
|
6
7
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
7
8
|
import { resolveRunPath } from "../../run/locate.js";
|
|
@@ -30,7 +31,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
30
31
|
.option("--run <id>", RUN_OPTION_DESCRIPTION, "latest")
|
|
31
32
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
32
33
|
.option("--provider <name>", "Analyst: openai (default) or restricted codex account. No provider fallback.")
|
|
33
|
-
.option("--max-cost <usd>",
|
|
34
|
+
.option("--max-cost <usd>", `Required for OpenAI, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
|
|
34
35
|
.option("--model <id>", "Supported vision model. OpenAI uses high effort; qualified Codex account analysis uses low effort.", DEFAULT_ANALYSIS_MODEL)
|
|
35
36
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
36
37
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
3
4
|
/**
|
|
4
5
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
5
6
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
@@ -38,16 +39,12 @@ function sandboxIdsLine(result) {
|
|
|
38
39
|
: [];
|
|
39
40
|
}
|
|
40
41
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* made before the route resolved a URL has none either, so it names the URL the study declares.
|
|
42
|
+
* The `subject:` line's name for the study's declared subject. A refusal made before the route
|
|
43
|
+
* resolved a URL has none, so it names the URL the study declares.
|
|
44
44
|
*/
|
|
45
|
-
function
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (subject.product !== undefined)
|
|
49
|
-
return `${subject.product.name} (${subject.source})`;
|
|
50
|
-
return subject.appUrl ?? subject.serve?.url ?? subject.source;
|
|
45
|
+
function subjectLine(appUrl, subject) {
|
|
46
|
+
const named = subjectName(appUrl, { source: subject.source, product: subject.product?.name });
|
|
47
|
+
return named !== "" ? named : (subject.appUrl ?? subject.serve?.url ?? subject.source);
|
|
51
48
|
}
|
|
52
49
|
/** A participant's status in words: a dry run's placeholder status says that nothing ran live. */
|
|
53
50
|
function participantStatus(status) {
|
|
@@ -107,7 +104,7 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
107
104
|
...runHeader(result, "scripted"),
|
|
108
105
|
`run: ${result.runId}`,
|
|
109
106
|
`actor: ${result.actor}`,
|
|
110
|
-
`subject: ${
|
|
107
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
111
108
|
...(result.scenario
|
|
112
109
|
? [
|
|
113
110
|
`scenario: ${result.scenario.id} @ ${result.scenario.sourceDigest.slice(0, 12)} (${result.scenario.source}, ${result.scenario.steps} step${result.scenario.steps === 1 ? "" : "s"})`,
|
|
@@ -133,7 +130,7 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
133
130
|
...runHeader(result, "computer-use"),
|
|
134
131
|
`run: ${result.runId}`,
|
|
135
132
|
`actor: ${result.actor}`,
|
|
136
|
-
`subject: ${
|
|
133
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
137
134
|
...(result.subject?.source === "clone"
|
|
138
135
|
? [
|
|
139
136
|
`repo: ${result.subject.repo}${result.subject.commit ? `@${result.subject.commit.slice(0, 12)}` : ""}${result.subject.envNames && result.subject.envNames.length > 0 ? ` env=[${result.subject.envNames.join(", ")}]` : ""}`,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import {} from "../../run-study.js";
|
|
4
4
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
@@ -237,7 +237,7 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
writeResult(args.command, args.io, output, (value) => formatCuaStudyHuman(value, args.config.subject));
|
|
240
|
-
args.io.setExitCode(result.ok &&
|
|
240
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
241
241
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
242
242
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
243
243
|
const followResult = output.observer?.ok
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function scriptedRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, (value) => formatScriptedStudyHuman(value, args.config.subject));
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
41
41
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
4
4
|
import { serveObserver } from "../../observer/render.js";
|
|
@@ -39,7 +39,9 @@ export function sharedWorldRouteRun(args) {
|
|
|
39
39
|
topology: "shared-world",
|
|
40
40
|
topologyMode: "concurrent",
|
|
41
41
|
roleCount: participantList(args.config)?.length ?? 0,
|
|
42
|
-
|
|
42
|
+
// Refused before it planned: the declared cap, else every participant.
|
|
43
|
+
concurrency: args.config.execution?.concurrency ??
|
|
44
|
+
Math.max(1, participantList(args.config)?.length ?? 0),
|
|
43
45
|
dryRun,
|
|
44
46
|
runId: runId ?? args.options.runId ?? "not-created",
|
|
45
47
|
roles: [],
|
|
@@ -119,7 +121,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
119
121
|
};
|
|
120
122
|
}
|
|
121
123
|
writeResult(args.command, args.io, output, formatConcurrentSharedWorldStudyHuman);
|
|
122
|
-
args.io.setExitCode(result.ok &&
|
|
124
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
123
125
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
124
126
|
if (server && output.observer?.ok) {
|
|
125
127
|
await followObserver(args.io, output.observer, server);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { studyAnalysisSucceeded } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function terminalRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, formatTerminalStudyHuman);
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && studyAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
41
41
|
await showObserver({
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../analysis/automatic-config.js";
|
|
2
1
|
import { resolve } from "node:path";
|
|
3
2
|
import { Command } from "commander";
|
|
4
3
|
import { deriveRunFacts } from "../telemetry.js";
|
|
5
4
|
import { resolveStudyManifest } from "../../study/discover.js";
|
|
5
|
+
import { formatStudyAnalysisBudget, studyAnalysisBudget, } from "../../study/automatic-analysis-plan.js";
|
|
6
6
|
import { warnAndQueue } from "../deprecations.js";
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
|
@@ -131,9 +131,9 @@ export async function runStudyCommand(args) {
|
|
|
131
131
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
132
132
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
133
133
|
}
|
|
134
|
-
const analysisBudget =
|
|
134
|
+
const analysisBudget = studyAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
135
135
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
136
|
-
args.io.writeErr(`${
|
|
136
|
+
args.io.writeErr(`${formatStudyAnalysisBudget(analysisBudget)}\n`);
|
|
137
137
|
}
|
|
138
138
|
return scorer === undefined ? {} : { scorer };
|
|
139
139
|
},
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { formatStudyAnalysisBudget } from "../../study/automatic-analysis-plan.js";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
@@ -136,7 +136,7 @@ function formatStudyCheckHuman(result) {
|
|
|
136
136
|
`targets: ${checkedTargets.length ? `${reachableTargets.length}/${checkedTargets.length} reachable` : `${result.targets.length} declared, not checked`}`,
|
|
137
137
|
...(blockedTargets.length ? [`blocked-targets: ${blockedTargets.length}`] : []),
|
|
138
138
|
`spend: ${result.spend.e2bDesktop ? "one e2b desktop, no model calls" : "none"}`,
|
|
139
|
-
...(result.analysis ? [
|
|
139
|
+
...(result.analysis ? [formatStudyAnalysisBudget(result.analysis)] : []),
|
|
140
140
|
...(result.sandbox.created
|
|
141
141
|
? [`sandbox: created=yes killed=${result.sandbox.killed === true ? "yes" : "no"}`]
|
|
142
142
|
: []),
|
package/dist/cli/doctor.js
CHANGED
|
@@ -4,7 +4,7 @@ import { existsSync } from "node:fs";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { DOCTOR_KEYS, probeKeySources, } from "../keys/key-resolution.js";
|
|
6
6
|
import { desktopPeerAdvice, humanishInstall, } from "../substrates/e2b/peer-install.js";
|
|
7
|
-
import { describeSandboxCeiling, sandboxCeiling } from "../substrates/e2b/lifetime.js";
|
|
7
|
+
import { concurrentSandboxes, describeConcurrentSandboxes, describeSandboxCeiling, sandboxCeiling, } from "../substrates/e2b/lifetime.js";
|
|
8
8
|
import { nodeSupportsTui, terminalSurfaceMessage, TUI_BUNDLE_URL } from "../tui/contract.js";
|
|
9
9
|
import { detectLocalAgents, localAgentDoctorMessage, localAgentReady, NO_LOCAL_AGENT_MESSAGE, } from "../actors/local-agent/cli.js";
|
|
10
10
|
import { leftoverClaudeTranscripts } from "../actors/local-agent/claude-participant.js";
|
|
@@ -97,6 +97,7 @@ export async function doctor(cwdInput, options = {}) {
|
|
|
97
97
|
...(await projectChecks(projectRoot)),
|
|
98
98
|
await desktopSdkCheck(setup),
|
|
99
99
|
sandboxCeilingCheck(env),
|
|
100
|
+
concurrentSandboxesCheck(env),
|
|
100
101
|
terminalSurfaceCheck(),
|
|
101
102
|
...versionCheck(env),
|
|
102
103
|
...localAgentChecks(agents),
|
|
@@ -198,6 +199,15 @@ function sandboxCeilingCheck(env) {
|
|
|
198
199
|
const ceiling = sandboxCeiling(env);
|
|
199
200
|
return { name: "e2b sandbox ceiling", ok: ceiling.ok, message: describeSandboxCeiling(ceiling) };
|
|
200
201
|
}
|
|
202
|
+
/** How many sandboxes the E2B planners run at once, and where that number came from. */
|
|
203
|
+
function concurrentSandboxesCheck(env) {
|
|
204
|
+
const limit = concurrentSandboxes(env);
|
|
205
|
+
return {
|
|
206
|
+
name: "e2b concurrent sandboxes",
|
|
207
|
+
ok: limit.ok,
|
|
208
|
+
message: describeConcurrentSandboxes(limit),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
201
211
|
/**
|
|
202
212
|
* The optional peer dep every live browser and terminal route needs. `npx -y humanish` does not
|
|
203
213
|
* pull optional peers, so without this check an adopter's first live run fails on it, safely and at
|
package/dist/cli/io.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js";
|
|
2
1
|
import { costRefusal, refusedCost } from "../analysis/admission.js";
|
|
3
2
|
import { Command, Option } from "commander";
|
|
4
3
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
5
4
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
6
5
|
import { deriveRunFacts } from "./telemetry.js";
|
|
7
6
|
import { warnAndQueue, withQueuedWarnings } from "./deprecations.js";
|
|
7
|
+
import { PARTICIPANT_LIMIT_SKIP, PARTICIPANT_LIMIT_SKIP_TEXT, studyAnalysisSucceeded, } from "../study/automatic-analysis-plan.js";
|
|
8
8
|
import { forTerminal } from "../routes/terminal/encoding.js";
|
|
9
9
|
import { cli } from "./invocation.js";
|
|
10
10
|
export const CLI_RESPONSE_SCHEMA = "humanish.cli-response.v1";
|
|
@@ -295,6 +295,7 @@ const ANALYSIS_REASON_TEXT = {
|
|
|
295
295
|
AUTOMATIC_ANALYSIS_BUSY: "skipped because another analysis is running",
|
|
296
296
|
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "refused before it started: its configuration, question, evidence or cost did not pass admission",
|
|
297
297
|
AUTOMATIC_ANALYSIS_CLEANUP_UNCONFIRMED: "not started because the run's cleanup was not confirmed",
|
|
298
|
+
[PARTICIPANT_LIMIT_SKIP]: PARTICIPANT_LIMIT_SKIP_TEXT,
|
|
298
299
|
AUTOMATIC_ANALYSIS_REUSED: "complete, reusing an earlier analysis of the same evidence",
|
|
299
300
|
AUTOMATIC_ANALYSIS_LIMITATIONS: "partial, because it covered only part of the evidence",
|
|
300
301
|
AUTOMATIC_ANALYSIS_ADMISSION_EXCEEDED: "partial, because it went over its estimated cost or output",
|
|
@@ -320,5 +321,5 @@ export function automaticAnalysisEnvelope(result) {
|
|
|
320
321
|
!("ok" in result))
|
|
321
322
|
return result;
|
|
322
323
|
const run = result;
|
|
323
|
-
return { ...run, runOk: run.ok, ok: run.ok &&
|
|
324
|
+
return { ...run, runOk: run.ok, ok: run.ok && studyAnalysisSucceeded(run) };
|
|
324
325
|
}
|
|
@@ -6,6 +6,7 @@ import { homedir, hostname } from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { isRecord } from "../run/type-guards.js";
|
|
8
8
|
import { isReceivingProviderId } from "./providers.js";
|
|
9
|
+
import { MAX_RECEIVING_INBOXES, } from "./receiving-types.js";
|
|
9
10
|
const SCHEMA = "humanish.comms-lease-journal.v1";
|
|
10
11
|
const MAX_BYTES = 1_048_576;
|
|
11
12
|
const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
|
|
@@ -117,7 +118,7 @@ function parseJournal(text) {
|
|
|
117
118
|
!safeString(value.updatedAt) ||
|
|
118
119
|
!Array.isArray(value.leases) ||
|
|
119
120
|
value.leases.length < 1 ||
|
|
120
|
-
value.leases.length >
|
|
121
|
+
value.leases.length > MAX_RECEIVING_INBOXES)
|
|
121
122
|
fail();
|
|
122
123
|
const participants = new Set();
|
|
123
124
|
const leaseIds = new Set();
|
|
@@ -470,7 +471,7 @@ export class CommsLeaseStore {
|
|
|
470
471
|
!KEY_ENV.test(options.apiKeyEnv) ||
|
|
471
472
|
!validReceivingIdentity(options.identity) ||
|
|
472
473
|
options.participants.length < 1 ||
|
|
473
|
-
options.participants.length >
|
|
474
|
+
options.participants.length > MAX_RECEIVING_INBOXES ||
|
|
474
475
|
new Set(options.participants).size !== options.participants.length ||
|
|
475
476
|
options.participants.some((id) => !LOCAL_ID.test(id)))
|
|
476
477
|
fail();
|
|
@@ -3,6 +3,8 @@ import type { ReceivingProviderId } from "./providers.js";
|
|
|
3
3
|
import type { CommsInlineImage } from "./types.js";
|
|
4
4
|
/** authenticate() rejects with this code when the credential cannot acquire fresh inboxes. */
|
|
5
5
|
export declare const RECEIVING_SCOPE_UNSUPPORTED = "comms_scope_unsupported";
|
|
6
|
+
/** The most inboxes one run leases, one per participant; the lease journal holds this many. */
|
|
7
|
+
export declare const MAX_RECEIVING_INBOXES = 64;
|
|
6
8
|
export interface ReceivingContext {
|
|
7
9
|
signal?: AbortSignal;
|
|
8
10
|
timeoutMs?: number;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
/** authenticate() rejects with this code when the credential cannot acquire fresh inboxes. */
|
|
2
2
|
export const RECEIVING_SCOPE_UNSUPPORTED = "comms_scope_unsupported";
|
|
3
|
+
/** The most inboxes one run leases, one per participant; the lease journal holds this many. */
|
|
4
|
+
export const MAX_RECEIVING_INBOXES = 64;
|
|
3
5
|
export const COMMS_RECEIVING_SCHEMA = "humanish.comms-receiving.v2";
|