humanish 0.145.0 → 0.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/actors/computer-use/loop/spend.js +5 -4
- package/dist/actors/computer-use/loop/trace.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -0
- package/dist/actors/computer-use/loop/usage.js +10 -2
- package/dist/actors/computer-use/openai-conversation.d.ts +7 -0
- package/dist/actors/computer-use/openai-conversation.js +19 -2
- package/dist/actors/computer-use/openai-provider.d.ts +2 -0
- package/dist/actors/computer-use/openai-provider.js +2 -0
- package/dist/actors/computer-use/openai-wire.d.ts +4 -0
- package/dist/actors/computer-use/openai-wire.js +9 -1
- package/dist/actors/contract.d.ts +5 -0
- package/dist/analysis/admission.d.ts +8 -3
- package/dist/analysis/admission.js +7 -3
- package/dist/analysis/automatic-config.js +2 -1
- package/dist/analysis/automatic.js +2 -2
- package/dist/analysis/evidence.js +8 -4
- package/dist/analysis/execute.d.ts +7 -10
- package/dist/analysis/execute.js +79 -49
- package/dist/analysis/image-tokens.js +9 -3
- package/dist/analysis/output-limit.d.ts +12 -0
- package/dist/analysis/output-limit.js +34 -0
- package/dist/analysis/output-retry.d.ts +20 -0
- package/dist/analysis/output-retry.js +63 -0
- package/dist/analysis/provider.d.ts +1 -1
- package/dist/analysis/provider.js +3 -3
- package/dist/analysis/responses.js +1 -1
- package/dist/analysis/service.js +2 -2
- package/dist/cli/commands/analyze.js +1 -1
- package/dist/cli/io.js +2 -0
- package/dist/cli/run-findings.js +10 -2
- package/dist/evidence/image.d.ts +2 -0
- package/dist/evidence/image.js +24 -3
- package/dist/evidence/webp-encode-worker.d.ts +1 -0
- package/dist/evidence/webp-encode-worker.js +31 -0
- package/dist/evidence/webp-transcoder.d.ts +15 -0
- package/dist/evidence/webp-transcoder.js +98 -0
- package/dist/evidence/webp.d.ts +31 -0
- package/dist/evidence/webp.js +94 -0
- package/dist/feedback/derivative-text.d.ts +7 -2
- package/dist/feedback/derivative-text.js +20 -1
- package/dist/feedback/export-bundle.js +79 -25
- package/dist/observer-app.html +1 -1
- package/dist/remote/launch.js +93 -48
- package/dist/remote/runner.d.ts +47 -14
- package/dist/remote/runner.js +199 -64
- package/dist/routes/computer-use/e2b-desktop/desktop.js +47 -7
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +4 -2
- package/dist/routes/computer-use/e2b-desktop/prepare.js +4 -2
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +14 -6
- package/dist/routes/computer-use/e2b-desktop/start.js +72 -34
- package/dist/routes/computer-use/participant-execution.d.ts +0 -7
- package/dist/routes/computer-use/participant-execution.js +6 -28
- package/dist/routes/computer-use/participant-frames.d.ts +32 -0
- package/dist/routes/computer-use/participant-frames.js +93 -0
- package/dist/routes/computer-use/participant-model.js +3 -0
- package/dist/routes/computer-use/participant-runs.js +4 -0
- package/dist/routes/computer-use/plan.js +5 -1
- package/dist/routes/computer-use/result.js +6 -0
- package/dist/routes/computer-use/types.d.ts +4 -0
- package/dist/routes/shared-world/plan.js +4 -1
- package/dist/run/pricing.d.ts +7 -0
- package/dist/run/pricing.js +58 -15
- package/dist/study/config.js +4 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -0
- package/dist/study/parse/actors.js +5 -0
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +2 -0
- package/dist/study/types.d.ts +7 -0
- package/dist/study/validation.d.ts +5 -0
- package/dist/study/validation.js +23 -0
- package/dist/substrates/e2b/cdp-probe.d.ts +16 -1
- package/dist/substrates/e2b/cdp-probe.js +67 -5
- package/dist/substrates/e2b/desktop-browser.js +4 -2
- package/dist/substrates/e2b/desktop-cdp.d.ts +9 -2
- package/dist/substrates/e2b/desktop-cdp.js +26 -4
- package/dist/verify/artifacts.js +2 -1
- package/docs/contracts/run-bundle.md +29 -11
- package/docs/contracts/schemas.md +20 -12
- package/docs/contracts/study-analysis.md +64 -11
- package/docs/product/automatic-analysis.md +9 -6
- package/package.json +6 -5
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,27 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.147.0: Screenshots take half the space (2026-10-25)
|
|
12
|
+
|
|
13
|
+
humanish 0.147.0 keeps a computer-use participant's screenshots as lossless WebP, 0.47 to 0.55 of
|
|
14
|
+
the PNG size with the same pixels, and `verify`, study analysis and export read them; earlier
|
|
15
|
+
humanish versions refuse a bundle with WebP screenshots. `actor.serviceTier: fast` runs OpenAI
|
|
16
|
+
computer-use participants in Fast mode. On the Taskly benchmark a Fast mode step took 14% less
|
|
17
|
+
time at 2.15 times the participant cost, and each request is priced at the tier that served it.
|
|
18
|
+
|
|
19
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.147.0)
|
|
20
|
+
|
|
21
|
+
## 0.146.0: Studies start and finish sooner (2026-10-23)
|
|
22
|
+
|
|
23
|
+
humanish 0.146.0 cuts the wait around a hosted study. A Chrome participant starts once its page
|
|
24
|
+
has loaded, in place of a fixed 8 s sleep, and a remote run sets up its runner and returns its
|
|
25
|
+
evidence faster; on a one-participant study, desktop to session start fell from about 29 s to 11
|
|
26
|
+
to 18 s, and a remote run's overhead from 17.8 s to about 10 s. An analysis request that runs out
|
|
27
|
+
of output is sent again once within its cost cap, and a low- or medium-effort analysis asks for an
|
|
28
|
+
output limit sized to its participants, so a runaway request stops sooner.
|
|
29
|
+
|
|
30
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.146.0)
|
|
31
|
+
|
|
11
32
|
## 0.145.0: A remote runner's keys stay in E2B's proxy (2026-10-22)
|
|
12
33
|
|
|
13
34
|
humanish 0.145.0 keeps the OpenAI, Gemini and E2B keys out of a remote E2B runner's environment.
|
|
@@ -58,11 +58,12 @@ export function bookLostRequest(session, turnNumber, lost) {
|
|
|
58
58
|
}
|
|
59
59
|
const input = session.usage.lastReportedInput();
|
|
60
60
|
session.usage.reserve(input, maxOutputTokens);
|
|
61
|
+
const serviceTier = session.provider.modelSettings?.serviceTier;
|
|
62
|
+
const booked = { input, output: maxOutputTokens, cachedInput: 0, cacheWriteInput: input };
|
|
61
63
|
const usd = session.settings.estimateTurnCostUsd?.({
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
cacheWriteInput: input,
|
|
64
|
+
...booked,
|
|
65
|
+
// A request in Fast mode is booked at the fast rate, which a per-request record selects.
|
|
66
|
+
...(serviceTier === undefined ? {} : { turns: [{ ...booked, serviceTier }] }),
|
|
66
67
|
});
|
|
67
68
|
const charge = `${input} input tokens at the highest input rate and ${maxOutputTokens} output tokens${typeof usd === "number" ? ` ($${usd})` : ""}`;
|
|
68
69
|
const over = spendStop(session);
|
|
@@ -127,6 +127,9 @@ export function loopResult(session, stop, debrief) {
|
|
|
127
127
|
...(provider.modelSettings.maxOutputTokens === undefined
|
|
128
128
|
? {}
|
|
129
129
|
: { maxOutputTokens: provider.modelSettings.maxOutputTokens }),
|
|
130
|
+
...(provider.modelSettings.serviceTier === undefined
|
|
131
|
+
? {}
|
|
132
|
+
: { serviceTier: provider.modelSettings.serviceTier }),
|
|
130
133
|
},
|
|
131
134
|
}),
|
|
132
135
|
waitSettings: {
|
|
@@ -191,12 +191,15 @@ export interface CuaTurn {
|
|
|
191
191
|
output?: number;
|
|
192
192
|
cachedInput?: number;
|
|
193
193
|
cacheWriteInput?: number;
|
|
194
|
+
/** The processing tier that served this request, when the session asked for one. */
|
|
195
|
+
serviceTier?: string;
|
|
194
196
|
/** Per model inference inside this provider interaction. */
|
|
195
197
|
turns?: Array<{
|
|
196
198
|
input?: number;
|
|
197
199
|
output?: number;
|
|
198
200
|
cachedInput?: number;
|
|
199
201
|
cacheWriteInput?: number;
|
|
202
|
+
serviceTier?: string;
|
|
200
203
|
}>;
|
|
201
204
|
};
|
|
202
205
|
/** True when the model reported a natural endpoint (no further action). */
|
|
@@ -234,6 +237,8 @@ export interface CuaProvider {
|
|
|
234
237
|
readonly modelSettings?: {
|
|
235
238
|
readonly reasoningEffort: ReasoningEffort;
|
|
236
239
|
readonly maxOutputTokens?: number;
|
|
240
|
+
/** The processing tier every request asks for; absent asks for none. */
|
|
241
|
+
readonly serviceTier?: "fast";
|
|
237
242
|
};
|
|
238
243
|
readonly capabilities: ActorCapabilities;
|
|
239
244
|
/**
|
|
@@ -14,12 +14,19 @@ function normalizedUsageTurns(usage) {
|
|
|
14
14
|
return undefined;
|
|
15
15
|
if ((raw.cachedInput ?? 0) + (raw.cacheWriteInput ?? 0) > (raw.input ?? 0))
|
|
16
16
|
return undefined;
|
|
17
|
-
turns.push(
|
|
17
|
+
turns.push({
|
|
18
|
+
...Object.fromEntries(fields.flatMap((field) => (raw[field] === undefined ? [] : [[field, raw[field]]]))),
|
|
19
|
+
...servedTier(raw.serviceTier),
|
|
20
|
+
});
|
|
18
21
|
}
|
|
19
22
|
return fields.every((field) => turns.reduce((sum, turn) => sum + (turn[field] ?? 0), 0) === (usage[field] ?? 0))
|
|
20
23
|
? turns
|
|
21
24
|
: undefined;
|
|
22
25
|
}
|
|
26
|
+
/** The served-tier field of a usage record, kept only when the provider named one. */
|
|
27
|
+
function servedTier(tier) {
|
|
28
|
+
return typeof tier === "string" && tier !== "" ? { serviceTier: tier } : {};
|
|
29
|
+
}
|
|
23
30
|
export function isCompleteTurnUsage(usage) {
|
|
24
31
|
return (usage !== undefined &&
|
|
25
32
|
validTokenCount(usage.input) &&
|
|
@@ -130,7 +137,7 @@ export class UsageLedger {
|
|
|
130
137
|
this.cacheWriteInput += usage.cacheWriteInput ?? 0;
|
|
131
138
|
this.output += usage.output ?? 0;
|
|
132
139
|
if (raw.turns === undefined)
|
|
133
|
-
this.turns.push(usage);
|
|
140
|
+
this.turns.push({ ...usage, ...servedTier(raw.serviceTier) });
|
|
134
141
|
else if (turns !== undefined)
|
|
135
142
|
this.turns.push(...turns);
|
|
136
143
|
}
|
|
@@ -218,6 +225,7 @@ export class UsageLedger {
|
|
|
218
225
|
output,
|
|
219
226
|
cachedInput: 0,
|
|
220
227
|
cacheWriteInput: input,
|
|
228
|
+
...servedTier(this.provider.modelSettings?.serviceTier),
|
|
221
229
|
}));
|
|
222
230
|
const sum = (field) => booked.reduce((total, turn) => total + turn[field], 0);
|
|
223
231
|
const input = (settled.input ?? 0) + sum("input");
|
|
@@ -8,6 +8,8 @@ export interface OpenAiConversationSettings {
|
|
|
8
8
|
readonly reasoningEffort: ReasoningEffort;
|
|
9
9
|
readonly maxOutputTokens?: number;
|
|
10
10
|
readonly safetyIdentifier?: string;
|
|
11
|
+
/** OpenAI Fast mode for every request of the session. */
|
|
12
|
+
readonly serviceTier?: "fast";
|
|
11
13
|
/** The summaries to ask for until the account or model rejects them; undefined asks for none. */
|
|
12
14
|
readonly reasoningSummary: OpenAiReasoningSummary | undefined;
|
|
13
15
|
/** Carry the conversation on the client from the first request (explicit_context). */
|
|
@@ -98,6 +100,11 @@ export declare class OpenAiConversation {
|
|
|
98
100
|
*/
|
|
99
101
|
private waitOutcome;
|
|
100
102
|
private context;
|
|
103
|
+
/**
|
|
104
|
+
* A reply, parsed. When the session asked for a tier, the reply's usage records the tier that
|
|
105
|
+
* served it, which is what the request bills at: Fast mode can fall back to `default`.
|
|
106
|
+
*/
|
|
107
|
+
private parse;
|
|
101
108
|
/**
|
|
102
109
|
* A participant request: the opening request, or a continuation carrying `sent`. A threaded
|
|
103
110
|
* continuation names the previous response; an explicit-context one carries the conversation
|
|
@@ -75,7 +75,7 @@ export class OpenAiConversation {
|
|
|
75
75
|
if (closing === "impressions")
|
|
76
76
|
this.evidence.requested(this.mode, this.carried, sent, closing);
|
|
77
77
|
const completed = asRecord(reply).status === "completed";
|
|
78
|
-
const parsed =
|
|
78
|
+
const parsed = this.parse(reply);
|
|
79
79
|
const account = completed ? closingAccount(parsed.turn.message, closing, strict) : undefined;
|
|
80
80
|
// A refusal would most likely stand under the strict schema too, so it is not asked again.
|
|
81
81
|
if (completed && account === undefined && !strict && !refused(reply)) {
|
|
@@ -164,6 +164,23 @@ export class OpenAiConversation {
|
|
|
164
164
|
...(settings.safetyIdentifier === undefined
|
|
165
165
|
? {}
|
|
166
166
|
: { safetyIdentifier: settings.safetyIdentifier }),
|
|
167
|
+
...(settings.serviceTier === undefined ? {} : { serviceTier: settings.serviceTier }),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* A reply, parsed. When the session asked for a tier, the reply's usage records the tier that
|
|
172
|
+
* served it, which is what the request bills at: Fast mode can fall back to `default`.
|
|
173
|
+
*/
|
|
174
|
+
parse(reply) {
|
|
175
|
+
const parsed = parseOpenAiResponse(reply);
|
|
176
|
+
const { usage } = parsed.turn;
|
|
177
|
+
if (this.settings.serviceTier === undefined || usage === undefined)
|
|
178
|
+
return parsed;
|
|
179
|
+
if (parsed.serviceTier === undefined)
|
|
180
|
+
return parsed;
|
|
181
|
+
return {
|
|
182
|
+
...parsed,
|
|
183
|
+
turn: { ...parsed.turn, usage: { ...usage, serviceTier: parsed.serviceTier } },
|
|
167
184
|
};
|
|
168
185
|
}
|
|
169
186
|
/**
|
|
@@ -204,7 +221,7 @@ export class OpenAiConversation {
|
|
|
204
221
|
// Recorded before the reply is accepted: accepting can cut the carried history, and the record
|
|
205
222
|
// counts what this request carried.
|
|
206
223
|
this.evidence.requested(this.mode, this.carried, sent);
|
|
207
|
-
const parsed =
|
|
224
|
+
const parsed = this.parse(reply);
|
|
208
225
|
if (parsed.turn.interruption === "output_limit")
|
|
209
226
|
return parsed.turn;
|
|
210
227
|
this.moveOn(parsed, sent);
|
|
@@ -68,6 +68,8 @@ export interface OpenAiResponsesProviderOptions {
|
|
|
68
68
|
reasoningEffort?: ReasoningEffort;
|
|
69
69
|
/** Optional positive integer output limit per response, including reasoning. Not a spend cap. */
|
|
70
70
|
maxOutputTokens?: number;
|
|
71
|
+
/** OpenAI Fast mode on every request; absent sends no `service_tier`. */
|
|
72
|
+
serviceTier?: "fast";
|
|
71
73
|
/**
|
|
72
74
|
* Reasoning-summary capture. Defaults to "auto" (the provider picks the best
|
|
73
75
|
* summarizer the model supports); "off" never asks. If the account/model rejects the
|
|
@@ -418,6 +418,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
418
418
|
...(options.safetyIdentifier === undefined
|
|
419
419
|
? {}
|
|
420
420
|
: { safetyIdentifier: options.safetyIdentifier }),
|
|
421
|
+
...(options.serviceTier === undefined ? {} : { serviceTier: options.serviceTier }),
|
|
421
422
|
reasoningSummary: options.reasoningSummary === "off" ? undefined : (options.reasoningSummary ?? "auto"),
|
|
422
423
|
zeroDataRetention: options.zeroDataRetention === true,
|
|
423
424
|
now: options.now ?? Date.now,
|
|
@@ -471,6 +472,7 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
471
472
|
modelSettings: {
|
|
472
473
|
reasoningEffort,
|
|
473
474
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
475
|
+
...(options.serviceTier === undefined ? {} : { serviceTier: options.serviceTier }),
|
|
474
476
|
},
|
|
475
477
|
capabilities: OPENAI_RESPONSES_CU_CAPABILITIES,
|
|
476
478
|
// This is a vision provider: nextTurn sends the screenshot as the computer_call_output, so
|
|
@@ -15,6 +15,8 @@ export interface ParsedOpenAiResponse {
|
|
|
15
15
|
/** The function calls, in reply order, each owed a function_call_output. */
|
|
16
16
|
functionCalls: FunctionCall[];
|
|
17
17
|
outputItems: unknown[];
|
|
18
|
+
/** The processing tier the reply says served it (`service_tier`), when it says one. */
|
|
19
|
+
serviceTier?: string;
|
|
18
20
|
}
|
|
19
21
|
/** A function call a reply made: its id, and its name, which is WAIT_TOOL_NAME unless it erred. */
|
|
20
22
|
export interface FunctionCall {
|
|
@@ -52,6 +54,8 @@ export interface OpenAiCuContext {
|
|
|
52
54
|
/** When set, request provider-sanctioned reasoning summaries. Absent = do not ask. */
|
|
53
55
|
reasoningSummary?: OpenAiReasoningSummary;
|
|
54
56
|
safetyIdentifier?: string;
|
|
57
|
+
/** OpenAI Fast mode (`service_tier: "fast"`). Absent sends no tier: the project's default. */
|
|
58
|
+
serviceTier?: "fast";
|
|
55
59
|
}
|
|
56
60
|
/**
|
|
57
61
|
* Build the first-turn request body: the instructions and the first screen, so the first decision
|
|
@@ -271,7 +271,14 @@ export function parseOpenAiResponse(raw) {
|
|
|
271
271
|
...(message.length > 0 ? { message } : {}),
|
|
272
272
|
...(usage === undefined ? {} : { usage }),
|
|
273
273
|
};
|
|
274
|
-
|
|
274
|
+
const serviceTier = optionalString(root.service_tier);
|
|
275
|
+
return {
|
|
276
|
+
turn,
|
|
277
|
+
callIds,
|
|
278
|
+
functionCalls,
|
|
279
|
+
outputItems: output,
|
|
280
|
+
...(serviceTier === undefined ? {} : { serviceTier }),
|
|
281
|
+
};
|
|
275
282
|
}
|
|
276
283
|
/**
|
|
277
284
|
* A function the participant names its wait length with; the computer tool's `wait` names none.
|
|
@@ -324,6 +331,7 @@ function sharedRequestFields(ctx) {
|
|
|
324
331
|
...(ctx.reasoningSummary === undefined ? {} : { summary: ctx.reasoningSummary }),
|
|
325
332
|
},
|
|
326
333
|
...(ctx.safetyIdentifier === undefined ? {} : { safety_identifier: ctx.safetyIdentifier }),
|
|
334
|
+
...(ctx.serviceTier === undefined ? {} : { service_tier: ctx.serviceTier }),
|
|
327
335
|
...(ctx.stateless === true ? { store: false, include: ["reasoning.encrypted_content"] } : {}),
|
|
328
336
|
};
|
|
329
337
|
}
|
|
@@ -187,6 +187,10 @@ export interface ActorTokenUsage {
|
|
|
187
187
|
cachedInput?: number;
|
|
188
188
|
cacheWriteInput?: number;
|
|
189
189
|
output?: number;
|
|
190
|
+
/** The processing tier the provider reported serving this request at (OpenAI `service_tier`:
|
|
191
|
+
* `default`, `priority`, `fast`). Recorded when the participant asked for a tier; Fast mode
|
|
192
|
+
* can fall back to `default`, which bills the standard rate. */
|
|
193
|
+
serviceTier?: string;
|
|
190
194
|
}>;
|
|
191
195
|
total?: number;
|
|
192
196
|
costUsd?: number;
|
|
@@ -364,6 +368,7 @@ export interface ActorTrace {
|
|
|
364
368
|
modelSettings?: {
|
|
365
369
|
reasoningEffort: string;
|
|
366
370
|
maxOutputTokens?: number;
|
|
371
|
+
serviceTier?: string;
|
|
367
372
|
};
|
|
368
373
|
/**
|
|
369
374
|
* Additive + optional: the wait lengths the computer-use loop applied, defaults included. Absent
|
|
@@ -30,14 +30,19 @@ export interface AnalysisRequests {
|
|
|
30
30
|
/** The analyst's EXPECTED_OUTPUT_SCALE; 1 when omitted. */
|
|
31
31
|
outputScale?: number;
|
|
32
32
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
expectedOutputTokens: number;
|
|
33
|
+
/** What one request is expected to cost, its worst case, and the figure admission compares. */
|
|
34
|
+
export interface RequestCost {
|
|
36
35
|
expectedCostUsd: number;
|
|
37
36
|
worstCaseCostUsd: number;
|
|
38
37
|
/** The expected cost with the margin, never above the worst case: what admission compares. */
|
|
39
38
|
admittedCostUsd: number;
|
|
40
39
|
}
|
|
40
|
+
export interface AnalysisCostEstimate extends RequestCost {
|
|
41
|
+
inputTokens: number;
|
|
42
|
+
expectedOutputTokens: number;
|
|
43
|
+
/** Each request's own costs: the cohorts' in order, then the merge request's. */
|
|
44
|
+
requests: RequestCost[];
|
|
45
|
+
}
|
|
41
46
|
/**
|
|
42
47
|
* Every request of one analysis, summed. A merge request reads each cohort's report, priced at
|
|
43
48
|
* that request's whole output allowance since the report and its reasoning share it, and is
|
|
@@ -85,11 +85,15 @@ export function estimateAnalysisCost(rate, requests) {
|
|
|
85
85
|
return {
|
|
86
86
|
inputTokens: sum("inputTokens"),
|
|
87
87
|
expectedOutputTokens: sum("expectedOutputTokens"),
|
|
88
|
-
expectedCostUsd,
|
|
89
|
-
worstCaseCostUsd,
|
|
90
|
-
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
88
|
+
...admitted(expectedCostUsd, worstCaseCostUsd),
|
|
89
|
+
requests: estimates.map((estimate) => admitted(estimate.expectedCostUsd, estimate.worstCaseCostUsd)),
|
|
91
90
|
};
|
|
92
91
|
}
|
|
92
|
+
const admitted = (expectedCostUsd, worstCaseCostUsd) => ({
|
|
93
|
+
expectedCostUsd,
|
|
94
|
+
worstCaseCostUsd,
|
|
95
|
+
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
96
|
+
});
|
|
93
97
|
/**
|
|
94
98
|
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
95
99
|
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import {
|
|
2
|
+
import { analysisCostRange } from "./execute.js";
|
|
3
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS } from "./output-limit.js";
|
|
3
4
|
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
4
5
|
import { isReasoningEffort } from "../actors/reasoning-effort.js";
|
|
5
6
|
import { admissionRule } from "./admission.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { resolveRunPath } from "../run/locate.js";
|
|
3
3
|
import { refreshObserver, analyzeRun, readCompletedAnalysisSource, resolveAnalysisRun, } from "./service.js";
|
|
4
|
-
import {
|
|
4
|
+
import { defaultAnalysisOutput, ANALYSIS_PROMPT_VERSION } from "./execute.js";
|
|
5
5
|
import { captureEvidence } from "./evidence.js";
|
|
6
6
|
import { analysisApiKey } from "../keys/key-resolution.js";
|
|
7
7
|
import { loadRunTraces } from "../run/stream-trace.js";
|
|
@@ -103,7 +103,7 @@ async function readAutomaticSource(cwd, prepared, configInput, deps, hasKey) {
|
|
|
103
103
|
deps.preferLargerOutput &&
|
|
104
104
|
(!deps.defaultRequest || participantEvidence) &&
|
|
105
105
|
hasKey) {
|
|
106
|
-
config =
|
|
106
|
+
config = defaultAnalysisOutput(await captureEvidence(prepared, bytes), config);
|
|
107
107
|
}
|
|
108
108
|
return { participantEvidence, config };
|
|
109
109
|
}
|
|
@@ -348,13 +348,13 @@ function packEvidence(selected, packings, textByEntry, evidence, images) {
|
|
|
348
348
|
eventId: source.eventId,
|
|
349
349
|
path: source.capturePath,
|
|
350
350
|
sha256: sha256(bytes),
|
|
351
|
-
mimeType:
|
|
351
|
+
mimeType: captureMimeType(source.capturePath),
|
|
352
352
|
}
|
|
353
353
|
: null;
|
|
354
|
-
if (
|
|
354
|
+
if (capture)
|
|
355
355
|
images.push({
|
|
356
356
|
evidenceId: id,
|
|
357
|
-
dataUrl: `data
|
|
357
|
+
dataUrl: `data:${capture.mimeType};base64,${bytes.toString("base64")}`,
|
|
358
358
|
});
|
|
359
359
|
evidence.push({
|
|
360
360
|
id,
|
|
@@ -448,7 +448,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
448
448
|
if (source.capturePath === null ||
|
|
449
449
|
entry.capture.path !== source.capturePath ||
|
|
450
450
|
entry.capture.eventId !== source.eventId ||
|
|
451
|
-
entry.capture.mimeType !==
|
|
451
|
+
entry.capture.mimeType !== captureMimeType(source.capturePath))
|
|
452
452
|
throw new Error("ANALYSIS_CAPTURE_REFERENCE_INVALID");
|
|
453
453
|
let hash = checkedCaptures.get(source.capturePath);
|
|
454
454
|
if (hash === undefined) {
|
|
@@ -474,3 +474,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
474
474
|
if (!current?.equals(bundleBytes))
|
|
475
475
|
throw new Error("ANALYSIS_SOURCE_CHANGED");
|
|
476
476
|
}
|
|
477
|
+
/** A capture's image type, from the extension screenshotEvidenceError checked it against. */
|
|
478
|
+
function captureMimeType(capturePath) {
|
|
479
|
+
return capturePath.toLowerCase().endsWith(".webp") ? "image/webp" : "image/png";
|
|
480
|
+
}
|
|
@@ -2,10 +2,6 @@ import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "
|
|
|
2
2
|
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
4
|
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
|
-
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
6
|
-
* returns at most 32,768 output tokens from one generateContent request. */
|
|
7
|
-
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
|
8
|
-
export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
|
|
9
5
|
export interface AnalysisAdmission {
|
|
10
6
|
allowed: boolean;
|
|
11
7
|
error: string | null;
|
|
@@ -65,14 +61,15 @@ export type AnalysisCostRange = {
|
|
|
65
61
|
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
66
62
|
* passes the cap. There is one packet per cohort, each a share of its own limits, and the merge
|
|
67
63
|
* request when there is more than one cohort. With `preferLargerOutput` each request gets the
|
|
68
|
-
* output allowance dispatch would give it (
|
|
69
|
-
*
|
|
64
|
+
* output allowance dispatch would give it (defaultAnalysisOutput). Undefined for Codex and for a
|
|
65
|
+
* model without rates.
|
|
70
66
|
*/
|
|
71
67
|
export declare function analysisCostRange(config: AnalysisConfig, participants: number, preferLargerOutput?: boolean): AnalysisCostRange | undefined;
|
|
72
|
-
/** Only for an omitted output limit
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
|
|
68
|
+
/** Only for an omitted output limit, which configuration fills with the 16,384-token default.
|
|
69
|
+
* Gives the analysis the allowance sized to its participants, effort and analyst, and keeps the
|
|
70
|
+
* default when a larger sized allowance would refuse a study its declared budget admits. This is
|
|
71
|
+
* one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
72
|
+
export declare function defaultAnalysisOutput(input: AnalysisInput, config: AnalysisConfig): AnalysisConfig;
|
|
76
73
|
export type AnalysisDispatchContext = Pick<AnalysisArtifact, "id" | "runId" | "sourceRunSha256" | "inputDigest" | "configDigest" | "promptVersion">;
|
|
77
74
|
interface RunAnalysisOptions {
|
|
78
75
|
apiKey?: string;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { validCodexAnalysisConfig } from "./codex-config.js";
|
|
2
2
|
import { INPUT_IMAGE_DATA_URL, highDetailImageTokens, imageTokenCeiling } from "./image-tokens.js";
|
|
3
|
-
import { EXPECTED_OUTPUT_SCALE, estimateAnalysisCost } from "./admission.js";
|
|
3
|
+
import { EXPECTED_OUTPUT_SCALE, estimateAnalysisCost, } from "./admission.js";
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
5
|
import { modelRate } from "../run/pricing.js";
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
@@ -13,11 +13,9 @@ import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analys
|
|
|
13
13
|
import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema, } from "./cohorts.js";
|
|
14
14
|
import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
|
|
15
15
|
import { mapWithConcurrency } from "../run/concurrency.js";
|
|
16
|
+
import { outputRetries } from "./output-retry.js";
|
|
17
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, defaultOutputTokens } from "./output-limit.js";
|
|
16
18
|
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
17
|
-
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
18
|
-
* returns at most 32,768 output tokens from one generateContent request. */
|
|
19
|
-
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
20
|
-
export const MAX_ANALYSIS_OUTPUT_TOKENS = 32_768;
|
|
21
19
|
const MAX_EVIDENCE_BYTES = 1024 * 1024;
|
|
22
20
|
/**
|
|
23
21
|
* Cohort requests in flight at once. OpenAI's lowest paid tier allows gpt-6-astra 1,000,000 tokens
|
|
@@ -214,21 +212,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
214
212
|
!Number.isFinite(rate.outputUsdPerToken) ||
|
|
215
213
|
rate.outputUsdPerToken < 0)
|
|
216
214
|
return denied("analysis_rate_unknown");
|
|
217
|
-
const
|
|
218
|
-
const cost = estimateAnalysisCost(rate, {
|
|
219
|
-
cohorts: cohortInputs(input).map((cohort) => {
|
|
220
|
-
const request = cohortRequest(cohort, config);
|
|
221
|
-
return {
|
|
222
|
-
textBytes: requestBytes(request),
|
|
223
|
-
imageTokens: request.images.reduce((sum, image) => sum + highDetailImageTokens(config.model, image.dataUrl), 0),
|
|
224
|
-
participants: cohort.participants.length,
|
|
225
|
-
outputAllowance: maxOutputTokens,
|
|
226
|
-
};
|
|
227
|
-
}),
|
|
228
|
-
// Without reports: estimateAnalysisCost prices them at the cohorts' output allowances.
|
|
229
|
-
mergeTextBytes: requestBytes(mergeRequest(input, config, [])),
|
|
230
|
-
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
231
|
-
});
|
|
215
|
+
const cost = analysisCost(input, config, rate);
|
|
232
216
|
if (!Number.isFinite(cost.worstCaseCostUsd))
|
|
233
217
|
return denied("analysis_rate_unknown");
|
|
234
218
|
const allowed = cost.admittedCostUsd <= config.maxCostUsd;
|
|
@@ -245,6 +229,24 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
245
229
|
requests,
|
|
246
230
|
};
|
|
247
231
|
}
|
|
232
|
+
/** Every request of an API analysis priced at `rate`: each cohort's, then the merge request's. */
|
|
233
|
+
function analysisCost(input, config, rate) {
|
|
234
|
+
const { maxOutputTokens } = config;
|
|
235
|
+
return estimateAnalysisCost(rate, {
|
|
236
|
+
cohorts: cohortInputs(input).map((cohort) => {
|
|
237
|
+
const request = cohortRequest(cohort, config);
|
|
238
|
+
return {
|
|
239
|
+
textBytes: requestBytes(request),
|
|
240
|
+
imageTokens: request.images.reduce((sum, image) => sum + highDetailImageTokens(config.model, image.dataUrl), 0),
|
|
241
|
+
participants: cohort.participants.length,
|
|
242
|
+
outputAllowance: maxOutputTokens,
|
|
243
|
+
};
|
|
244
|
+
}),
|
|
245
|
+
// Without reports: estimateAnalysisCost prices them at the cohorts' output allowances.
|
|
246
|
+
mergeTextBytes: requestBytes(mergeRequest(input, config, [])),
|
|
247
|
+
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
248
|
+
});
|
|
249
|
+
}
|
|
248
250
|
/** Structure each evidence entry adds to the packet: ids, kind, timestamps and flags. The largest
|
|
249
251
|
* retained packet averaged 200 bytes. A merge packet's participant entry is smaller. */
|
|
250
252
|
const ENTRY_STRUCTURE_BYTES = 256;
|
|
@@ -276,8 +278,8 @@ const RANGE_BISECTIONS = 40;
|
|
|
276
278
|
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
277
279
|
* passes the cap. There is one packet per cohort, each a share of its own limits, and the merge
|
|
278
280
|
* request when there is more than one cohort. With `preferLargerOutput` each request gets the
|
|
279
|
-
* output allowance dispatch would give it (
|
|
280
|
-
*
|
|
281
|
+
* output allowance dispatch would give it (defaultAnalysisOutput). Undefined for Codex and for a
|
|
282
|
+
* model without rates.
|
|
281
283
|
*/
|
|
282
284
|
export function analysisCostRange(config, participants, preferLargerOutput = false) {
|
|
283
285
|
if (config.provider === "codex")
|
|
@@ -324,13 +326,16 @@ export function analysisCostRange(config, participants, preferLargerOutput = fal
|
|
|
324
326
|
}
|
|
325
327
|
return admitted;
|
|
326
328
|
};
|
|
327
|
-
// Dispatch tries the
|
|
328
|
-
// least the same share, and the packets past the
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
329
|
+
// Dispatch tries the sized allowance first. When it is the larger one, the smaller allowance
|
|
330
|
+
// costs no more, so it admits at least the same share, and the packets past the sized one's
|
|
331
|
+
// share get the smaller one. A sized allowance below the configured one is never refused where
|
|
332
|
+
// the configured one is admitted.
|
|
333
|
+
const sized = preferLargerOutput ? defaultOutput(config, covered) : undefined;
|
|
334
|
+
const allowances = sized === undefined
|
|
335
|
+
? [config.maxOutputTokens]
|
|
336
|
+
: sized.maxOutputTokens < config.maxOutputTokens
|
|
337
|
+
? [sized.maxOutputTokens]
|
|
338
|
+
: [sized.maxOutputTokens, config.maxOutputTokens];
|
|
334
339
|
const admitted = allowances.filter((allowance) => admits(0, allowance));
|
|
335
340
|
if (admitted.length === 0)
|
|
336
341
|
return { refusedFromUsd: request(0, config.maxOutputTokens).expectedCostUsd };
|
|
@@ -341,20 +346,23 @@ export function analysisCostRange(config, participants, preferLargerOutput = fal
|
|
|
341
346
|
},
|
|
342
347
|
};
|
|
343
348
|
}
|
|
344
|
-
/** The
|
|
345
|
-
|
|
346
|
-
|
|
349
|
+
/** The output allowance sized for an omitted output limit (output-limit.ts), or undefined when the
|
|
350
|
+
* limit was declared. */
|
|
351
|
+
function defaultOutput(config, participants) {
|
|
352
|
+
const tokens = defaultOutputTokens(config, participants);
|
|
353
|
+
if (config.provider === "codex" ||
|
|
354
|
+
tokens === undefined ||
|
|
355
|
+
config.maxOutputTokens !== DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS)
|
|
347
356
|
return undefined;
|
|
348
|
-
return { ...config, maxOutputTokens:
|
|
357
|
+
return { ...config, maxOutputTokens: tokens };
|
|
349
358
|
}
|
|
350
|
-
/** Only for an omitted output limit
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
: config;
|
|
359
|
+
/** Only for an omitted output limit, which configuration fills with the 16,384-token default.
|
|
360
|
+
* Gives the analysis the allowance sized to its participants, effort and analyst, and keeps the
|
|
361
|
+
* default when a larger sized allowance would refuse a study its declared budget admits. This is
|
|
362
|
+
* one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
363
|
+
export function defaultAnalysisOutput(input, config) {
|
|
364
|
+
const sized = defaultOutput(config, input.coverage.includedStreamIds.length);
|
|
365
|
+
return sized !== undefined && estimateAnalysisAdmission(input, sized).allowed ? sized : config;
|
|
358
366
|
}
|
|
359
367
|
/** Check the caller's id, input metadata and admission; throw the stable code a direct caller receives. */
|
|
360
368
|
function admitDirectAnalysis(input, config, analysisId) {
|
|
@@ -423,14 +431,15 @@ async function analysisProvider(config, options) {
|
|
|
423
431
|
* The attempt's requests: one per cohort, `COHORT_CONCURRENCY` at a time (one at a time for Codex),
|
|
424
432
|
* then the merge request for more than one cohort. After one request fails no further request
|
|
425
433
|
* starts, and the first failure in cohort order is the attempt's, with its cohort. `notSent` means
|
|
426
|
-
* cancellation stopped a request from starting.
|
|
434
|
+
* cancellation stopped a request from starting. `send` gets each request's index: its cohort's,
|
|
435
|
+
* and the cohort count for the merge request.
|
|
427
436
|
*/
|
|
428
437
|
async function requestReport(cohorts, input, config, send, signal) {
|
|
429
438
|
let stopped = false;
|
|
430
|
-
const outcomes = await mapWithConcurrency([...cohorts], config.provider === "codex" ? 1 : COHORT_CONCURRENCY, async (cohort) => {
|
|
439
|
+
const outcomes = await mapWithConcurrency([...cohorts], config.provider === "codex" ? 1 : COHORT_CONCURRENCY, async (cohort, index) => {
|
|
431
440
|
if (stopped || signal?.aborted)
|
|
432
441
|
return undefined;
|
|
433
|
-
const outcome = await send(cohortRequest(cohort, config), cohort);
|
|
442
|
+
const outcome = await send(cohortRequest(cohort, config), cohort, index);
|
|
434
443
|
if (!outcome.ok)
|
|
435
444
|
stopped = true;
|
|
436
445
|
return outcome;
|
|
@@ -446,7 +455,16 @@ async function requestReport(cohorts, input, config, send, signal) {
|
|
|
446
455
|
if (signal?.aborted)
|
|
447
456
|
return { ok: false, notSent: true };
|
|
448
457
|
// The merge request sends the cohort reports, no evidence entries or captures.
|
|
449
|
-
return send(mergeRequest(input, config, reports), { ...input, evidence: [], images: [] });
|
|
458
|
+
return send(mergeRequest(input, config, reports), { ...input, evidence: [], images: [] }, cohorts.length);
|
|
459
|
+
}
|
|
460
|
+
/** Request `index` of an attempt at the start of a sentence, numbered as requestReport numbers it. */
|
|
461
|
+
function requestName(cohorts, index) {
|
|
462
|
+
if (cohorts.length === 1)
|
|
463
|
+
return "The analysis request";
|
|
464
|
+
if (index === cohorts.length)
|
|
465
|
+
return `The request that merges the ${cohorts.length} cohort reports`;
|
|
466
|
+
const total = cohorts.reduce((sum, cohort) => sum + cohort.participants.length, 0);
|
|
467
|
+
return `The request for one cohort (${cohorts[index].participants.length} of the ${total} participants)`;
|
|
450
468
|
}
|
|
451
469
|
/**
|
|
452
470
|
* Why a run analysed in cohorts has no report, for the attempt's warnings. Null for one request,
|
|
@@ -520,7 +538,10 @@ export async function runAnalysis(input, config, options) {
|
|
|
520
538
|
const provider = await analysisProvider(config, options);
|
|
521
539
|
const cohorts = cohortInputs(input);
|
|
522
540
|
const responses = [];
|
|
523
|
-
const
|
|
541
|
+
const retries = config.provider === "codex"
|
|
542
|
+
? undefined
|
|
543
|
+
: outputRetries(config, analysisCost(input, config, modelRate(config.model)).requests, (index) => requestName(cohorts, index), options.warnings);
|
|
544
|
+
const dispatch = async (request, part) => {
|
|
524
545
|
progress("requesting", part);
|
|
525
546
|
const response = await provider({
|
|
526
547
|
model: config.model,
|
|
@@ -538,6 +559,14 @@ export async function runAnalysis(input, config, options) {
|
|
|
538
559
|
for (const warning of [...codexWarnings(config, response), httpStatusWarning(config, response)])
|
|
539
560
|
if (warning !== undefined && !options.warnings?.includes(warning))
|
|
540
561
|
options.warnings?.push(warning);
|
|
562
|
+
return response;
|
|
563
|
+
};
|
|
564
|
+
const send = async (request, part, index) => {
|
|
565
|
+
let response = await dispatch(request, part);
|
|
566
|
+
// A request that ran out of output is sent once more while the cap admits it.
|
|
567
|
+
while (!options.signal?.aborted && retries?.retry(index, response, artifact.usage))
|
|
568
|
+
response = await dispatch(request, part);
|
|
569
|
+
retries?.settle(index);
|
|
541
570
|
if (response.status !== "completed")
|
|
542
571
|
return {
|
|
543
572
|
ok: false,
|
|
@@ -569,11 +598,12 @@ export async function runAnalysis(input, config, options) {
|
|
|
569
598
|
artifact.result = outcome.result;
|
|
570
599
|
artifact.status = input.coverage.complete ? "complete" : "partial";
|
|
571
600
|
artifact.error = null;
|
|
572
|
-
// A bill above the expected cost is a normal outcome. Only one above the worst case
|
|
573
|
-
// shows the estimate was wrong.
|
|
601
|
+
// A bill above the expected cost is a normal outcome. Only one above the worst case, with the
|
|
602
|
+
// worst case of each request sent again, or above the cap shows the estimate was wrong.
|
|
603
|
+
const worstCaseUsd = (admission.worstCaseCostUsd ?? config.maxCostUsd ?? 0) + (retries?.worstCaseUsd() ?? 0);
|
|
574
604
|
if (config.provider !== "codex" &&
|
|
575
605
|
(responses.some((response) => (response.usage?.output ?? 0) > config.maxOutputTokens) ||
|
|
576
|
-
(artifact.usage.estimatedCostUsd ?? 0) >
|
|
606
|
+
(artifact.usage.estimatedCostUsd ?? 0) > worstCaseUsd ||
|
|
577
607
|
(artifact.usage.estimatedCostUsd ?? 0) > config.maxCostUsd)) {
|
|
578
608
|
artifact.status = "partial";
|
|
579
609
|
artifact.error = "analysis_admission_estimate_exceeded";
|