humanish 0.145.0 → 0.146.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/analysis/admission.d.ts +8 -3
- package/dist/analysis/admission.js +7 -3
- package/dist/analysis/automatic-config.js +2 -1
- package/dist/analysis/automatic.js +2 -2
- package/dist/analysis/execute.d.ts +7 -10
- package/dist/analysis/execute.js +79 -49
- package/dist/analysis/output-limit.d.ts +12 -0
- package/dist/analysis/output-limit.js +34 -0
- package/dist/analysis/output-retry.d.ts +20 -0
- package/dist/analysis/output-retry.js +63 -0
- package/dist/analysis/provider.d.ts +1 -1
- package/dist/analysis/provider.js +3 -3
- package/dist/analysis/responses.js +1 -1
- package/dist/analysis/service.js +2 -2
- package/dist/cli/commands/analyze.js +1 -1
- package/dist/cli/io.js +2 -0
- package/dist/cli/run-findings.js +10 -2
- package/dist/observer-app.html +1 -1
- package/dist/remote/launch.js +84 -44
- package/dist/remote/runner.d.ts +47 -14
- package/dist/remote/runner.js +199 -64
- package/dist/routes/computer-use/e2b-desktop/desktop.js +47 -7
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +4 -2
- package/dist/routes/computer-use/e2b-desktop/prepare.js +4 -2
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +14 -6
- package/dist/routes/computer-use/e2b-desktop/start.js +72 -34
- package/dist/substrates/e2b/cdp-probe.d.ts +16 -1
- package/dist/substrates/e2b/cdp-probe.js +67 -5
- package/dist/substrates/e2b/desktop-browser.js +4 -2
- package/dist/substrates/e2b/desktop-cdp.d.ts +9 -2
- package/dist/substrates/e2b/desktop-cdp.js +26 -4
- package/docs/contracts/schemas.md +1 -1
- package/docs/contracts/study-analysis.md +62 -9
- package/docs/product/automatic-analysis.md +9 -6
- package/package.json +5 -5
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,17 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.146.0: Studies start and finish sooner (2026-10-23)
|
|
12
|
+
|
|
13
|
+
humanish 0.146.0 cuts the wait around a hosted study. A Chrome participant starts once its page
|
|
14
|
+
has loaded, in place of a fixed 8 s sleep, and a remote run sets up its runner and returns its
|
|
15
|
+
evidence faster; on a one-participant study, desktop to session start fell from about 29 s to 11
|
|
16
|
+
to 18 s, and a remote run's overhead from 17.8 s to about 10 s. An analysis request that runs out
|
|
17
|
+
of output is sent again once within its cost cap, and a low- or medium-effort analysis asks for an
|
|
18
|
+
output limit sized to its participants, so a runaway request stops sooner.
|
|
19
|
+
|
|
20
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.146.0)
|
|
21
|
+
|
|
11
22
|
## 0.145.0: A remote runner's keys stay in E2B's proxy (2026-10-22)
|
|
12
23
|
|
|
13
24
|
humanish 0.145.0 keeps the OpenAI, Gemini and E2B keys out of a remote E2B runner's environment.
|
|
@@ -30,14 +30,19 @@ export interface AnalysisRequests {
|
|
|
30
30
|
/** The analyst's EXPECTED_OUTPUT_SCALE; 1 when omitted. */
|
|
31
31
|
outputScale?: number;
|
|
32
32
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
expectedOutputTokens: number;
|
|
33
|
+
/** What one request is expected to cost, its worst case, and the figure admission compares. */
|
|
34
|
+
export interface RequestCost {
|
|
36
35
|
expectedCostUsd: number;
|
|
37
36
|
worstCaseCostUsd: number;
|
|
38
37
|
/** The expected cost with the margin, never above the worst case: what admission compares. */
|
|
39
38
|
admittedCostUsd: number;
|
|
40
39
|
}
|
|
40
|
+
export interface AnalysisCostEstimate extends RequestCost {
|
|
41
|
+
inputTokens: number;
|
|
42
|
+
expectedOutputTokens: number;
|
|
43
|
+
/** Each request's own costs: the cohorts' in order, then the merge request's. */
|
|
44
|
+
requests: RequestCost[];
|
|
45
|
+
}
|
|
41
46
|
/**
|
|
42
47
|
* Every request of one analysis, summed. A merge request reads each cohort's report, priced at
|
|
43
48
|
* that request's whole output allowance since the report and its reasoning share it, and is
|
|
@@ -85,11 +85,15 @@ export function estimateAnalysisCost(rate, requests) {
|
|
|
85
85
|
return {
|
|
86
86
|
inputTokens: sum("inputTokens"),
|
|
87
87
|
expectedOutputTokens: sum("expectedOutputTokens"),
|
|
88
|
-
expectedCostUsd,
|
|
89
|
-
worstCaseCostUsd,
|
|
90
|
-
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
88
|
+
...admitted(expectedCostUsd, worstCaseCostUsd),
|
|
89
|
+
requests: estimates.map((estimate) => admitted(estimate.expectedCostUsd, estimate.worstCaseCostUsd)),
|
|
91
90
|
};
|
|
92
91
|
}
|
|
92
|
+
const admitted = (expectedCostUsd, worstCaseCostUsd) => ({
|
|
93
|
+
expectedCostUsd,
|
|
94
|
+
worstCaseCostUsd,
|
|
95
|
+
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
96
|
+
});
|
|
93
97
|
/**
|
|
94
98
|
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
95
99
|
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import {
|
|
2
|
+
import { analysisCostRange } from "./execute.js";
|
|
3
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS } from "./output-limit.js";
|
|
3
4
|
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
4
5
|
import { isReasoningEffort } from "../actors/reasoning-effort.js";
|
|
5
6
|
import { admissionRule } from "./admission.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { resolveRunPath } from "../run/locate.js";
|
|
3
3
|
import { refreshObserver, analyzeRun, readCompletedAnalysisSource, resolveAnalysisRun, } from "./service.js";
|
|
4
|
-
import {
|
|
4
|
+
import { defaultAnalysisOutput, ANALYSIS_PROMPT_VERSION } from "./execute.js";
|
|
5
5
|
import { captureEvidence } from "./evidence.js";
|
|
6
6
|
import { analysisApiKey } from "../keys/key-resolution.js";
|
|
7
7
|
import { loadRunTraces } from "../run/stream-trace.js";
|
|
@@ -103,7 +103,7 @@ async function readAutomaticSource(cwd, prepared, configInput, deps, hasKey) {
|
|
|
103
103
|
deps.preferLargerOutput &&
|
|
104
104
|
(!deps.defaultRequest || participantEvidence) &&
|
|
105
105
|
hasKey) {
|
|
106
|
-
config =
|
|
106
|
+
config = defaultAnalysisOutput(await captureEvidence(prepared, bytes), config);
|
|
107
107
|
}
|
|
108
108
|
return { participantEvidence, config };
|
|
109
109
|
}
|
|
@@ -2,10 +2,6 @@ import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "
|
|
|
2
2
|
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
4
|
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
|
-
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
6
|
-
* returns at most 32,768 output tokens from one generateContent request. */
|
|
7
|
-
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
|
8
|
-
export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
|
|
9
5
|
export interface AnalysisAdmission {
|
|
10
6
|
allowed: boolean;
|
|
11
7
|
error: string | null;
|
|
@@ -65,14 +61,15 @@ export type AnalysisCostRange = {
|
|
|
65
61
|
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
66
62
|
* passes the cap. There is one packet per cohort, each a share of its own limits, and the merge
|
|
67
63
|
* request when there is more than one cohort. With `preferLargerOutput` each request gets the
|
|
68
|
-
* output allowance dispatch would give it (
|
|
69
|
-
*
|
|
64
|
+
* output allowance dispatch would give it (defaultAnalysisOutput). Undefined for Codex and for a
|
|
65
|
+
* model without rates.
|
|
70
66
|
*/
|
|
71
67
|
export declare function analysisCostRange(config: AnalysisConfig, participants: number, preferLargerOutput?: boolean): AnalysisCostRange | undefined;
|
|
72
|
-
/** Only for an omitted output limit
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
|
|
68
|
+
/** Only for an omitted output limit, which configuration fills with the 16,384-token default.
|
|
69
|
+
* Gives the analysis the allowance sized to its participants, effort and analyst, and keeps the
|
|
70
|
+
* default when a larger sized allowance would refuse a study its declared budget admits. This is
|
|
71
|
+
* one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
72
|
+
export declare function defaultAnalysisOutput(input: AnalysisInput, config: AnalysisConfig): AnalysisConfig;
|
|
76
73
|
export type AnalysisDispatchContext = Pick<AnalysisArtifact, "id" | "runId" | "sourceRunSha256" | "inputDigest" | "configDigest" | "promptVersion">;
|
|
77
74
|
interface RunAnalysisOptions {
|
|
78
75
|
apiKey?: string;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { validCodexAnalysisConfig } from "./codex-config.js";
|
|
2
2
|
import { INPUT_IMAGE_DATA_URL, highDetailImageTokens, imageTokenCeiling } from "./image-tokens.js";
|
|
3
|
-
import { EXPECTED_OUTPUT_SCALE, estimateAnalysisCost } from "./admission.js";
|
|
3
|
+
import { EXPECTED_OUTPUT_SCALE, estimateAnalysisCost, } from "./admission.js";
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
5
|
import { modelRate } from "../run/pricing.js";
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
@@ -13,11 +13,9 @@ import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analys
|
|
|
13
13
|
import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema, } from "./cohorts.js";
|
|
14
14
|
import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
|
|
15
15
|
import { mapWithConcurrency } from "../run/concurrency.js";
|
|
16
|
+
import { outputRetries } from "./output-retry.js";
|
|
17
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, defaultOutputTokens } from "./output-limit.js";
|
|
16
18
|
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
17
|
-
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
18
|
-
* returns at most 32,768 output tokens from one generateContent request. */
|
|
19
|
-
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
20
|
-
export const MAX_ANALYSIS_OUTPUT_TOKENS = 32_768;
|
|
21
19
|
const MAX_EVIDENCE_BYTES = 1024 * 1024;
|
|
22
20
|
/**
|
|
23
21
|
* Cohort requests in flight at once. OpenAI's lowest paid tier allows gpt-6-astra 1,000,000 tokens
|
|
@@ -214,21 +212,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
214
212
|
!Number.isFinite(rate.outputUsdPerToken) ||
|
|
215
213
|
rate.outputUsdPerToken < 0)
|
|
216
214
|
return denied("analysis_rate_unknown");
|
|
217
|
-
const
|
|
218
|
-
const cost = estimateAnalysisCost(rate, {
|
|
219
|
-
cohorts: cohortInputs(input).map((cohort) => {
|
|
220
|
-
const request = cohortRequest(cohort, config);
|
|
221
|
-
return {
|
|
222
|
-
textBytes: requestBytes(request),
|
|
223
|
-
imageTokens: request.images.reduce((sum, image) => sum + highDetailImageTokens(config.model, image.dataUrl), 0),
|
|
224
|
-
participants: cohort.participants.length,
|
|
225
|
-
outputAllowance: maxOutputTokens,
|
|
226
|
-
};
|
|
227
|
-
}),
|
|
228
|
-
// Without reports: estimateAnalysisCost prices them at the cohorts' output allowances.
|
|
229
|
-
mergeTextBytes: requestBytes(mergeRequest(input, config, [])),
|
|
230
|
-
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
231
|
-
});
|
|
215
|
+
const cost = analysisCost(input, config, rate);
|
|
232
216
|
if (!Number.isFinite(cost.worstCaseCostUsd))
|
|
233
217
|
return denied("analysis_rate_unknown");
|
|
234
218
|
const allowed = cost.admittedCostUsd <= config.maxCostUsd;
|
|
@@ -245,6 +229,24 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
245
229
|
requests,
|
|
246
230
|
};
|
|
247
231
|
}
|
|
232
|
+
/** Every request of an API analysis priced at `rate`: each cohort's, then the merge request's. */
|
|
233
|
+
function analysisCost(input, config, rate) {
|
|
234
|
+
const { maxOutputTokens } = config;
|
|
235
|
+
return estimateAnalysisCost(rate, {
|
|
236
|
+
cohorts: cohortInputs(input).map((cohort) => {
|
|
237
|
+
const request = cohortRequest(cohort, config);
|
|
238
|
+
return {
|
|
239
|
+
textBytes: requestBytes(request),
|
|
240
|
+
imageTokens: request.images.reduce((sum, image) => sum + highDetailImageTokens(config.model, image.dataUrl), 0),
|
|
241
|
+
participants: cohort.participants.length,
|
|
242
|
+
outputAllowance: maxOutputTokens,
|
|
243
|
+
};
|
|
244
|
+
}),
|
|
245
|
+
// Without reports: estimateAnalysisCost prices them at the cohorts' output allowances.
|
|
246
|
+
mergeTextBytes: requestBytes(mergeRequest(input, config, [])),
|
|
247
|
+
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
248
|
+
});
|
|
249
|
+
}
|
|
248
250
|
/** Structure each evidence entry adds to the packet: ids, kind, timestamps and flags. The largest
|
|
249
251
|
* retained packet averaged 200 bytes. A merge packet's participant entry is smaller. */
|
|
250
252
|
const ENTRY_STRUCTURE_BYTES = 256;
|
|
@@ -276,8 +278,8 @@ const RANGE_BISECTIONS = 40;
|
|
|
276
278
|
* the image-token ceiling, and the range ends at the largest one admission admits, so it never
|
|
277
279
|
* passes the cap. There is one packet per cohort, each a share of its own limits, and the merge
|
|
278
280
|
* request when there is more than one cohort. With `preferLargerOutput` each request gets the
|
|
279
|
-
* output allowance dispatch would give it (
|
|
280
|
-
*
|
|
281
|
+
* output allowance dispatch would give it (defaultAnalysisOutput). Undefined for Codex and for a
|
|
282
|
+
* model without rates.
|
|
281
283
|
*/
|
|
282
284
|
export function analysisCostRange(config, participants, preferLargerOutput = false) {
|
|
283
285
|
if (config.provider === "codex")
|
|
@@ -324,13 +326,16 @@ export function analysisCostRange(config, participants, preferLargerOutput = fal
|
|
|
324
326
|
}
|
|
325
327
|
return admitted;
|
|
326
328
|
};
|
|
327
|
-
// Dispatch tries the
|
|
328
|
-
// least the same share, and the packets past the
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
329
|
+
// Dispatch tries the sized allowance first. When it is the larger one, the smaller allowance
|
|
330
|
+
// costs no more, so it admits at least the same share, and the packets past the sized one's
|
|
331
|
+
// share get the smaller one. A sized allowance below the configured one is never refused where
|
|
332
|
+
// the configured one is admitted.
|
|
333
|
+
const sized = preferLargerOutput ? defaultOutput(config, covered) : undefined;
|
|
334
|
+
const allowances = sized === undefined
|
|
335
|
+
? [config.maxOutputTokens]
|
|
336
|
+
: sized.maxOutputTokens < config.maxOutputTokens
|
|
337
|
+
? [sized.maxOutputTokens]
|
|
338
|
+
: [sized.maxOutputTokens, config.maxOutputTokens];
|
|
334
339
|
const admitted = allowances.filter((allowance) => admits(0, allowance));
|
|
335
340
|
if (admitted.length === 0)
|
|
336
341
|
return { refusedFromUsd: request(0, config.maxOutputTokens).expectedCostUsd };
|
|
@@ -341,20 +346,23 @@ export function analysisCostRange(config, participants, preferLargerOutput = fal
|
|
|
341
346
|
},
|
|
342
347
|
};
|
|
343
348
|
}
|
|
344
|
-
/** The
|
|
345
|
-
|
|
346
|
-
|
|
349
|
+
/** The output allowance sized for an omitted output limit (output-limit.ts), or undefined when the
|
|
350
|
+
* limit was declared. */
|
|
351
|
+
function defaultOutput(config, participants) {
|
|
352
|
+
const tokens = defaultOutputTokens(config, participants);
|
|
353
|
+
if (config.provider === "codex" ||
|
|
354
|
+
tokens === undefined ||
|
|
355
|
+
config.maxOutputTokens !== DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS)
|
|
347
356
|
return undefined;
|
|
348
|
-
return { ...config, maxOutputTokens:
|
|
357
|
+
return { ...config, maxOutputTokens: tokens };
|
|
349
358
|
}
|
|
350
|
-
/** Only for an omitted output limit
|
|
351
|
-
*
|
|
352
|
-
*
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
: config;
|
|
359
|
+
/** Only for an omitted output limit, which configuration fills with the 16,384-token default.
|
|
360
|
+
* Gives the analysis the allowance sized to its participants, effort and analyst, and keeps the
|
|
361
|
+
* default when a larger sized allowance would refuse a study its declared budget admits. This is
|
|
362
|
+
* one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
363
|
+
export function defaultAnalysisOutput(input, config) {
|
|
364
|
+
const sized = defaultOutput(config, input.coverage.includedStreamIds.length);
|
|
365
|
+
return sized !== undefined && estimateAnalysisAdmission(input, sized).allowed ? sized : config;
|
|
358
366
|
}
|
|
359
367
|
/** Check the caller's id, input metadata and admission; throw the stable code a direct caller receives. */
|
|
360
368
|
function admitDirectAnalysis(input, config, analysisId) {
|
|
@@ -423,14 +431,15 @@ async function analysisProvider(config, options) {
|
|
|
423
431
|
* The attempt's requests: one per cohort, `COHORT_CONCURRENCY` at a time (one at a time for Codex),
|
|
424
432
|
* then the merge request for more than one cohort. After one request fails no further request
|
|
425
433
|
* starts, and the first failure in cohort order is the attempt's, with its cohort. `notSent` means
|
|
426
|
-
* cancellation stopped a request from starting.
|
|
434
|
+
* cancellation stopped a request from starting. `send` gets each request's index: its cohort's,
|
|
435
|
+
* and the cohort count for the merge request.
|
|
427
436
|
*/
|
|
428
437
|
async function requestReport(cohorts, input, config, send, signal) {
|
|
429
438
|
let stopped = false;
|
|
430
|
-
const outcomes = await mapWithConcurrency([...cohorts], config.provider === "codex" ? 1 : COHORT_CONCURRENCY, async (cohort) => {
|
|
439
|
+
const outcomes = await mapWithConcurrency([...cohorts], config.provider === "codex" ? 1 : COHORT_CONCURRENCY, async (cohort, index) => {
|
|
431
440
|
if (stopped || signal?.aborted)
|
|
432
441
|
return undefined;
|
|
433
|
-
const outcome = await send(cohortRequest(cohort, config), cohort);
|
|
442
|
+
const outcome = await send(cohortRequest(cohort, config), cohort, index);
|
|
434
443
|
if (!outcome.ok)
|
|
435
444
|
stopped = true;
|
|
436
445
|
return outcome;
|
|
@@ -446,7 +455,16 @@ async function requestReport(cohorts, input, config, send, signal) {
|
|
|
446
455
|
if (signal?.aborted)
|
|
447
456
|
return { ok: false, notSent: true };
|
|
448
457
|
// The merge request sends the cohort reports, no evidence entries or captures.
|
|
449
|
-
return send(mergeRequest(input, config, reports), { ...input, evidence: [], images: [] });
|
|
458
|
+
return send(mergeRequest(input, config, reports), { ...input, evidence: [], images: [] }, cohorts.length);
|
|
459
|
+
}
|
|
460
|
+
/** Request `index` of an attempt at the start of a sentence, numbered as requestReport numbers it. */
|
|
461
|
+
function requestName(cohorts, index) {
|
|
462
|
+
if (cohorts.length === 1)
|
|
463
|
+
return "The analysis request";
|
|
464
|
+
if (index === cohorts.length)
|
|
465
|
+
return `The request that merges the ${cohorts.length} cohort reports`;
|
|
466
|
+
const total = cohorts.reduce((sum, cohort) => sum + cohort.participants.length, 0);
|
|
467
|
+
return `The request for one cohort (${cohorts[index].participants.length} of the ${total} participants)`;
|
|
450
468
|
}
|
|
451
469
|
/**
|
|
452
470
|
* Why a run analysed in cohorts has no report, for the attempt's warnings. Null for one request,
|
|
@@ -520,7 +538,10 @@ export async function runAnalysis(input, config, options) {
|
|
|
520
538
|
const provider = await analysisProvider(config, options);
|
|
521
539
|
const cohorts = cohortInputs(input);
|
|
522
540
|
const responses = [];
|
|
523
|
-
const
|
|
541
|
+
const retries = config.provider === "codex"
|
|
542
|
+
? undefined
|
|
543
|
+
: outputRetries(config, analysisCost(input, config, modelRate(config.model)).requests, (index) => requestName(cohorts, index), options.warnings);
|
|
544
|
+
const dispatch = async (request, part) => {
|
|
524
545
|
progress("requesting", part);
|
|
525
546
|
const response = await provider({
|
|
526
547
|
model: config.model,
|
|
@@ -538,6 +559,14 @@ export async function runAnalysis(input, config, options) {
|
|
|
538
559
|
for (const warning of [...codexWarnings(config, response), httpStatusWarning(config, response)])
|
|
539
560
|
if (warning !== undefined && !options.warnings?.includes(warning))
|
|
540
561
|
options.warnings?.push(warning);
|
|
562
|
+
return response;
|
|
563
|
+
};
|
|
564
|
+
const send = async (request, part, index) => {
|
|
565
|
+
let response = await dispatch(request, part);
|
|
566
|
+
// A request that ran out of output is sent once more while the cap admits it.
|
|
567
|
+
while (!options.signal?.aborted && retries?.retry(index, response, artifact.usage))
|
|
568
|
+
response = await dispatch(request, part);
|
|
569
|
+
retries?.settle(index);
|
|
541
570
|
if (response.status !== "completed")
|
|
542
571
|
return {
|
|
543
572
|
ok: false,
|
|
@@ -569,11 +598,12 @@ export async function runAnalysis(input, config, options) {
|
|
|
569
598
|
artifact.result = outcome.result;
|
|
570
599
|
artifact.status = input.coverage.complete ? "complete" : "partial";
|
|
571
600
|
artifact.error = null;
|
|
572
|
-
// A bill above the expected cost is a normal outcome. Only one above the worst case
|
|
573
|
-
// shows the estimate was wrong.
|
|
601
|
+
// A bill above the expected cost is a normal outcome. Only one above the worst case, with the
|
|
602
|
+
// worst case of each request sent again, or above the cap shows the estimate was wrong.
|
|
603
|
+
const worstCaseUsd = (admission.worstCaseCostUsd ?? config.maxCostUsd ?? 0) + (retries?.worstCaseUsd() ?? 0);
|
|
574
604
|
if (config.provider !== "codex" &&
|
|
575
605
|
(responses.some((response) => (response.usage?.output ?? 0) > config.maxOutputTokens) ||
|
|
576
|
-
(artifact.usage.estimatedCostUsd ?? 0) >
|
|
606
|
+
(artifact.usage.estimatedCostUsd ?? 0) > worstCaseUsd ||
|
|
577
607
|
(artifact.usage.estimatedCostUsd ?? 0) > config.maxCostUsd)) {
|
|
578
608
|
artifact.status = "partial";
|
|
579
609
|
artifact.error = "analysis_admission_estimate_exceeded";
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AnalysisConfig } from "./types.js";
|
|
2
|
+
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
3
|
+
* returns at most 32,768 output tokens from one generateContent request. */
|
|
4
|
+
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
|
5
|
+
export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
|
|
6
|
+
/**
|
|
7
|
+
* The output limit an analysis of `participants` asks for when its study declares none, or
|
|
8
|
+
* undefined for Codex, which takes no limit. A high-effort or Gemini analysis, and one analysed in
|
|
9
|
+
* cohorts, gets the most the analysts return: at those efforts one finished report of three
|
|
10
|
+
* participants used up to 26,439 tokens.
|
|
11
|
+
*/
|
|
12
|
+
export declare function defaultOutputTokens(config: AnalysisConfig, participants: number): number | undefined;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// The output limit an API analysis asks for when its study declares none. A request that runs
|
|
2
|
+
// away writes until it reaches the limit, so the limit is sized to what a finished report of the
|
|
3
|
+
// packet needs, with room to spare.
|
|
4
|
+
import { analysisCohortCount } from "./analysis-limits.js";
|
|
5
|
+
import { analysisEffort } from "./models.js";
|
|
6
|
+
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
7
|
+
* returns at most 32,768 output tokens from one generateContent request. */
|
|
8
|
+
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
9
|
+
export const MAX_ANALYSIS_OUTPUT_TOKENS = 32_768;
|
|
10
|
+
/**
|
|
11
|
+
* The limit of a low- or medium-effort OpenAI request: this many tokens, and this many more for
|
|
12
|
+
* each participant. On the 2026-10-22 and 2026-10-23 benchmark recordings the longest finished
|
|
13
|
+
* report at those efforts used 8,354 tokens for one participant and 11,313 for three (gpt-6-luna at
|
|
14
|
+
* medium; gpt-6.1-sol at medium used at most 5,659 and 9,000), so the limit is 1.7 and 1.6 times
|
|
15
|
+
* those. docs/contracts/study-analysis.md has the table.
|
|
16
|
+
*/
|
|
17
|
+
const LIMIT_BASE_TOKENS = 12_288;
|
|
18
|
+
const LIMIT_TOKENS_PER_PARTICIPANT = 2_048;
|
|
19
|
+
const SIZED_EFFORTS = new Set(["none", "low", "medium"]);
|
|
20
|
+
/**
|
|
21
|
+
* The output limit an analysis of `participants` asks for when its study declares none, or
|
|
22
|
+
* undefined for Codex, which takes no limit. A high-effort or Gemini analysis, and one analysed in
|
|
23
|
+
* cohorts, gets the most the analysts return: at those efforts one finished report of three
|
|
24
|
+
* participants used up to 26,439 tokens.
|
|
25
|
+
*/
|
|
26
|
+
export function defaultOutputTokens(config, participants) {
|
|
27
|
+
if (config.provider === "codex")
|
|
28
|
+
return undefined;
|
|
29
|
+
if (config.provider === "gemini" ||
|
|
30
|
+
!SIZED_EFFORTS.has(analysisEffort(config)) ||
|
|
31
|
+
analysisCohortCount(participants) > 1)
|
|
32
|
+
return MAX_ANALYSIS_OUTPUT_TOKENS;
|
|
33
|
+
return Math.min(MAX_ANALYSIS_OUTPUT_TOKENS, LIMIT_BASE_TOKENS + LIMIT_TOKENS_PER_PARTICIPANT * participants);
|
|
34
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { RequestCost } from "./admission.js";
|
|
2
|
+
import type { AnalysisProviderResult } from "./provider.js";
|
|
3
|
+
import type { AnalysisConfig, AnalysisUsage } from "./types.js";
|
|
4
|
+
export interface OutputRetries {
|
|
5
|
+
/**
|
|
6
|
+
* Whether to send request `index` again after `response`, given the attempt's usage with that
|
|
7
|
+
* response counted. Adds a warning that says what ran out and what humanish did.
|
|
8
|
+
*/
|
|
9
|
+
retry(index: number, response: AnalysisProviderResult, usage: AnalysisUsage): boolean;
|
|
10
|
+
/** Request `index` has its final answer, so its cost is no longer held against the cap. */
|
|
11
|
+
settle(index: number): void;
|
|
12
|
+
/** The summed worst case of every request sent again, which admission did not price. */
|
|
13
|
+
worstCaseUsd(): number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The retries of one attempt. `costs` are each request's costs from admission (the cohorts', then
|
|
17
|
+
* the merge request's), `describe` names request `index` at the start of a sentence, and warnings
|
|
18
|
+
* receive what happened.
|
|
19
|
+
*/
|
|
20
|
+
export declare function outputRetries(config: AnalysisConfig, costs: readonly RequestCost[], describe: (index: number) => string, warnings: string[] | undefined): OutputRetries;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// One more request when an API analyst runs out of output before it finishes its report. The same
|
|
2
|
+
// request is sent again, at most once, and only while the analysis's cost cap still admits it
|
|
3
|
+
// after counting what every request so far cost. Codex account analysis has no dollar cap to
|
|
4
|
+
// check, so it is never sent again.
|
|
5
|
+
import { estimateActorCost } from "../run/pricing.js";
|
|
6
|
+
const usd = (value) => `$${value.toFixed(2)}`;
|
|
7
|
+
/**
|
|
8
|
+
* The retries of one attempt. `costs` are each request's costs from admission (the cohorts', then
|
|
9
|
+
* the merge request's), `describe` names request `index` at the start of a sentence, and warnings
|
|
10
|
+
* receive what happened.
|
|
11
|
+
*/
|
|
12
|
+
export function outputRetries(config, costs, describe, warnings) {
|
|
13
|
+
// What each request without a final answer may still cost: admission's figure for it.
|
|
14
|
+
const held = new Map(costs.map((cost, index) => [index, cost.admittedCostUsd]));
|
|
15
|
+
const retried = new Set();
|
|
16
|
+
let worstCase = 0;
|
|
17
|
+
const warn = (text) => {
|
|
18
|
+
warnings?.push(text);
|
|
19
|
+
};
|
|
20
|
+
return {
|
|
21
|
+
retry(index, response, usage) {
|
|
22
|
+
if (response.errorCode !== "output_incomplete" || config.maxCostUsd === null)
|
|
23
|
+
return false;
|
|
24
|
+
const which = describe(index);
|
|
25
|
+
if (retried.has(index)) {
|
|
26
|
+
warn(`${which} ran out of output again, so this attempt has no report.`);
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
const cap = config.maxCostUsd;
|
|
30
|
+
const tokens = response.usage;
|
|
31
|
+
const cost = tokens === null
|
|
32
|
+
? null
|
|
33
|
+
: estimateActorCost({ ...tokens, turns: [tokens] }, config.model).estimatedCostUsd;
|
|
34
|
+
if (tokens === null ||
|
|
35
|
+
cost === null ||
|
|
36
|
+
!usage.usageComplete ||
|
|
37
|
+
usage.estimatedCostUsd === null) {
|
|
38
|
+
warn(`${which} ran out of output before it finished its report and did not report its usage, so humanish cannot tell whether a second request fits the $${cap} cap. It was not sent again.`);
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
const ranOut = `${which} ran out of output before it finished its report: ${tokens.output} of ${config.maxOutputTokens} output tokens, ${tokens.input} input tokens, ${usd(cost)}.`;
|
|
42
|
+
const again = costs[index];
|
|
43
|
+
let others = 0;
|
|
44
|
+
for (const [other, amount] of held)
|
|
45
|
+
if (other !== index)
|
|
46
|
+
others += amount;
|
|
47
|
+
const total = usage.estimatedCostUsd + others + again.admittedCostUsd;
|
|
48
|
+
if (total > cap) {
|
|
49
|
+
const running = others > 0 ? `, the ${usd(others)} admission holds for requests still running,` : "";
|
|
50
|
+
warn(`${ranOut} humanish did not send it again: the ${usd(usage.estimatedCostUsd)} spent so far${running} plus ${usd(again.admittedCostUsd)} for one more request at admission's estimate is over the $${cap} cap. To retry, run the analysis again; a higher cost cap leaves room for a second request.`);
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
retried.add(index);
|
|
54
|
+
worstCase += again.worstCaseCostUsd;
|
|
55
|
+
warn(`${ranOut} humanish sent the same request once more; the attempt's usage counts both requests.`);
|
|
56
|
+
return true;
|
|
57
|
+
},
|
|
58
|
+
settle(index) {
|
|
59
|
+
held.delete(index);
|
|
60
|
+
},
|
|
61
|
+
worstCaseUsd: () => worstCase,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -73,7 +73,7 @@ export interface AnalysisWire {
|
|
|
73
73
|
body(request: AnalysisProviderRequest): unknown;
|
|
74
74
|
parse(raw: unknown): AnalysisProviderResult;
|
|
75
75
|
}
|
|
76
|
-
/** One request to the vendor's fixed endpoint,
|
|
76
|
+
/** One request to the vendor's fixed endpoint, sent once and never elsewhere. */
|
|
77
77
|
export declare function createAnalysisProvider(options: {
|
|
78
78
|
apiKey: string;
|
|
79
79
|
/** Omitted is OpenAI, the analyst humanish has always called. */
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// The analysis provider seam and the HTTP transport its API adapters share. Each API vendor owns
|
|
2
2
|
// only its wire format (gemini-wire.ts, openai-wire.ts): the endpoint, the request body, and how a
|
|
3
3
|
// response becomes a result. The transport owns what must hold for every vendor: one fixed endpoint
|
|
4
|
-
// per vendor, request and response byte caps, the caller's wall-clock deadline,
|
|
5
|
-
// closed error vocabulary that never carries provider error prose. The Codex account analyst is
|
|
4
|
+
// per vendor, request and response byte caps, the caller's wall-clock deadline, one send per call
|
|
5
|
+
// (runAnalysis decides whether to call again, in output-retry.ts), and a closed error vocabulary that never carries provider error prose. The Codex account analyst is
|
|
6
6
|
// the third adapter at the same seam (restricted-codex.ts).
|
|
7
7
|
import { Agent, fetch as undiciFetch } from "undici";
|
|
8
8
|
import { isRecord } from "../run/type-guards.js";
|
|
@@ -49,7 +49,7 @@ const WIRES = {
|
|
|
49
49
|
openai: openaiWire,
|
|
50
50
|
gemini: geminiWire,
|
|
51
51
|
};
|
|
52
|
-
/** One request to the vendor's fixed endpoint,
|
|
52
|
+
/** One request to the vendor's fixed endpoint, sent once and never elsewhere. */
|
|
53
53
|
export function createAnalysisProvider(options) {
|
|
54
54
|
const fetchFn = options.fetchFn ?? undiciFetch;
|
|
55
55
|
const wire = WIRES[options.vendor ?? "openai"];
|
|
@@ -202,7 +202,7 @@ export function httpStatusWarning(config, response) {
|
|
|
202
202
|
return undefined;
|
|
203
203
|
const api = config.provider === "gemini" ? "Gemini" : "OpenAI";
|
|
204
204
|
const meaning = HTTP_STATUS_MEANINGS[response.httpStatus];
|
|
205
|
-
return `The ${api} API answered HTTP ${response.httpStatus}${meaning === undefined ? "" : ` (${meaning})`}. humanish
|
|
205
|
+
return `The ${api} API answered HTTP ${response.httpStatus}${meaning === undefined ? "" : ` (${meaning})`}. humanish sends a request again only when it ran out of output; run the command again to retry.`;
|
|
206
206
|
}
|
|
207
207
|
/** The warnings a Codex response carries about its CLI release and protocol. */
|
|
208
208
|
export function codexWarnings(config, response) {
|
package/dist/analysis/service.js
CHANGED
|
@@ -17,7 +17,7 @@ import { withAnalysisLock } from "./analysis-lock.js";
|
|
|
17
17
|
import { captureEvidence, EVIDENCE_LIMITS } from "./evidence.js";
|
|
18
18
|
import { AnalysisSourceRefusedError, runFileTooLarge, throwAsAnalysisRefusal } from "./source.js";
|
|
19
19
|
import { pathMissing, readBoundedFile, readBoundedFileResult } from "../run/evidence-files.js";
|
|
20
|
-
import { estimateAnalysisAdmission,
|
|
20
|
+
import { estimateAnalysisAdmission, defaultAnalysisOutput, runAnalysis, ANALYSIS_PROMPT_VERSION, } from "./execute.js";
|
|
21
21
|
import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalyses, writeAnalysis, } from "./store.js";
|
|
22
22
|
import { beginAnalysisExecution, writeAnalysisExecutionReceipt } from "./store-executions.js";
|
|
23
23
|
import { keepRejectedAnalysisOutput } from "./diagnostics.js";
|
|
@@ -261,7 +261,7 @@ async function executeAnalysis(attempt) {
|
|
|
261
261
|
if (input.evidence.length === 0)
|
|
262
262
|
return fail(input.runId, dryRun, "ANALYSIS_NO_PARTICIPANTS");
|
|
263
263
|
if (options.preferLargerOutput)
|
|
264
|
-
config =
|
|
264
|
+
config = defaultAnalysisOutput(input, config);
|
|
265
265
|
const admission = estimateAnalysisAdmission(input, config);
|
|
266
266
|
const base = {
|
|
267
267
|
schema: ANALYZE_RESULT_SCHEMA,
|
|
@@ -38,7 +38,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
38
38
|
.option("--reasoning-effort <level>", `OpenAI reasoning.effort or Gemini thinking level, from none, minimal, low, medium, high, xhigh and max; each model takes only the levels its provider documents for it, and others are refused before any request. When omitted: ${DEFAULT_ANALYSIS_EFFORT} for the default OpenAI analyst, high with --model or --provider gemini. Unsupported for Codex.`)
|
|
39
39
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
40
40
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
41
|
-
.option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for
|
|
41
|
+
.option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for a limit sized to the participants and effort. Unsupported for Codex.")
|
|
42
42
|
.option("--dry-run", "Capture and validate local input and estimate admission; no request or analysis artifact.")
|
|
43
43
|
.option("--rerun", "Create a new immutable version even when the same input and configuration were analyzed.")
|
|
44
44
|
.option("--json", JSON_OPTION_DESCRIPTION)
|
package/dist/cli/io.js
CHANGED
|
@@ -208,6 +208,8 @@ export function writeResult(command, io, result, formatHuman, analysisLines) {
|
|
|
208
208
|
io.writeOut(`analysis: refused before it started. ${refusal.text} ${refusal.command}\n`);
|
|
209
209
|
else if (analysis)
|
|
210
210
|
io.writeOut(`analysis: ${analysisOutcomeText(analysis)}\n`);
|
|
211
|
+
for (const warning of analysis?.result?.warnings ?? [])
|
|
212
|
+
io.writeOut(`analysis: ${warning}\n`);
|
|
211
213
|
const rejected = analysis?.result?.rejectedOutputPath;
|
|
212
214
|
if (rejected)
|
|
213
215
|
io.writeOut(`analysis: ${analysis.result?.error?.code ?? "rejected"}; the rejected output is kept locally at ${rejected}\n`);
|
package/dist/cli/run-findings.js
CHANGED
|
@@ -33,10 +33,12 @@ async function readRunReport(cwd, result, deps) {
|
|
|
33
33
|
if (outcome === undefined || result.runId === undefined || result.runId === "not-created")
|
|
34
34
|
return undefined;
|
|
35
35
|
const outcomeLine = `analysis: ${analysisOutcomeText(outcome)}`;
|
|
36
|
+
// What the attempt warned about, such as a request sent again after it ran out of output.
|
|
37
|
+
const warningLines = (outcome.result?.warnings ?? []).map((warning) => `analysis: ${warning}`);
|
|
36
38
|
const plain = {
|
|
37
39
|
message: `The automatic analysis was ${analysisOutcomeText(outcome)}.`,
|
|
38
40
|
nextCommand: null,
|
|
39
|
-
lines: [outcomeLine],
|
|
41
|
+
lines: [outcomeLine, ...warningLines],
|
|
40
42
|
};
|
|
41
43
|
// A dry run left no evidence, and its own output names the live run.
|
|
42
44
|
if (outcome.reason === "AUTOMATIC_ANALYSIS_DRY_RUN")
|
|
@@ -61,7 +63,12 @@ async function readRunReport(cwd, result, deps) {
|
|
|
61
63
|
return {
|
|
62
64
|
message: view.message,
|
|
63
65
|
nextCommand: review,
|
|
64
|
-
lines: [
|
|
66
|
+
lines: [
|
|
67
|
+
outcomeLine,
|
|
68
|
+
...warningLines,
|
|
69
|
+
...rejectedLine,
|
|
70
|
+
...formatFindingsSummary(view, review),
|
|
71
|
+
],
|
|
65
72
|
findings: findingsSummary(view, review, result.observer?.observerPath),
|
|
66
73
|
};
|
|
67
74
|
// The line reads "analysis: skipped because ...", as it does without the view.
|
|
@@ -71,6 +78,7 @@ async function readRunReport(cwd, result, deps) {
|
|
|
71
78
|
nextCommand: view.next,
|
|
72
79
|
lines: [
|
|
73
80
|
`analysis: ${said}`,
|
|
81
|
+
...warningLines,
|
|
74
82
|
...rejectedLine,
|
|
75
83
|
...(view.next === null ? [] : [`next: ${view.next}`]),
|
|
76
84
|
],
|