humanish 0.141.0 → 0.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -0
- package/CHANGELOG.md +21 -0
- package/README.md +3 -1
- package/dist/analysis/admission.d.ts +8 -0
- package/dist/analysis/admission.js +15 -4
- package/dist/analysis/automatic-config.d.ts +17 -3
- package/dist/analysis/automatic-config.js +56 -12
- package/dist/analysis/automatic.js +13 -7
- package/dist/analysis/execute.d.ts +6 -5
- package/dist/analysis/execute.js +27 -26
- package/dist/analysis/gemini-wire.d.ts +9 -0
- package/dist/analysis/gemini-wire.js +166 -0
- package/dist/analysis/image-tokens.d.ts +4 -2
- package/dist/analysis/image-tokens.js +24 -3
- package/dist/analysis/job.d.ts +1 -0
- package/dist/analysis/job.js +1 -0
- package/dist/analysis/models.d.ts +12 -0
- package/dist/analysis/models.js +47 -0
- package/dist/analysis/openai-wire.d.ts +6 -0
- package/dist/analysis/openai-wire.js +102 -0
- package/dist/analysis/provider.d.ts +19 -7
- package/dist/analysis/provider.js +21 -101
- package/dist/analysis/responses.d.ts +3 -0
- package/dist/analysis/responses.js +19 -0
- package/dist/analysis/service.js +11 -3
- package/dist/analysis/types.d.ts +12 -2
- package/dist/analysis/validation.d.ts +1 -0
- package/dist/analysis/validation.js +17 -15
- package/dist/cli/commands/analyze.js +11 -7
- package/dist/cli/commands/observe.js +26 -1
- package/dist/cli/commands/run-command.d.ts +1 -1
- package/dist/cli/commands/run-command.js +12 -0
- package/dist/cli/commands/study-remote.d.ts +13 -0
- package/dist/cli/commands/study-remote.js +125 -0
- package/dist/cli/commands/study-route-computer-use.js +26 -2
- package/dist/cli/commands/study-route-scripted.js +1 -0
- package/dist/cli/commands/study-route-shared-world.js +1 -0
- package/dist/cli/commands/study-route-terminal.js +1 -0
- package/dist/cli/commands/study-run.js +34 -0
- package/dist/cli/commands/watch.js +5 -0
- package/dist/cli/doctor.js +7 -2
- package/dist/cli/findings.d.ts +9 -1
- package/dist/cli/findings.js +62 -17
- package/dist/cli/io.d.ts +8 -0
- package/dist/cli/io.js +2 -1
- package/dist/keys/key-resolution.d.ts +10 -0
- package/dist/keys/key-resolution.js +33 -5
- package/dist/observer/listen.d.ts +7 -2
- package/dist/observer/listen.js +10 -4
- package/dist/observer/render.d.ts +2 -0
- package/dist/observer/render.js +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/remote/launch.d.ts +56 -0
- package/dist/remote/launch.js +365 -0
- package/dist/remote/observer-proxy.d.ts +11 -0
- package/dist/remote/observer-proxy.js +66 -0
- package/dist/remote/plan.d.ts +35 -0
- package/dist/remote/plan.js +114 -0
- package/dist/remote/reattach.d.ts +17 -0
- package/dist/remote/reattach.js +81 -0
- package/dist/remote/runner.d.ts +84 -0
- package/dist/remote/runner.js +241 -0
- package/dist/routes/computer-use/e2b-desktop/start.js +1 -1
- package/dist/routes/computer-use/types.d.ts +2 -0
- package/dist/routes/scripted/types.d.ts +2 -0
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/routes/terminal/types.d.ts +2 -0
- package/dist/run/costs.d.ts +2 -1
- package/dist/run/pricing.d.ts +13 -1
- package/dist/run/pricing.js +50 -4
- package/dist/run/results.d.ts +1 -1
- package/dist/run/run-cost.d.ts +2 -1
- package/dist/run/run-cost.js +6 -1
- package/dist/run/run.d.ts +13 -0
- package/dist/run/run.js +18 -7
- package/dist/run/sandbox-creates.d.ts +10 -3
- package/dist/run/sandbox-creates.js +48 -6
- package/dist/run-study.d.ts +5 -0
- package/dist/study/doctor.js +14 -1
- package/dist/study/requirements.d.ts +1 -1
- package/dist/study/requirements.js +8 -6
- package/dist/study/route-inputs.js +4 -0
- package/dist/study/run-study-events.d.ts +1 -1
- package/dist/study/run-study-events.js +9 -2
- package/dist/substrates/e2b/base-module.d.ts +3 -0
- package/dist/substrates/e2b/base-module.js +40 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.js +24 -6
- package/dist/substrates/e2b/sdk.d.ts +24 -0
- package/dist/substrates/e2b/sdk.js +5 -1
- package/dist/substrates/e2b/shell-only.d.ts +5 -0
- package/dist/substrates/e2b/shell-only.js +27 -0
- package/dist/substrates/modal/sandbox.js +27 -28
- package/dist/substrates/modal/sdk.d.ts +7 -0
- package/dist/tui-app.js +46 -46
- package/docs/contracts/study-analysis.md +85 -4
- package/docs/product/automatic-analysis.md +42 -8
- package/package.json +2 -2
- package/skills/humanish/SKILL.md +18 -1
package/ARCHITECTURE.md
CHANGED
|
@@ -86,6 +86,7 @@ the analysis of the run:
|
|
|
86
86
|
| `src/actors/` | The actor contract, the registry and each actor's session code | `src/actors/registry.ts` |
|
|
87
87
|
| `src/subject/` | Subject provisioning over a `Shell`: clone, local tree, desktop CLI, serve | `src/subject/serve.ts` |
|
|
88
88
|
| `src/substrates/` | Provider primitives: E2B in `e2b/`, Firecracker and Lima VMs in `local/`, the `Shell` | `src/substrates/desktop-session.ts` |
|
|
89
|
+
| `src/remote/` | `--remote`: plan a run for a runner sandbox, launch and follow it there, copy it back | `src/remote/launch.ts` |
|
|
89
90
|
| `src/run/` | Run lifecycle, bundle types, bundle guards, paths, status, receipts, reclaim | `src/run/run.ts` |
|
|
90
91
|
| `src/verify/` | `humanish verify`: evidence checks, share-safety grades, cost and rerun checks | `src/verify/verify.ts` |
|
|
91
92
|
| `src/evidence/` | Redaction, screenshot checks and desktop recordings | `src/evidence/redaction.ts` |
|
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,27 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.143.0: Analysis costs about one seventh as much: gpt-6.1-sol at medium effort is the default (2026-10-22)
|
|
12
|
+
|
|
13
|
+
humanish 0.143.0 analyzes studies with `gpt-6.1-sol` at medium reasoning effort by default, in
|
|
14
|
+
place of `gpt-6-astra` at high. On the same recordings it found what `gpt-6-astra` found, invented
|
|
15
|
+
nothing, and cost about one seventh as much: $0.06 against $0.41 per analysis on the Taskly
|
|
16
|
+
benchmark. A study can set the analysis effort with `review.analysis.reasoningEffort`, and one
|
|
17
|
+
that names its model keeps it at high effort with an unchanged configuration hash.
|
|
18
|
+
|
|
19
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.143.0)
|
|
20
|
+
|
|
21
|
+
## 0.142.0: Remote runs, Gemini and GPT-6 analysis models, and a view-only live stream (2026-10-21)
|
|
22
|
+
|
|
23
|
+
humanish 0.142.0 can run a study remotely: `humanish run <study> --remote` puts the study's own
|
|
24
|
+
process in a sandbox, so a long study keeps going after this machine sleeps, and `observe --run
|
|
25
|
+
<id>` reconnects to it. Study analysis can run on `gpt-6.1-sol` and `gpt-6-luna` as well as the
|
|
26
|
+
default `gpt-6-astra`, and on Google's Gemini API with `humanish keys set gemini`. A hosted
|
|
27
|
+
desktop's live stream is now view-only on the VNC server itself: before, anyone holding a
|
|
28
|
+
participant's stream URL could type into that desktop.
|
|
29
|
+
|
|
30
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.142.0)
|
|
31
|
+
|
|
11
32
|
## 0.141.0: A run says when it is writing findings, and email links no longer open a dialog (2026-10-21)
|
|
12
33
|
|
|
13
34
|
humanish 0.141.0 says when a run is writing its findings: the CLI prints progress with the
|
package/README.md
CHANGED
|
@@ -242,12 +242,14 @@ beside Participants in Observer, with evidence links and separate participant
|
|
|
242
242
|
feedback. You can explicitly select a restricted Codex account analyst with
|
|
243
243
|
`npx humanish analyze --run latest --provider codex`; it uses remote inference,
|
|
244
244
|
requires the qualified CLI and account login, and reports dollar cost as unknown.
|
|
245
|
+
`npx humanish analyze --run latest --provider gemini --max-cost 2` sends the same
|
|
246
|
+
evidence to Google's Gemini API instead, with the key from `npx humanish keys set gemini`.
|
|
245
247
|
A study whose participant is a Codex `local-agent` uses that analyst by default. See the [analysis contract](docs/contracts/study-analysis.md) for
|
|
246
248
|
coverage limits, estimated cost controls, corrections and sharing behavior.
|
|
247
249
|
|
|
248
250
|
Supported live studies generate findings automatically after participants finish.
|
|
249
251
|
A Codex participant's run is analyzed by the same Codex account; every other run uses
|
|
250
|
-
`gpt-6-
|
|
252
|
+
`gpt-6.1-sol` at medium reasoning effort and a separate $10 cap on the expected cost.
|
|
251
253
|
This is additional to participant execution costs, and is not a hard provider
|
|
252
254
|
billing cap. To disable the extra request:
|
|
253
255
|
|
|
@@ -1,4 +1,10 @@
|
|
|
1
1
|
import type { ModelRate } from "../run/pricing.js";
|
|
2
|
+
/**
|
|
3
|
+
* How many times that expectation an API analyst writes. The OpenAI figures above come from its
|
|
4
|
+
* billed analyses. A gemini-3.8-flash analysis of one participant at high thinking wrote 24,246
|
|
5
|
+
* output tokens against 13,000 expected, so Gemini's expectation is doubled. That rests on one run.
|
|
6
|
+
*/
|
|
7
|
+
export declare const EXPECTED_OUTPUT_SCALE: Readonly<Record<"openai" | "gemini", number>>;
|
|
2
8
|
/**
|
|
3
9
|
* When admission refuses a request, in words, for the cap as the sentence names it: `$3`, or `this`
|
|
4
10
|
* in the help of the option that sets it. Admission compares the smaller of the worst case and the
|
|
@@ -21,6 +27,8 @@ interface AnalysisRequestSize {
|
|
|
21
27
|
export interface AnalysisRequests {
|
|
22
28
|
cohorts: readonly AnalysisRequestSize[];
|
|
23
29
|
mergeTextBytes: number;
|
|
30
|
+
/** The analyst's EXPECTED_OUTPUT_SCALE; 1 when omitted. */
|
|
31
|
+
outputScale?: number;
|
|
24
32
|
}
|
|
25
33
|
export interface AnalysisCostEstimate {
|
|
26
34
|
inputTokens: number;
|
|
@@ -15,6 +15,15 @@ const FRAMING_TOKENS = 2048;
|
|
|
15
15
|
*/
|
|
16
16
|
const EXPECTED_OUTPUT_TOKENS = 12_000;
|
|
17
17
|
const EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT = 1_000;
|
|
18
|
+
/**
|
|
19
|
+
* How many times that expectation an API analyst writes. The OpenAI figures above come from its
|
|
20
|
+
* billed analyses. A gemini-3.8-flash analysis of one participant at high thinking wrote 24,246
|
|
21
|
+
* output tokens against 13,000 expected, so Gemini's expectation is doubled. That rests on one run.
|
|
22
|
+
*/
|
|
23
|
+
export const EXPECTED_OUTPUT_SCALE = {
|
|
24
|
+
openai: 1,
|
|
25
|
+
gemini: 2,
|
|
26
|
+
};
|
|
18
27
|
/** Admission compares the expected cost times this margin with the cap. */
|
|
19
28
|
const ADMISSION_MARGIN = 1.1;
|
|
20
29
|
const MARGIN_PERCENT = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
@@ -38,12 +47,13 @@ function requestCost(rate, inputTokens, outputTokens) {
|
|
|
38
47
|
outputTokens * rate.outputUsdPerToken * (long ? rate.longContext.outputMultiplier : 1));
|
|
39
48
|
}
|
|
40
49
|
/** One request's input and costs. `reportTokens` are earlier requests' reports it reads. */
|
|
41
|
-
function estimateRequest(rate, size, reportTokens = 0) {
|
|
50
|
+
function estimateRequest(rate, size, outputScale, reportTokens = 0) {
|
|
42
51
|
const inputTokens = Math.ceil(size.textBytes / BYTES_PER_INPUT_TOKEN) +
|
|
43
52
|
FRAMING_TOKENS +
|
|
44
53
|
size.imageTokens +
|
|
45
54
|
reportTokens;
|
|
46
|
-
const expectedOutputTokens = Math.min(size.outputAllowance,
|
|
55
|
+
const expectedOutputTokens = Math.min(size.outputAllowance, outputScale *
|
|
56
|
+
(EXPECTED_OUTPUT_TOKENS + EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT * size.participants));
|
|
47
57
|
return {
|
|
48
58
|
inputTokens,
|
|
49
59
|
expectedOutputTokens,
|
|
@@ -58,14 +68,15 @@ function estimateRequest(rate, size, reportTokens = 0) {
|
|
|
58
68
|
*/
|
|
59
69
|
export function estimateAnalysisCost(rate, requests) {
|
|
60
70
|
const { cohorts } = requests;
|
|
61
|
-
const
|
|
71
|
+
const scale = requests.outputScale ?? 1;
|
|
72
|
+
const estimates = cohorts.map((size) => estimateRequest(rate, size, scale));
|
|
62
73
|
if (cohorts.length > 1)
|
|
63
74
|
estimates.push(estimateRequest(rate, {
|
|
64
75
|
textBytes: requests.mergeTextBytes,
|
|
65
76
|
imageTokens: 0,
|
|
66
77
|
participants: cohorts.reduce((total, size) => total + size.participants, 0),
|
|
67
78
|
outputAllowance: Math.max(...cohorts.map((size) => size.outputAllowance)),
|
|
68
|
-
}, cohorts.reduce((total, size) => total + size.outputAllowance, 0)));
|
|
79
|
+
}, scale, cohorts.reduce((total, size) => total + size.outputAllowance, 0)));
|
|
69
80
|
const sum = (key) => estimates.reduce((total, estimate) => total + estimate[key], 0);
|
|
70
81
|
// Each request's cost is a whole number of micro-dollars, so rounding the sum to the nearest one
|
|
71
82
|
// only removes the addition's float noise.
|
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
import type { StudyRoute } from "../study/plan.js";
|
|
2
|
+
import { type ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
3
|
import type { AnalysisConfig } from "./types.js";
|
|
3
4
|
export declare const DEFAULT_ANALYSIS_TIMEOUT_MS = 600000;
|
|
4
|
-
|
|
5
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
6
|
+
export declare const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
7
|
+
/**
|
|
8
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
9
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
12
|
+
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
13
|
+
* stable Flash model. */
|
|
14
|
+
export declare const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
5
15
|
interface StudyAnalysisSettings {
|
|
6
16
|
model?: string;
|
|
7
17
|
question?: string;
|
|
@@ -9,13 +19,15 @@ interface StudyAnalysisSettings {
|
|
|
9
19
|
}
|
|
10
20
|
/** The study's review.analysis: a provider selection for a separate review after a live run. */
|
|
11
21
|
export type StudyAnalysis = StudyAnalysisSettings & ({
|
|
12
|
-
provider?: "openai";
|
|
22
|
+
provider?: "openai" | "gemini";
|
|
13
23
|
maxCostUsd: number;
|
|
14
24
|
maxOutputTokens?: number;
|
|
25
|
+
reasoningEffort?: ReasoningEffort;
|
|
15
26
|
} | {
|
|
16
27
|
provider: "codex";
|
|
17
28
|
maxCostUsd?: null;
|
|
18
29
|
maxOutputTokens?: null;
|
|
30
|
+
reasoningEffort?: never;
|
|
19
31
|
});
|
|
20
32
|
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
21
33
|
export declare const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
@@ -37,9 +49,11 @@ export declare function resolveAutomaticAnalysis(raw: unknown, participant?: Ana
|
|
|
37
49
|
message: string;
|
|
38
50
|
};
|
|
39
51
|
export interface AutomaticAnalysisBudget {
|
|
40
|
-
provider?: "openai" | "codex";
|
|
52
|
+
provider?: "openai" | "gemini" | "codex";
|
|
41
53
|
billing?: "api-estimate" | "account-unknown";
|
|
42
54
|
model: string;
|
|
55
|
+
/** The reasoning effort the analysis runs at. */
|
|
56
|
+
effort: ReasoningEffort;
|
|
43
57
|
maxCostUsd: number | null;
|
|
44
58
|
trigger: "default" | "explicit";
|
|
45
59
|
/** The participants the expected cost range is for. */
|
|
@@ -1,12 +1,23 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange,
|
|
2
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, } from "./execute.js";
|
|
3
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
4
|
+
import { isReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
5
|
import { admissionRule } from "./admission.js";
|
|
4
6
|
import { analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
5
7
|
import { plural } from "../run/text.js";
|
|
6
8
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
7
9
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
8
10
|
const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
9
|
-
|
|
11
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
12
|
+
export const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
13
|
+
/**
|
|
14
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
15
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
18
|
+
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
19
|
+
* stable Flash model. */
|
|
20
|
+
export const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
10
21
|
const FIELDS = new Set([
|
|
11
22
|
"provider",
|
|
12
23
|
"maxCostUsd",
|
|
@@ -14,6 +25,7 @@ const FIELDS = new Set([
|
|
|
14
25
|
"question",
|
|
15
26
|
"timeoutMs",
|
|
16
27
|
"maxOutputTokens",
|
|
28
|
+
"reasoningEffort",
|
|
17
29
|
]);
|
|
18
30
|
const validTimeout = (value) => typeof value === "number" &&
|
|
19
31
|
Number.isSafeInteger(value) &&
|
|
@@ -56,14 +68,22 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
56
68
|
Object.keys(raw).some((key) => !FIELDS.has(key))) {
|
|
57
69
|
return {
|
|
58
70
|
ok: false,
|
|
59
|
-
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs and
|
|
71
|
+
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs, maxOutputTokens and reasoningEffort.",
|
|
60
72
|
};
|
|
61
73
|
}
|
|
62
74
|
const value = raw;
|
|
63
|
-
if (value.provider !== undefined &&
|
|
64
|
-
|
|
75
|
+
if (value.provider !== undefined &&
|
|
76
|
+
value.provider !== "openai" &&
|
|
77
|
+
value.provider !== "gemini" &&
|
|
78
|
+
value.provider !== "codex") {
|
|
79
|
+
return { ok: false, message: "review.analysis.provider must be openai, gemini or codex." };
|
|
65
80
|
}
|
|
66
81
|
if (value.provider === "codex") {
|
|
82
|
+
if (value.reasoningEffort !== undefined)
|
|
83
|
+
return {
|
|
84
|
+
ok: false,
|
|
85
|
+
message: "Codex account analysis runs at its fixed low reasoning effort, so it takes no reasoningEffort. Omit it, or choose provider openai or gemini to set one.",
|
|
86
|
+
};
|
|
67
87
|
const model = value.model === undefined ? CODEX_ANALYSIS_MODEL : value.model;
|
|
68
88
|
const timeoutMs = value.timeoutMs === undefined ? DEFAULT_ANALYSIS_TIMEOUT_MS : value.timeoutMs;
|
|
69
89
|
const question = value.question === undefined ? null : value.question;
|
|
@@ -93,7 +113,12 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
93
113
|
};
|
|
94
114
|
}
|
|
95
115
|
const { maxCostUsd } = value;
|
|
96
|
-
const
|
|
116
|
+
const vendor = value.provider === "gemini" ? "gemini" : "openai";
|
|
117
|
+
const model = value.model !== undefined
|
|
118
|
+
? value.model
|
|
119
|
+
: vendor === "gemini"
|
|
120
|
+
? DEFAULT_GEMINI_ANALYSIS_MODEL
|
|
121
|
+
: DEFAULT_ANALYSIS_MODEL;
|
|
97
122
|
const question = value.question === undefined ? null : value.question;
|
|
98
123
|
const timeoutMs = value.timeoutMs === undefined ? DEFAULT_ANALYSIS_TIMEOUT_MS : value.timeoutMs;
|
|
99
124
|
const maxOutputTokens = value.maxOutputTokens === undefined
|
|
@@ -104,7 +129,7 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
104
129
|
maxCostUsd <= 0 ||
|
|
105
130
|
maxCostUsd > 1000 ||
|
|
106
131
|
typeof model !== "string" ||
|
|
107
|
-
!isSupportedAnalysisModel(model) ||
|
|
132
|
+
!isSupportedAnalysisModel(model, vendor) ||
|
|
108
133
|
!validTimeout(timeoutMs) ||
|
|
109
134
|
typeof maxOutputTokens !== "number" ||
|
|
110
135
|
!Number.isSafeInteger(maxOutputTokens) ||
|
|
@@ -113,20 +138,35 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
113
138
|
!validQuestion(value.question)) {
|
|
114
139
|
return {
|
|
115
140
|
ok: false,
|
|
116
|
-
message: `review.analysis requires a positive maxCostUsd up to 1000, a supported analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
|
|
141
|
+
message: `review.analysis requires a positive maxCostUsd up to 1000, a supported ${vendor === "gemini" ? "Gemini" : "OpenAI"} analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
|
|
117
142
|
};
|
|
118
143
|
}
|
|
144
|
+
// Only the default model gets the default effort, written out because an omitted effort is high.
|
|
145
|
+
const reasoningEffort = value.reasoningEffort === undefined && value.model === undefined && vendor === "openai"
|
|
146
|
+
? DEFAULT_ANALYSIS_EFFORT
|
|
147
|
+
: value.reasoningEffort;
|
|
148
|
+
const efforts = analysisEfforts(model, vendor);
|
|
149
|
+
if (reasoningEffort !== undefined &&
|
|
150
|
+
(!isReasoningEffort(reasoningEffort) || !efforts.includes(reasoningEffort)))
|
|
151
|
+
return {
|
|
152
|
+
ok: false,
|
|
153
|
+
message: `review.analysis.reasoningEffort for ${model} must be ${efforts.slice(0, -1).join(", ")} or ${efforts.at(-1)}, the efforts its provider documents for it.`,
|
|
154
|
+
};
|
|
119
155
|
if (question !== null && scanEncodedText(question).sensitive)
|
|
120
156
|
return SENSITIVE_QUESTION;
|
|
121
157
|
return {
|
|
122
158
|
ok: true,
|
|
123
159
|
config: {
|
|
124
|
-
|
|
160
|
+
// An omitted provider, or an omitted effort on a named model, stays omitted, so the
|
|
161
|
+
// configuration keeps the hash it had before the field existed. An omitted provider is the
|
|
162
|
+
// OpenAI contract.
|
|
163
|
+
...(value.provider === undefined ? {} : { provider: vendor }),
|
|
125
164
|
model,
|
|
126
165
|
maxCostUsd,
|
|
127
166
|
timeoutMs,
|
|
128
167
|
maxOutputTokens,
|
|
129
168
|
question: question,
|
|
169
|
+
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
130
170
|
},
|
|
131
171
|
...(value.maxOutputTokens === undefined ? { preferLargerOutput: true } : {}),
|
|
132
172
|
};
|
|
@@ -148,8 +188,11 @@ export function automaticAnalysisBudget(raw, route, participants, participant) {
|
|
|
148
188
|
return {
|
|
149
189
|
...(config.provider === "codex"
|
|
150
190
|
? { provider: "codex", billing: "account-unknown" }
|
|
151
|
-
:
|
|
191
|
+
: config.provider === "gemini"
|
|
192
|
+
? { provider: "gemini" }
|
|
193
|
+
: {}),
|
|
152
194
|
model: config.model,
|
|
195
|
+
effort: analysisEffort(config),
|
|
153
196
|
maxCostUsd: config.maxCostUsd,
|
|
154
197
|
trigger: raw === undefined ? "default" : "explicit",
|
|
155
198
|
...(range === undefined || participants === undefined ? {} : { participants, ...range }),
|
|
@@ -166,7 +209,7 @@ export function analysisRequestsText(requests) {
|
|
|
166
209
|
}
|
|
167
210
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
168
211
|
if (budget.provider === "codex")
|
|
169
|
-
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
212
|
+
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} at ${budget.effort} effort · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
170
213
|
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
171
214
|
// The participants, and the requests that analyse them when there is more than one.
|
|
172
215
|
const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
|
|
@@ -179,5 +222,6 @@ export function formatAutomaticAnalysisBudget(budget) {
|
|
|
179
222
|
const refused = budget.refusedFromUsd === undefined || who === undefined
|
|
180
223
|
? `refused before it starts if ${rule}`
|
|
181
224
|
: `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
|
|
182
|
-
|
|
225
|
+
const where = budget.provider === "gemini" ? " on Gemini, which sends the evidence to Google" : "";
|
|
226
|
+
return `After live runs: ${budget.trigger} analysis${where} · ${budget.model} at ${budget.effort} effort${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
183
227
|
}
|
|
@@ -3,6 +3,7 @@ import { resolveRunPath } from "../run/locate.js";
|
|
|
3
3
|
import { refreshObserver, analyzeRun, readCompletedAnalysisSource, resolveAnalysisRun, } from "./service.js";
|
|
4
4
|
import { preferLargerAnalysisOutput, ANALYSIS_PROMPT_VERSION } from "./execute.js";
|
|
5
5
|
import { captureEvidence } from "./evidence.js";
|
|
6
|
+
import { analysisApiKey } from "../keys/key-resolution.js";
|
|
6
7
|
import { loadRunTraces } from "../run/stream-trace.js";
|
|
7
8
|
import { hashAnalysisValue } from "./validation.js";
|
|
8
9
|
import { refusedCost } from "./admission.js";
|
|
@@ -56,7 +57,11 @@ function statusReason(result) {
|
|
|
56
57
|
return "AUTOMATIC_ANALYSIS_CANCELLED";
|
|
57
58
|
return null;
|
|
58
59
|
}
|
|
59
|
-
|
|
60
|
+
/** The skip reason for a missing API key, which names the key the selected analyst reads. */
|
|
61
|
+
const keyMissingReason = (config) => config.provider === "gemini"
|
|
62
|
+
? "AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING"
|
|
63
|
+
: "AUTOMATIC_ANALYSIS_KEY_MISSING";
|
|
64
|
+
function outcomeOf(result, config) {
|
|
60
65
|
if (result.error?.code === "ANALYSIS_PUBLICATION_FAILED")
|
|
61
66
|
return { state: "failed", reason: "AUTOMATIC_ANALYSIS_PUBLICATION_FAILED", result };
|
|
62
67
|
if (result.error?.code?.startsWith("analysis_codex_"))
|
|
@@ -67,7 +72,7 @@ function outcomeOf(result) {
|
|
|
67
72
|
if (code === "ANALYSIS_CANCELLED")
|
|
68
73
|
return { state: "cancelled", reason: "AUTOMATIC_ANALYSIS_CANCELLED", result };
|
|
69
74
|
if (code === "ANALYSIS_API_KEY_MISSING")
|
|
70
|
-
return { ...skipped(
|
|
75
|
+
return { ...skipped(keyMissingReason(config)), result };
|
|
71
76
|
if (code === "ANALYSIS_BUSY")
|
|
72
77
|
return { ...skipped("AUTOMATIC_ANALYSIS_BUSY"), result };
|
|
73
78
|
if (code === "ANALYSIS_HISTORY_UNAVAILABLE")
|
|
@@ -158,7 +163,10 @@ export async function runAutomaticAnalysis(cwdInput, runId, configInput, deps =
|
|
|
158
163
|
if (!prepared)
|
|
159
164
|
return skipped("AUTOMATIC_ANALYSIS_SOURCE_UNAVAILABLE");
|
|
160
165
|
cwd = physicalCwdOf(prepared);
|
|
161
|
-
const hasKey = Boolean((deps.apiKey ??
|
|
166
|
+
const hasKey = Boolean((deps.apiKey ??
|
|
167
|
+
(configInput.provider === "codex"
|
|
168
|
+
? undefined
|
|
169
|
+
: analysisApiKey(configInput.provider ?? "openai", process.env)))?.trim());
|
|
162
170
|
const source = await readAutomaticSource(cwd, prepared, structuredClone(configInput), deps, hasKey);
|
|
163
171
|
if ("state" in source)
|
|
164
172
|
return source;
|
|
@@ -225,9 +233,7 @@ export async function runAutomaticAnalysis(cwdInput, runId, configInput, deps =
|
|
|
225
233
|
else if (deps.defaultRequest === true && (missingKey || !participantEvidence)) {
|
|
226
234
|
outcome = signal.aborted
|
|
227
235
|
? { state: "cancelled", reason: "AUTOMATIC_ANALYSIS_CANCELLED" }
|
|
228
|
-
: skipped(missingKey
|
|
229
|
-
? "AUTOMATIC_ANALYSIS_KEY_MISSING"
|
|
230
|
-
: "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE");
|
|
236
|
+
: skipped(missingKey ? keyMissingReason(config) : "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE");
|
|
231
237
|
}
|
|
232
238
|
else {
|
|
233
239
|
const result = await analyzeRun(cwd, runId, { config }, {
|
|
@@ -250,7 +256,7 @@ export async function runAutomaticAnalysis(cwdInput, runId, configInput, deps =
|
|
|
250
256
|
});
|
|
251
257
|
},
|
|
252
258
|
});
|
|
253
|
-
outcome = outcomeOf(result);
|
|
259
|
+
outcome = outcomeOf(result, config);
|
|
254
260
|
}
|
|
255
261
|
if (cancellationFailed)
|
|
256
262
|
outcome = {
|
|
@@ -2,8 +2,8 @@ import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "
|
|
|
2
2
|
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
4
|
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
6
|
+
* returns at most 32,768 output tokens from one generateContent request. */
|
|
7
7
|
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
|
8
8
|
export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
|
|
9
9
|
export interface AnalysisAdmission {
|
|
@@ -38,9 +38,10 @@ export interface AnalysisProgress {
|
|
|
38
38
|
* Local admission estimate, not a provider-enforced billed-spend guarantee. No token-count API
|
|
39
39
|
* call, credential, or network access. admission.ts holds the cost model: text at a calibrated
|
|
40
40
|
* bytes-per-token ratio, the expected output, and the worst case that spends the whole output
|
|
41
|
-
* allowance.
|
|
42
|
-
* formula
|
|
43
|
-
*
|
|
41
|
+
* allowance. image-tokens.ts prices each image: on OpenAI from its PNG size by the vision guide's
|
|
42
|
+
* patch formula, or at the 3,000-token ceiling when its size or the model's sizing is unknown; on
|
|
43
|
+
* Gemini at the fixed high-resolution count. The rates are those of the day admission runs.
|
|
44
|
+
* Unknown models/rates fail closed rather than inheriting those assumptions.
|
|
44
45
|
* Known-sensitive decoded text denies admission with analysis_input_sensitive or
|
|
45
46
|
* analysis_question_sensitive. Neither error includes rejected input values.
|
|
46
47
|
*/
|
package/dist/analysis/execute.js
CHANGED
|
@@ -1,29 +1,21 @@
|
|
|
1
1
|
import { validCodexAnalysisConfig } from "./codex-config.js";
|
|
2
|
-
import {
|
|
3
|
-
import { estimateAnalysisCost } from "./admission.js";
|
|
2
|
+
import { INPUT_IMAGE_DATA_URL, highDetailImageTokens, imageTokenCeiling } from "./image-tokens.js";
|
|
3
|
+
import { EXPECTED_OUTPUT_SCALE, estimateAnalysisCost } from "./admission.js";
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
|
-
import {
|
|
5
|
+
import { modelRate } from "../run/pricing.js";
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
7
7
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
8
8
|
import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
|
|
9
|
-
import {
|
|
9
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
10
|
+
import { createAnalysisProvider, } from "./provider.js";
|
|
10
11
|
import { hashAnalysisValue, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
|
|
11
12
|
import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
12
13
|
import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema, } from "./cohorts.js";
|
|
13
|
-
import { checkProviderAnalysis, codexWarnings, recordProviderUsage, } from "./responses.js";
|
|
14
|
+
import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
|
|
14
15
|
import { mapWithConcurrency } from "../run/concurrency.js";
|
|
15
16
|
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
"gpt-5.5",
|
|
19
|
-
"gpt-5.6",
|
|
20
|
-
"gpt-5.6-sol",
|
|
21
|
-
"gpt-5.6-terra",
|
|
22
|
-
"gpt-5.6-luna",
|
|
23
|
-
]);
|
|
24
|
-
const SUPPORTED_MODELS = new Set(SUPPORTED_ANALYSIS_MODELS);
|
|
25
|
-
export const isSupportedAnalysisModel = (model) => SUPPORTED_MODELS.has(model);
|
|
26
|
-
/** The OpenAI output limit when a study omits maxOutputTokens, and the most it may ask for. */
|
|
17
|
+
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
18
|
+
* returns at most 32,768 output tokens from one generateContent request. */
|
|
27
19
|
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
28
20
|
export const MAX_ANALYSIS_OUTPUT_TOKENS = 32_768;
|
|
29
21
|
const MAX_EVIDENCE_BYTES = 1024 * 1024;
|
|
@@ -154,9 +146,10 @@ function inputError(input) {
|
|
|
154
146
|
* Local admission estimate, not a provider-enforced billed-spend guarantee. No token-count API
|
|
155
147
|
* call, credential, or network access. admission.ts holds the cost model: text at a calibrated
|
|
156
148
|
* bytes-per-token ratio, the expected output, and the worst case that spends the whole output
|
|
157
|
-
* allowance.
|
|
158
|
-
* formula
|
|
159
|
-
*
|
|
149
|
+
* allowance. image-tokens.ts prices each image: on OpenAI from its PNG size by the vision guide's
|
|
150
|
+
* patch formula, or at the 3,000-token ceiling when its size or the model's sizing is unknown; on
|
|
151
|
+
* Gemini at the fixed high-resolution count. The rates are those of the day admission runs.
|
|
152
|
+
* Unknown models/rates fail closed rather than inheriting those assumptions.
|
|
160
153
|
* Known-sensitive decoded text denies admission with analysis_input_sensitive or
|
|
161
154
|
* analysis_question_sensitive. Neither error includes rejected input values.
|
|
162
155
|
*/
|
|
@@ -175,8 +168,12 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
175
168
|
});
|
|
176
169
|
if ((config.provider === "codex"
|
|
177
170
|
? !validCodexAnalysisConfig(config)
|
|
178
|
-
: (config.provider !== undefined &&
|
|
179
|
-
|
|
171
|
+
: (config.provider !== undefined &&
|
|
172
|
+
config.provider !== "openai" &&
|
|
173
|
+
config.provider !== "gemini") ||
|
|
174
|
+
!isSupportedAnalysisModel(config.model, config.provider) ||
|
|
175
|
+
(config.reasoningEffort !== undefined &&
|
|
176
|
+
!analysisEfforts(config.model, config.provider).includes(config.reasoningEffort)) ||
|
|
180
177
|
!Number.isFinite(config.maxCostUsd) ||
|
|
181
178
|
config.maxCostUsd <= 0 ||
|
|
182
179
|
config.maxCostUsd > 1000 ||
|
|
@@ -209,7 +206,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
209
206
|
ratesAsOf: null,
|
|
210
207
|
requests,
|
|
211
208
|
};
|
|
212
|
-
const rate =
|
|
209
|
+
const rate = modelRate(config.model);
|
|
213
210
|
if (!rate ||
|
|
214
211
|
rate.placeholder ||
|
|
215
212
|
!Number.isFinite(rate.inputUsdPerToken) ||
|
|
@@ -230,6 +227,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
230
227
|
}),
|
|
231
228
|
// Without reports: estimateAnalysisCost prices them at the cohorts' output allowances.
|
|
232
229
|
mergeTextBytes: requestBytes(mergeRequest(input, config, [])),
|
|
230
|
+
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
233
231
|
});
|
|
234
232
|
if (!Number.isFinite(cost.worstCaseCostUsd))
|
|
235
233
|
return denied("analysis_rate_unknown");
|
|
@@ -284,7 +282,7 @@ const RANGE_BISECTIONS = 40;
|
|
|
284
282
|
export function analysisCostRange(config, participants, preferLargerOutput = false) {
|
|
285
283
|
if (config.provider === "codex")
|
|
286
284
|
return undefined;
|
|
287
|
-
const rate =
|
|
285
|
+
const rate = modelRate(config.model);
|
|
288
286
|
if (!rate || rate.placeholder)
|
|
289
287
|
return undefined;
|
|
290
288
|
const covered = Math.min(participants, EVIDENCE_LIMITS.participants);
|
|
@@ -306,9 +304,10 @@ export function analysisCostRange(config, participants, preferLargerOutput = fal
|
|
|
306
304
|
outputAllowance,
|
|
307
305
|
textBytes: cohortPrompt +
|
|
308
306
|
share * (EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES),
|
|
309
|
-
imageTokens: share * EVIDENCE_LIMITS.captures *
|
|
307
|
+
imageTokens: share * EVIDENCE_LIMITS.captures * imageTokenCeiling(config.model),
|
|
310
308
|
})),
|
|
311
309
|
mergeTextBytes: mergePrompt + share * covered * ENTRY_STRUCTURE_BYTES,
|
|
310
|
+
outputScale: EXPECTED_OUTPUT_SCALE[config.provider ?? "openai"],
|
|
312
311
|
});
|
|
313
312
|
const admits = (share, allowance) => request(share, allowance).admittedCostUsd <= config.maxCostUsd;
|
|
314
313
|
// Cost grows with the share, so admission admits each allowance up to one share of the limits.
|
|
@@ -415,6 +414,7 @@ async function analysisProvider(config, options) {
|
|
|
415
414
|
cliVersion: config.identity.cliVersion,
|
|
416
415
|
}))
|
|
417
416
|
: createAnalysisProvider({
|
|
417
|
+
vendor: config.provider ?? "openai",
|
|
418
418
|
apiKey: options.apiKey,
|
|
419
419
|
...(options.fetch === undefined ? {} : { fetchFn: options.fetch }),
|
|
420
420
|
});
|
|
@@ -529,13 +529,14 @@ export async function runAnalysis(input, config, options) {
|
|
|
529
529
|
images: request.images,
|
|
530
530
|
schema: request.schema,
|
|
531
531
|
maxOutputTokens: config.maxOutputTokens,
|
|
532
|
+
reasoningEffort: analysisEffort(config),
|
|
532
533
|
timeoutMs: config.timeoutMs,
|
|
533
534
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
534
535
|
});
|
|
535
536
|
responses.push(response);
|
|
536
537
|
recordProviderUsage(artifact, config, responses);
|
|
537
|
-
for (const warning of codexWarnings(config, response))
|
|
538
|
-
if (!options.warnings?.includes(warning))
|
|
538
|
+
for (const warning of [...codexWarnings(config, response), httpStatusWarning(config, response)])
|
|
539
|
+
if (warning !== undefined && !options.warnings?.includes(warning))
|
|
539
540
|
options.warnings?.push(warning);
|
|
540
541
|
if (response.status !== "completed")
|
|
541
542
|
return {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { AnalysisWire } from "./provider.js";
|
|
2
|
+
/**
|
|
3
|
+
* One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
|
|
4
|
+
* ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
|
|
5
|
+
* service tier, and `store: false` so the request stays out of the project's request logs. Each
|
|
6
|
+
* capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
|
|
7
|
+
* image analysis, after a text part naming its evidence key.
|
|
8
|
+
*/
|
|
9
|
+
export declare const geminiWire: AnalysisWire;
|