humanish 0.142.0 → 0.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +1 -1
- package/dist/analysis/automatic-config.d.ts +12 -1
- package/dist/analysis/automatic-config.js +35 -6
- package/dist/analysis/execute.d.ts +1 -11
- package/dist/analysis/execute.js +4 -21
- package/dist/analysis/gemini-wire.d.ts +2 -1
- package/dist/analysis/gemini-wire.js +3 -2
- package/dist/analysis/models.d.ts +12 -0
- package/dist/analysis/models.js +47 -0
- package/dist/analysis/openai-wire.d.ts +2 -1
- package/dist/analysis/openai-wire.js +3 -2
- package/dist/analysis/provider.d.ts +4 -0
- package/dist/analysis/types.d.ts +4 -0
- package/dist/analysis/validation.js +4 -3
- package/dist/cli/commands/analyze.js +5 -3
- package/dist/cli/findings.d.ts +2 -1
- package/dist/cli/findings.js +31 -9
- package/dist/remote/launch.d.ts +4 -1
- package/dist/remote/launch.js +53 -8
- package/dist/remote/runner.d.ts +16 -0
- package/dist/remote/runner.js +29 -4
- package/docs/contracts/study-analysis.md +35 -2
- package/docs/product/automatic-analysis.md +17 -7
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +8 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,16 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.143.0: Analysis costs about one seventh as much: gpt-6.1-sol at medium effort is the default (2026-10-22)
|
|
12
|
+
|
|
13
|
+
humanish 0.143.0 analyzes studies with `gpt-6.1-sol` at medium reasoning effort by default, in
|
|
14
|
+
place of `gpt-6-astra` at high. On the same recordings it found what `gpt-6-astra` found, invented
|
|
15
|
+
nothing, and cost about one seventh as much: $0.06 against $0.41 per analysis on the Taskly
|
|
16
|
+
benchmark. A study can set the analysis effort with `review.analysis.reasoningEffort`, and one
|
|
17
|
+
that names its model keeps it at high effort with an unchanged configuration hash.
|
|
18
|
+
|
|
19
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.143.0)
|
|
20
|
+
|
|
11
21
|
## 0.142.0: Remote runs, Gemini and GPT-6 analysis models, and a view-only live stream (2026-10-21)
|
|
12
22
|
|
|
13
23
|
humanish 0.142.0 can run a study remotely: `humanish run <study> --remote` puts the study's own
|
package/README.md
CHANGED
|
@@ -249,7 +249,7 @@ coverage limits, estimated cost controls, corrections and sharing behavior.
|
|
|
249
249
|
|
|
250
250
|
Supported live studies generate findings automatically after participants finish.
|
|
251
251
|
A Codex participant's run is analyzed by the same Codex account; every other run uses
|
|
252
|
-
`gpt-6-
|
|
252
|
+
`gpt-6.1-sol` at medium reasoning effort and a separate $10 cap on the expected cost.
|
|
253
253
|
This is additional to participant execution costs, and is not a hard provider
|
|
254
254
|
billing cap. To disable the extra request:
|
|
255
255
|
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import type { StudyRoute } from "../study/plan.js";
|
|
2
|
+
import { type ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
3
|
import type { AnalysisConfig } from "./types.js";
|
|
3
4
|
export declare const DEFAULT_ANALYSIS_TIMEOUT_MS = 600000;
|
|
4
|
-
|
|
5
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
6
|
+
export declare const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
7
|
+
/**
|
|
8
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
9
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
5
12
|
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
6
13
|
* stable Flash model. */
|
|
7
14
|
export declare const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
@@ -15,10 +22,12 @@ export type StudyAnalysis = StudyAnalysisSettings & ({
|
|
|
15
22
|
provider?: "openai" | "gemini";
|
|
16
23
|
maxCostUsd: number;
|
|
17
24
|
maxOutputTokens?: number;
|
|
25
|
+
reasoningEffort?: ReasoningEffort;
|
|
18
26
|
} | {
|
|
19
27
|
provider: "codex";
|
|
20
28
|
maxCostUsd?: null;
|
|
21
29
|
maxOutputTokens?: null;
|
|
30
|
+
reasoningEffort?: never;
|
|
22
31
|
});
|
|
23
32
|
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
24
33
|
export declare const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
@@ -43,6 +52,8 @@ export interface AutomaticAnalysisBudget {
|
|
|
43
52
|
provider?: "openai" | "gemini" | "codex";
|
|
44
53
|
billing?: "api-estimate" | "account-unknown";
|
|
45
54
|
model: string;
|
|
55
|
+
/** The reasoning effort the analysis runs at. */
|
|
56
|
+
effort: ReasoningEffort;
|
|
46
57
|
maxCostUsd: number | null;
|
|
47
58
|
trigger: "default" | "explicit";
|
|
48
59
|
/** The participants the expected cost range is for. */
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange,
|
|
2
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, } from "./execute.js";
|
|
3
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
4
|
+
import { isReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
5
|
import { admissionRule } from "./admission.js";
|
|
4
6
|
import { analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
5
7
|
import { plural } from "../run/text.js";
|
|
6
8
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
7
9
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
8
10
|
const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
9
|
-
|
|
11
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
12
|
+
export const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
13
|
+
/**
|
|
14
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
15
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
10
18
|
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
11
19
|
* stable Flash model. */
|
|
12
20
|
export const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
@@ -17,6 +25,7 @@ const FIELDS = new Set([
|
|
|
17
25
|
"question",
|
|
18
26
|
"timeoutMs",
|
|
19
27
|
"maxOutputTokens",
|
|
28
|
+
"reasoningEffort",
|
|
20
29
|
]);
|
|
21
30
|
const validTimeout = (value) => typeof value === "number" &&
|
|
22
31
|
Number.isSafeInteger(value) &&
|
|
@@ -59,7 +68,7 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
59
68
|
Object.keys(raw).some((key) => !FIELDS.has(key))) {
|
|
60
69
|
return {
|
|
61
70
|
ok: false,
|
|
62
|
-
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs and
|
|
71
|
+
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs, maxOutputTokens and reasoningEffort.",
|
|
63
72
|
};
|
|
64
73
|
}
|
|
65
74
|
const value = raw;
|
|
@@ -70,6 +79,11 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
70
79
|
return { ok: false, message: "review.analysis.provider must be openai, gemini or codex." };
|
|
71
80
|
}
|
|
72
81
|
if (value.provider === "codex") {
|
|
82
|
+
if (value.reasoningEffort !== undefined)
|
|
83
|
+
return {
|
|
84
|
+
ok: false,
|
|
85
|
+
message: "Codex account analysis runs at its fixed low reasoning effort, so it takes no reasoningEffort. Omit it, or choose provider openai or gemini to set one.",
|
|
86
|
+
};
|
|
73
87
|
const model = value.model === undefined ? CODEX_ANALYSIS_MODEL : value.model;
|
|
74
88
|
const timeoutMs = value.timeoutMs === undefined ? DEFAULT_ANALYSIS_TIMEOUT_MS : value.timeoutMs;
|
|
75
89
|
const question = value.question === undefined ? null : value.question;
|
|
@@ -127,18 +141,32 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
127
141
|
message: `review.analysis requires a positive maxCostUsd up to 1000, a supported ${vendor === "gemini" ? "Gemini" : "OpenAI"} analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
|
|
128
142
|
};
|
|
129
143
|
}
|
|
144
|
+
// Only the default model gets the default effort, written out because an omitted effort is high.
|
|
145
|
+
const reasoningEffort = value.reasoningEffort === undefined && value.model === undefined && vendor === "openai"
|
|
146
|
+
? DEFAULT_ANALYSIS_EFFORT
|
|
147
|
+
: value.reasoningEffort;
|
|
148
|
+
const efforts = analysisEfforts(model, vendor);
|
|
149
|
+
if (reasoningEffort !== undefined &&
|
|
150
|
+
(!isReasoningEffort(reasoningEffort) || !efforts.includes(reasoningEffort)))
|
|
151
|
+
return {
|
|
152
|
+
ok: false,
|
|
153
|
+
message: `review.analysis.reasoningEffort for ${model} must be ${efforts.slice(0, -1).join(", ")} or ${efforts.at(-1)}, the efforts its provider documents for it.`,
|
|
154
|
+
};
|
|
130
155
|
if (question !== null && scanEncodedText(question).sensitive)
|
|
131
156
|
return SENSITIVE_QUESTION;
|
|
132
157
|
return {
|
|
133
158
|
ok: true,
|
|
134
159
|
config: {
|
|
135
|
-
// An omitted provider
|
|
160
|
+
// An omitted provider, or an omitted effort on a named model, stays omitted, so the
|
|
161
|
+
// configuration keeps the hash it had before the field existed. An omitted provider is the
|
|
162
|
+
// OpenAI contract.
|
|
136
163
|
...(value.provider === undefined ? {} : { provider: vendor }),
|
|
137
164
|
model,
|
|
138
165
|
maxCostUsd,
|
|
139
166
|
timeoutMs,
|
|
140
167
|
maxOutputTokens,
|
|
141
168
|
question: question,
|
|
169
|
+
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
142
170
|
},
|
|
143
171
|
...(value.maxOutputTokens === undefined ? { preferLargerOutput: true } : {}),
|
|
144
172
|
};
|
|
@@ -164,6 +192,7 @@ export function automaticAnalysisBudget(raw, route, participants, participant) {
|
|
|
164
192
|
? { provider: "gemini" }
|
|
165
193
|
: {}),
|
|
166
194
|
model: config.model,
|
|
195
|
+
effort: analysisEffort(config),
|
|
167
196
|
maxCostUsd: config.maxCostUsd,
|
|
168
197
|
trigger: raw === undefined ? "default" : "explicit",
|
|
169
198
|
...(range === undefined || participants === undefined ? {} : { participants, ...range }),
|
|
@@ -180,7 +209,7 @@ export function analysisRequestsText(requests) {
|
|
|
180
209
|
}
|
|
181
210
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
182
211
|
if (budget.provider === "codex")
|
|
183
|
-
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
212
|
+
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} at ${budget.effort} effort · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
184
213
|
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
185
214
|
// The participants, and the requests that analyse them when there is more than one.
|
|
186
215
|
const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
|
|
@@ -194,5 +223,5 @@ export function formatAutomaticAnalysisBudget(budget) {
|
|
|
194
223
|
? `refused before it starts if ${rule}`
|
|
195
224
|
: `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
|
|
196
225
|
const where = budget.provider === "gemini" ? " on Gemini, which sends the evidence to Google" : "";
|
|
197
|
-
return `After live runs: ${budget.trigger} analysis${where} · ${budget.model}${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
226
|
+
return `After live runs: ${budget.trigger} analysis${where} · ${budget.model} at ${budget.effort} effort${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
198
227
|
}
|
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
|
|
2
|
-
import { type
|
|
2
|
+
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
4
|
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
|
-
export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
|
|
6
|
-
/**
|
|
7
|
-
* Gemini models that completed a live analysis. Each model's page under
|
|
8
|
-
* ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
|
|
9
|
-
* output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
|
|
10
|
-
* once a live analysis on each completes.
|
|
11
|
-
*/
|
|
12
|
-
export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
|
|
13
|
-
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
14
|
-
export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
|
|
15
5
|
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
16
6
|
* returns at most 32,768 output tokens from one generateContent request. */
|
|
17
7
|
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -6,6 +6,7 @@ import { modelRate } from "../run/pricing.js";
|
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
7
7
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
8
8
|
import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
|
|
9
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
9
10
|
import { createAnalysisProvider, } from "./provider.js";
|
|
10
11
|
import { hashAnalysisValue, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
|
|
11
12
|
import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
@@ -13,27 +14,6 @@ import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema,
|
|
|
13
14
|
import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
|
|
14
15
|
import { mapWithConcurrency } from "../run/concurrency.js";
|
|
15
16
|
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
16
|
-
export const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
|
|
17
|
-
"gpt-6-astra",
|
|
18
|
-
"gpt-6.1-sol",
|
|
19
|
-
"gpt-6-luna",
|
|
20
|
-
"gpt-5.5",
|
|
21
|
-
"gpt-5.6",
|
|
22
|
-
"gpt-5.6-sol",
|
|
23
|
-
"gpt-5.6-terra",
|
|
24
|
-
"gpt-5.6-luna",
|
|
25
|
-
]);
|
|
26
|
-
const SUPPORTED_MODELS = new Set(SUPPORTED_ANALYSIS_MODELS);
|
|
27
|
-
/**
|
|
28
|
-
* Gemini models that completed a live analysis. Each model's page under
|
|
29
|
-
* ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
|
|
30
|
-
* output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
|
|
31
|
-
* once a live analysis on each completes.
|
|
32
|
-
*/
|
|
33
|
-
export const GEMINI_ANALYSIS_MODELS = Object.freeze(["gemini-3.8-flash"]);
|
|
34
|
-
const GEMINI_MODELS = new Set(GEMINI_ANALYSIS_MODELS);
|
|
35
|
-
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
36
|
-
export const isSupportedAnalysisModel = (model, provider = "openai") => (provider === "gemini" ? GEMINI_MODELS : SUPPORTED_MODELS).has(model);
|
|
37
17
|
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
38
18
|
* returns at most 32,768 output tokens from one generateContent request. */
|
|
39
19
|
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
@@ -192,6 +172,8 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
192
172
|
config.provider !== "openai" &&
|
|
193
173
|
config.provider !== "gemini") ||
|
|
194
174
|
!isSupportedAnalysisModel(config.model, config.provider) ||
|
|
175
|
+
(config.reasoningEffort !== undefined &&
|
|
176
|
+
!analysisEfforts(config.model, config.provider).includes(config.reasoningEffort)) ||
|
|
195
177
|
!Number.isFinite(config.maxCostUsd) ||
|
|
196
178
|
config.maxCostUsd <= 0 ||
|
|
197
179
|
config.maxCostUsd > 1000 ||
|
|
@@ -547,6 +529,7 @@ export async function runAnalysis(input, config, options) {
|
|
|
547
529
|
images: request.images,
|
|
548
530
|
schema: request.schema,
|
|
549
531
|
maxOutputTokens: config.maxOutputTokens,
|
|
532
|
+
reasoningEffort: analysisEffort(config),
|
|
550
533
|
timeoutMs: config.timeoutMs,
|
|
551
534
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
552
535
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AnalysisWire } from "./provider.js";
|
|
2
2
|
/**
|
|
3
|
-
* One stateless request: no tools,
|
|
3
|
+
* One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
|
|
4
|
+
* ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
|
|
4
5
|
* service tier, and `store: false` so the request stays out of the project's request logs. Each
|
|
5
6
|
* capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
|
|
6
7
|
* image analysis, after a text part naming its evidence key.
|
|
@@ -116,7 +116,8 @@ function parseGenerateContent(raw) {
|
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
/**
|
|
119
|
-
* One stateless request: no tools,
|
|
119
|
+
* One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
|
|
120
|
+
* ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
|
|
120
121
|
* service tier, and `store: false` so the request stays out of the project's request logs. Each
|
|
121
122
|
* capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
|
|
122
123
|
* image analysis, after a text part naming its evidence key.
|
|
@@ -152,7 +153,7 @@ export const geminiWire = {
|
|
|
152
153
|
generationConfig: {
|
|
153
154
|
candidateCount: 1,
|
|
154
155
|
maxOutputTokens: request.maxOutputTokens,
|
|
155
|
-
thinkingConfig: { thinkingLevel:
|
|
156
|
+
thinkingConfig: { thinkingLevel: request.reasoningEffort.toUpperCase() },
|
|
156
157
|
responseFormat: {
|
|
157
158
|
text: { mimeType: "APPLICATION_JSON", schema },
|
|
158
159
|
},
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
|
+
import type { AnalysisApiVendor } from "./provider.js";
|
|
3
|
+
import type { AnalysisConfig } from "./types.js";
|
|
4
|
+
export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
|
|
5
|
+
export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
|
|
6
|
+
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
7
|
+
export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
|
|
8
|
+
/** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
|
|
9
|
+
export declare const analysisEfforts: (model: string, provider?: AnalysisApiVendor) => readonly ReasoningEffort[];
|
|
10
|
+
/** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
|
|
11
|
+
* and the Codex account's fixed effort. */
|
|
12
|
+
export declare function analysisEffort(config: AnalysisConfig): ReasoningEffort;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// The models each API analyst runs, and the reasoning efforts each model accepts. Config parsing,
|
|
2
|
+
// admission, the CLI help and the wire formats all read this one table.
|
|
3
|
+
/**
|
|
4
|
+
* `reasoning.effort` values from each model's page under developers.openai.com/api/docs/models
|
|
5
|
+
* (read 2026-10-21). None of them lists `minimal`; gpt-6-astra and gpt-6.1-sol also refuse `none`.
|
|
6
|
+
*/
|
|
7
|
+
const OPENAI_MODELS = {
|
|
8
|
+
"gpt-6-astra": ["low", "medium", "high", "xhigh", "max"],
|
|
9
|
+
"gpt-6.1-sol": ["low", "medium", "high", "xhigh", "max"],
|
|
10
|
+
"gpt-6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
11
|
+
"gpt-5.5": ["none", "low", "medium", "high", "xhigh"],
|
|
12
|
+
"gpt-5.6": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
13
|
+
"gpt-5.6-sol": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
14
|
+
"gpt-5.6-terra": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
15
|
+
"gpt-5.6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Gemini models that completed a live analysis, with the `thinkingLevel` values the thinking guide
|
|
19
|
+
* lists for each (ai.google.dev/gemini-api/docs/generate-content/thinking, read 2026-10-21):
|
|
20
|
+
* gemini-3.8-flash answers `minimal` with an error. Each model's page under
|
|
21
|
+
* ai.google.dev/gemini-api/docs/models also lists image input and JSON-schema output for
|
|
22
|
+
* gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this table once a
|
|
23
|
+
* live analysis on each completes.
|
|
24
|
+
*/
|
|
25
|
+
const GEMINI_MODELS = {
|
|
26
|
+
"gemini-3.8-flash": ["low", "medium", "high"],
|
|
27
|
+
};
|
|
28
|
+
const MODELS = { openai: OPENAI_MODELS, gemini: GEMINI_MODELS };
|
|
29
|
+
export const SUPPORTED_ANALYSIS_MODELS = Object.freeze(Object.keys(OPENAI_MODELS));
|
|
30
|
+
export const GEMINI_ANALYSIS_MODELS = Object.freeze(Object.keys(GEMINI_MODELS));
|
|
31
|
+
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
32
|
+
export const isSupportedAnalysisModel = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model);
|
|
33
|
+
/** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
|
|
34
|
+
export const analysisEfforts = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model) ? MODELS[provider][model] : [];
|
|
35
|
+
/**
|
|
36
|
+
* The effort an API analysis runs at when its configuration declares none. Every saved analysis
|
|
37
|
+
* without `reasoningEffort` ran at this effort, so it stays high: a different default must be
|
|
38
|
+
* written into the configuration it resolves, where it also changes the configuration's hash.
|
|
39
|
+
*/
|
|
40
|
+
const UNDECLARED_EFFORT = "high";
|
|
41
|
+
/** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
|
|
42
|
+
* and the Codex account's fixed effort. */
|
|
43
|
+
export function analysisEffort(config) {
|
|
44
|
+
if (config.provider === "codex")
|
|
45
|
+
return config.identity.reasoningEffort;
|
|
46
|
+
return config.reasoningEffort ?? UNDECLARED_EFFORT;
|
|
47
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AnalysisProviderResult, AnalysisWire } from "./provider.js";
|
|
2
2
|
/** Text and usage wire shapes reuse the captured closing-report contract; refusal fails closed. */
|
|
3
3
|
export declare function parseAnalysisResponse(raw: unknown): AnalysisProviderResult;
|
|
4
|
-
/** One stateless request: no stored response, no tools,
|
|
4
|
+
/** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
|
|
5
|
+
* schema. */
|
|
5
6
|
export declare const openaiWire: AnalysisWire;
|
|
@@ -67,7 +67,8 @@ export function parseAnalysisResponse(raw) {
|
|
|
67
67
|
return { ...base, status: "failed", errorCode: "invalid_response" };
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
|
-
/** One stateless request: no stored response, no tools,
|
|
70
|
+
/** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
|
|
71
|
+
* schema. */
|
|
71
72
|
export const openaiWire = {
|
|
72
73
|
url: () => OPENAI_RESPONSES_URL,
|
|
73
74
|
headers: (apiKey) => ({ Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }),
|
|
@@ -91,7 +92,7 @@ export const openaiWire = {
|
|
|
91
92
|
tool_choice: "none",
|
|
92
93
|
truncation: "disabled",
|
|
93
94
|
service_tier: "default",
|
|
94
|
-
reasoning: { effort:
|
|
95
|
+
reasoning: { effort: request.reasoningEffort },
|
|
95
96
|
max_output_tokens: request.maxOutputTokens,
|
|
96
97
|
text: {
|
|
97
98
|
format: { type: "json_schema", name: "study_analysis", strict: true, schema: request.schema },
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Dispatcher } from "undici";
|
|
2
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
3
|
/** Deliberately separate from the stateful computer-use actor: one request, no tools or retries. */
|
|
3
4
|
export interface AnalysisProviderRequest {
|
|
4
5
|
model: string;
|
|
@@ -10,6 +11,9 @@ export interface AnalysisProviderRequest {
|
|
|
10
11
|
}[];
|
|
11
12
|
schema: Record<string, unknown>;
|
|
12
13
|
maxOutputTokens: number | null;
|
|
14
|
+
/** The effort the analysis runs at (models.ts analysisEffort). The Codex account analyst runs
|
|
15
|
+
* its fixed low effort, which this then names. */
|
|
16
|
+
reasoningEffort: ReasoningEffort;
|
|
13
17
|
timeoutMs: number;
|
|
14
18
|
signal?: AbortSignal;
|
|
15
19
|
}
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
|
|
2
2
|
import type { CuaEndpointSource } from "../actors/goal-source.js";
|
|
3
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
4
|
import type { AutomaticAnalysisView } from "./job.js";
|
|
4
5
|
import type { RunAnalysisCost } from "../run/run-cost.js";
|
|
5
6
|
/** Independent interpretation of retained evidence; never a participant or harness verdict. */
|
|
@@ -160,6 +161,9 @@ interface OpenAIAnalysisConfig {
|
|
|
160
161
|
maxCostUsd: number;
|
|
161
162
|
timeoutMs: number;
|
|
162
163
|
maxOutputTokens: number;
|
|
164
|
+
/** Present only when the study or command declared it, so an omitted effort keeps the hash of
|
|
165
|
+
* every configuration before the field. An analysis without it ran at high (models.ts). */
|
|
166
|
+
reasoningEffort?: ReasoningEffort | undefined;
|
|
163
167
|
}
|
|
164
168
|
/** The Gemini API analyst: the same billed, capped request as OpenAI's, sent to Google. */
|
|
165
169
|
interface GeminiAnalysisConfig extends Omit<OpenAIAnalysisConfig, "provider"> {
|
|
@@ -3,6 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
5
|
import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
|
|
6
|
+
import { REASONING_EFFORTS } from "../actors/reasoning-effort.js";
|
|
6
7
|
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
7
8
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
8
9
|
const text = (max) => z
|
|
@@ -204,6 +205,7 @@ const apiAnalysisConfigSchema = z.object({
|
|
|
204
205
|
maxCostUsd: z.number().positive().max(1000),
|
|
205
206
|
timeoutMs: z.number().int().positive().max(3600000),
|
|
206
207
|
maxOutputTokens: z.number().int().positive().max(128000),
|
|
208
|
+
reasoningEffort: z.enum(REASONING_EFFORTS).optional(),
|
|
207
209
|
});
|
|
208
210
|
const analysisArtifactSchema = z
|
|
209
211
|
.object({
|
|
@@ -361,8 +363,7 @@ export function checkAnalysisResult(input, value) {
|
|
|
361
363
|
if (!distinct(observation.evidenceIds) ||
|
|
362
364
|
refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
363
365
|
errors.add("ANALYSIS_OBSERVATION_REFERENCE_INVALID");
|
|
364
|
-
if (observation.basis === "visual" &&
|
|
365
|
-
!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
|
|
366
|
+
if (observation.basis === "visual" && !refs.some((ref) => ref?.capture))
|
|
366
367
|
errors.add("ANALYSIS_VISUAL_WITHOUT_CAPTURE");
|
|
367
368
|
if (observation.basis === "action" &&
|
|
368
369
|
!refs.some((ref) => ref &&
|
|
@@ -394,7 +395,7 @@ export function checkAnalysisResult(input, value) {
|
|
|
394
395
|
const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
|
|
395
396
|
if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
396
397
|
errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
|
|
397
|
-
if (!refs.some((ref) => ref?.capture
|
|
398
|
+
if (!refs.some((ref) => ref?.capture))
|
|
398
399
|
errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
|
|
399
400
|
// Seen by means a cited capture of that participant shows the problem.
|
|
400
401
|
const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
|
|
1
|
+
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
5
5
|
import { admissionRule } from "../../analysis/admission.js";
|
|
6
|
-
import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS
|
|
6
|
+
import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS } from "../../analysis/models.js";
|
|
7
7
|
import { CODEX_ANALYSIS_MODEL } from "../../analysis/codex-config.js";
|
|
8
8
|
import { listAnalyses } from "../../analysis/store.js";
|
|
9
9
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
@@ -34,7 +34,8 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
34
34
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
35
35
|
.option("--provider <name>", "Analyst: openai (default), gemini (sends the evidence and captures to Google), or restricted codex account. No provider fallback.")
|
|
36
36
|
.option("--max-cost <usd>", `Required for OpenAI and Gemini, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
|
|
37
|
-
.option("--model <id>", `OpenAI analysis model
|
|
37
|
+
.option("--model <id>", `OpenAI analysis model: ${SUPPORTED_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_ANALYSIS_MODEL} at ${DEFAULT_ANALYSIS_EFFORT} effort when omitted. Gemini analysis model: ${GEMINI_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_GEMINI_ANALYSIS_MODEL} when omitted. Codex account analysis runs ${CODEX_ANALYSIS_MODEL} at low effort.`)
|
|
38
|
+
.option("--reasoning-effort <level>", `OpenAI reasoning.effort or Gemini thinking level, from none, minimal, low, medium, high, xhigh and max; each model takes only the levels its provider documents for it, and others are refused before any request. When omitted: ${DEFAULT_ANALYSIS_EFFORT} for the default OpenAI analyst, high with --model or --provider gemini. Unsupported for Codex.`)
|
|
38
39
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
39
40
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
40
41
|
.option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for admission-based sizing. Unsupported for Codex.")
|
|
@@ -84,6 +85,7 @@ async function handleAnalyze(io, options, command) {
|
|
|
84
85
|
...(options.maxOutputTokens === undefined
|
|
85
86
|
? {}
|
|
86
87
|
: { maxOutputTokens: Number(options.maxOutputTokens) }),
|
|
88
|
+
...(options.reasoningEffort === undefined ? {} : { reasoningEffort: options.reasoningEffort }),
|
|
87
89
|
});
|
|
88
90
|
if (!selected.ok || !selected.config) {
|
|
89
91
|
const refusal = await dryRunBundleRefusal(await resolvePhysicalCwd(options.cwd), options.run, options.dryRun === true);
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -158,10 +158,11 @@ interface LiveStudy {
|
|
|
158
158
|
}
|
|
159
159
|
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
160
160
|
export declare function cwdFlag(cwd: string): string;
|
|
161
|
-
/** An analysis provider and
|
|
161
|
+
/** An analysis provider, model and effort, as a study or an analysis record names them. */
|
|
162
162
|
interface Analyst {
|
|
163
163
|
provider?: string | undefined;
|
|
164
164
|
model?: string;
|
|
165
|
+
reasoningEffort?: string | undefined;
|
|
165
166
|
}
|
|
166
167
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
167
168
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
package/dist/cli/findings.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// starts an analysis.
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { costRefusal } from "../analysis/admission.js";
|
|
8
|
-
import { DEFAULT_ANALYSIS_MAX_COST_USD, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
|
|
8
|
+
import { DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MAX_COST_USD, DEFAULT_ANALYSIS_MODEL, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
|
|
9
9
|
import { ANALYSIS_USUALLY_TAKES } from "../analysis/analysis-limits.js";
|
|
10
10
|
import { bySeverity } from "../analysis/finding-lead.js";
|
|
11
11
|
import { loadAnalysis } from "../analysis/load.js";
|
|
@@ -35,15 +35,25 @@ const KEY_MISSING_REASONS = new Set([
|
|
|
35
35
|
"AUTOMATIC_ANALYSIS_KEY_MISSING",
|
|
36
36
|
"AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING",
|
|
37
37
|
]);
|
|
38
|
+
/** The effort an analysis configuration declares; Codex configurations declare none. */
|
|
39
|
+
const declaredEffort = (config) => "reasoningEffort" in config ? config.reasoningEffort : undefined;
|
|
38
40
|
/** The `analyze` command that runs `analyst` on the run; undefined is the OpenAI default. */
|
|
39
41
|
function analyzeCommand(source, analyst) {
|
|
40
42
|
const run = cli(`analyze --run ${source.runId}${source.cwdFlag}`);
|
|
41
43
|
if (analyst?.provider === "codex")
|
|
42
44
|
return `${run} --provider codex`;
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
if (analyst?.provider === "gemini") {
|
|
46
|
+
const model = analyst.model === undefined ? "" : ` --model ${analyst.model}`;
|
|
47
|
+
return `${run} --provider gemini${model} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
48
|
+
}
|
|
49
|
+
// A named model or effort stays in the command, so the rerun is the same analyst. The default
|
|
50
|
+
// analyst needs neither: `analyze` without flags runs it.
|
|
51
|
+
const isDefault = (analyst?.model ?? DEFAULT_ANALYSIS_MODEL) === DEFAULT_ANALYSIS_MODEL &&
|
|
52
|
+
(analyst?.model === undefined || analyst.reasoningEffort === DEFAULT_ANALYSIS_EFFORT);
|
|
53
|
+
const flags = isDefault
|
|
54
|
+
? ""
|
|
55
|
+
: ` --model ${analyst.model}${analyst.reasoningEffort === undefined ? "" : ` --reasoning-effort ${analyst.reasoningEffort}`}`;
|
|
56
|
+
return `${run}${flags} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
47
57
|
}
|
|
48
58
|
/** The cited items the analysis evidence resolves, each once, in citation order. */
|
|
49
59
|
function citedEvidence(ids, analysis, runRoot) {
|
|
@@ -180,7 +190,7 @@ function missingKeyFindings(source, reason) {
|
|
|
180
190
|
}));
|
|
181
191
|
const opening = "The automatic analysis was skipped because OPENAI_API_KEY is not set.";
|
|
182
192
|
if (source.codexSignedIn !== true)
|
|
183
|
-
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, undefined));
|
|
193
|
+
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, source.analyst?.provider === "gemini" ? undefined : source.analyst));
|
|
184
194
|
const file = source.study?.path === undefined ? "the study file" : source.study.path;
|
|
185
195
|
return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, { provider: "codex" }));
|
|
186
196
|
}
|
|
@@ -234,7 +244,11 @@ async function studyAnalyst(cwd, study) {
|
|
|
234
244
|
return undefined;
|
|
235
245
|
const analysis = resolveAutomaticAnalysis(resolved.config.review?.analysis, resolved.config.actor);
|
|
236
246
|
return analysis.ok && analysis.config !== undefined
|
|
237
|
-
? {
|
|
247
|
+
? {
|
|
248
|
+
provider: analysis.config.provider,
|
|
249
|
+
model: analysis.config.model,
|
|
250
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
251
|
+
}
|
|
238
252
|
: undefined;
|
|
239
253
|
}
|
|
240
254
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
@@ -255,7 +269,11 @@ export function analysisFindings(source) {
|
|
|
255
269
|
? `The run's evidence changed after analysis ${analysis.id}, so its findings may not match the run.`
|
|
256
270
|
: `Analysis ${analysis.id} is ${analysis.status}, with ${plural(result.findings.length, "finding")}.`,
|
|
257
271
|
next: stale
|
|
258
|
-
? analyzeCommand(source, {
|
|
272
|
+
? analyzeCommand(source, {
|
|
273
|
+
provider: analysis.provider,
|
|
274
|
+
model: analysis.config.model,
|
|
275
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
276
|
+
})
|
|
259
277
|
: null,
|
|
260
278
|
analysisId: analysis.id,
|
|
261
279
|
status: analysis.status,
|
|
@@ -279,7 +297,11 @@ export function analysisFindings(source) {
|
|
|
279
297
|
if (pending !== undefined)
|
|
280
298
|
return pending;
|
|
281
299
|
if (analysis && analysis.result === null)
|
|
282
|
-
return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, {
|
|
300
|
+
return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, {
|
|
301
|
+
provider: analysis.provider,
|
|
302
|
+
model: analysis.config.model,
|
|
303
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
304
|
+
}));
|
|
283
305
|
if (job?.state === "failed" || job?.state === "cancelled") {
|
|
284
306
|
const outcome = analysisOutcomeText(job);
|
|
285
307
|
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? { provider: "codex" } : undefined));
|
package/dist/remote/launch.d.ts
CHANGED
|
@@ -38,7 +38,10 @@ export declare function followRemoteRun(args: {
|
|
|
38
38
|
signals: Signals;
|
|
39
39
|
proxyUrl: string;
|
|
40
40
|
}): Promise<RemoteOutcome>;
|
|
41
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* Two sandbox journals as one: every line of each, in order, once. A copy that runs again adds
|
|
43
|
+
* nothing it already added.
|
|
44
|
+
*/
|
|
42
45
|
export declare function joinJournals(...journals: string[]): string;
|
|
43
46
|
export interface Signals {
|
|
44
47
|
readonly requested: "stop" | "detach" | undefined;
|
package/dist/remote/launch.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// participants' provider, follow it from here, and copy the finished run back. This machine only
|
|
3
3
|
// plans, launches and watches, so it may sleep or lose its network while the study goes on.
|
|
4
4
|
import { execFile } from "node:child_process";
|
|
5
|
-
import { randomBytes } from "node:crypto";
|
|
5
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
6
6
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
7
7
|
import { tmpdir } from "node:os";
|
|
8
8
|
import path from "node:path";
|
|
@@ -18,13 +18,19 @@ import { e2bConnection } from "../substrates/e2b/connection.js";
|
|
|
18
18
|
import { acquireE2BShellSandbox } from "../substrates/e2b/sandbox.js";
|
|
19
19
|
import { loadModalShellModule } from "../substrates/modal/sandbox.js";
|
|
20
20
|
import { serveObserverProxy } from "./observer-proxy.js";
|
|
21
|
-
import { installHumanish, listRemoteRun, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
|
|
21
|
+
import { installHumanish, listRemoteRun, listSettledEvidence, readRemoteBytes, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
|
|
22
22
|
/** How often the launcher reads the runner's run while it follows it. */
|
|
23
23
|
const POLL_MS = 5_000;
|
|
24
24
|
/** Reads in a row the runner may leave unanswered before the launcher gives it up. */
|
|
25
25
|
const UNANSWERED_LIMIT = 24;
|
|
26
26
|
/** How long the run has to write its status record after it is started. */
|
|
27
27
|
const START_DEADLINE_MS = 3 * 60_000;
|
|
28
|
+
/** How often the launcher copies the run's settled evidence while it goes on. */
|
|
29
|
+
const SYNC_EVERY_MS = 30_000;
|
|
30
|
+
/** How long a screenshot or recording must be unchanged before it is copied mid-run. */
|
|
31
|
+
const SETTLED_SECONDS = 10;
|
|
32
|
+
/** The largest file read back to compare digests: a recording's own limit, with room. */
|
|
33
|
+
const LARGEST_FILE_BYTES = 256 * 1024 * 1024;
|
|
28
34
|
/** How long a stopped run has to end and write its result. */
|
|
29
35
|
const STOP_DEADLINE_MS = 2 * 60_000;
|
|
30
36
|
/** Launch, follow and copy back one remote run. */
|
|
@@ -117,7 +123,15 @@ async function acquireRunner(launch, paths) {
|
|
|
117
123
|
export async function followRemoteRun(args) {
|
|
118
124
|
const { launch, runner, paths, runId, signals } = args;
|
|
119
125
|
const { say } = launch;
|
|
120
|
-
|
|
126
|
+
// Evidence already copied while the run went on, by path, with its size then.
|
|
127
|
+
const copied = new Map();
|
|
128
|
+
const copy = async () => {
|
|
129
|
+
const { files, alreadyHere } = await copyRunBack(runner, paths, runId);
|
|
130
|
+
if (alreadyHere > 0)
|
|
131
|
+
say(`${alreadyHere} of its ${files} files were already here.`);
|
|
132
|
+
return files;
|
|
133
|
+
};
|
|
134
|
+
let syncAt = Date.now() + SYNC_EVERY_MS;
|
|
121
135
|
let state;
|
|
122
136
|
let stopAt;
|
|
123
137
|
let unanswered = 0;
|
|
@@ -188,6 +202,11 @@ export async function followRemoteRun(args) {
|
|
|
188
202
|
...("files" in copied ? { files: copied.files } : {}),
|
|
189
203
|
});
|
|
190
204
|
}
|
|
205
|
+
if (state === "running" && Date.now() >= syncAt) {
|
|
206
|
+
syncAt = Date.now() + SYNC_EVERY_MS;
|
|
207
|
+
// Best effort: what this misses, the copy at the end takes.
|
|
208
|
+
await copySettledEvidence(runner, paths, runId, copied).catch(() => undefined);
|
|
209
|
+
}
|
|
191
210
|
await new Promise((resolve) => setTimeout(resolve, POLL_MS));
|
|
192
211
|
}
|
|
193
212
|
}
|
|
@@ -198,9 +217,16 @@ export async function followRemoteRun(args) {
|
|
|
198
217
|
*/
|
|
199
218
|
async function copyRunBack(runner, paths, runId) {
|
|
200
219
|
const files = await listRemoteRun(runner, runId);
|
|
220
|
+
let alreadyHere = 0;
|
|
201
221
|
for (const file of files) {
|
|
202
222
|
if (file.path === "owner-key")
|
|
203
223
|
continue;
|
|
224
|
+
// Already here with the same bytes, copied while the run went on or by an earlier attempt.
|
|
225
|
+
if (file.path !== SANDBOX_RECEIPTS_ARTIFACT &&
|
|
226
|
+
(await localDigest(paths, file.path)) === file.sha256) {
|
|
227
|
+
alreadyHere += 1;
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
204
230
|
const bytes = await readRemoteFile(runner, runId, file);
|
|
205
231
|
if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
|
|
206
232
|
const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
|
|
@@ -211,13 +237,32 @@ async function copyRunBack(runner, paths, runId) {
|
|
|
211
237
|
await writeContainedOutputFile(paths, file.path, bytes);
|
|
212
238
|
}
|
|
213
239
|
await pointLatestRunAt(paths, runId);
|
|
214
|
-
return files.length;
|
|
240
|
+
return { files: files.length, alreadyHere };
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Copy the run's screenshots and recordings that have settled and are not here yet, while the run
|
|
244
|
+
* goes on, so the copy at its end is only what is left.
|
|
245
|
+
*/
|
|
246
|
+
async function copySettledEvidence(runner, paths, runId, copied) {
|
|
247
|
+
for (const file of await listSettledEvidence(runner, runId, SETTLED_SECONDS)) {
|
|
248
|
+
if (copied.get(file.path) === file.size)
|
|
249
|
+
continue;
|
|
250
|
+
await writeContainedOutputFile(paths, file.path, await readRemoteBytes(runner, runId, file.path));
|
|
251
|
+
copied.set(file.path, file.size);
|
|
252
|
+
}
|
|
215
253
|
}
|
|
216
|
-
|
|
254
|
+
async function localDigest(paths, file) {
|
|
255
|
+
const read = await readContainedRegularFile(paths, file, LARGEST_FILE_BYTES);
|
|
256
|
+
return read.status === "read" ? createHash("sha256").update(read.bytes).digest("hex") : undefined;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Two sandbox journals as one: every line of each, in order, once. A copy that runs again adds
|
|
260
|
+
* nothing it already added.
|
|
261
|
+
*/
|
|
217
262
|
export function joinJournals(...journals) {
|
|
218
|
-
const lines =
|
|
219
|
-
.flatMap((journal) => journal.split("\n"))
|
|
220
|
-
|
|
263
|
+
const lines = [
|
|
264
|
+
...new Set(journals.flatMap((journal) => journal.split("\n")).filter((line) => line.trim() !== "")),
|
|
265
|
+
];
|
|
221
266
|
return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
|
|
222
267
|
}
|
|
223
268
|
/**
|
package/dist/remote/runner.d.ts
CHANGED
|
@@ -59,6 +59,22 @@ export interface RemoteFile {
|
|
|
59
59
|
export declare function listRemoteRun(runner: Runner, runId: string): Promise<RemoteFile[]>;
|
|
60
60
|
/** sha256sum lines, `<hex> ./<path>`, as RemoteFiles with paths relative to the directory. */
|
|
61
61
|
export declare function parseDigests(output: string): RemoteFile[];
|
|
62
|
+
/**
|
|
63
|
+
* The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
|
|
64
|
+
* with their sizes. A run writes each of them once, so one that has settled can be copied while
|
|
65
|
+
* the run goes on; files whose name starts with a dot are still being written.
|
|
66
|
+
*/
|
|
67
|
+
export declare function listSettledEvidence(runner: Runner, runId: string, settledSeconds: number): Promise<{
|
|
68
|
+
path: string;
|
|
69
|
+
size: number;
|
|
70
|
+
}[]>;
|
|
71
|
+
/** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
|
|
72
|
+
export declare function parseSizes(output: string): {
|
|
73
|
+
path: string;
|
|
74
|
+
size: number;
|
|
75
|
+
}[];
|
|
76
|
+
/** The bytes of one file of the runner's run directory, as they are now. */
|
|
77
|
+
export declare function readRemoteBytes(runner: Runner, runId: string, file: string): Promise<Uint8Array>;
|
|
62
78
|
/** The bytes of one file of the runner's run directory, checked against its listed digest. */
|
|
63
79
|
export declare function readRemoteFile(runner: Runner, runId: string, file: RemoteFile): Promise<Uint8Array>;
|
|
64
80
|
/**
|
package/dist/remote/runner.js
CHANGED
|
@@ -189,16 +189,41 @@ export function parseDigests(output) {
|
|
|
189
189
|
}
|
|
190
190
|
return files;
|
|
191
191
|
}
|
|
192
|
-
/**
|
|
193
|
-
|
|
192
|
+
/**
|
|
193
|
+
* The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
|
|
194
|
+
* with their sizes. A run writes each of them once, so one that has settled can be copied while
|
|
195
|
+
* the run goes on; files whose name starts with a dot are still being written.
|
|
196
|
+
*/
|
|
197
|
+
export async function listSettledEvidence(runner, runId, settledSeconds) {
|
|
198
|
+
const dir = `${RUNNER_PROJECT}/.humanish/runs/${runId}`;
|
|
199
|
+
const listed = await runner.shell.run(`cd ${shellQuote(dir)} 2>/dev/null && find screenshots recordings -type f ! -name '.*' ! -newermt '${settledSeconds} seconds ago' -printf '%p\\t%s\\n' 2>/dev/null`, { timeoutMs: 120_000 });
|
|
200
|
+
return parseSizes(listed.stdout);
|
|
201
|
+
}
|
|
202
|
+
/** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
|
|
203
|
+
export function parseSizes(output) {
|
|
204
|
+
const files = [];
|
|
205
|
+
for (const line of output.split("\n")) {
|
|
206
|
+
const match = /^((?:screenshots|recordings)\/[^\t]+)\t(\d+)$/.exec(line);
|
|
207
|
+
if (match === null || match[1].split("/").includes(".."))
|
|
208
|
+
continue;
|
|
209
|
+
files.push({ path: match[1], size: Number(match[2]) });
|
|
210
|
+
}
|
|
211
|
+
return files;
|
|
212
|
+
}
|
|
213
|
+
/** The bytes of one file of the runner's run directory, as they are now. */
|
|
214
|
+
export async function readRemoteBytes(runner, runId, file) {
|
|
194
215
|
const read = runner.sandbox.files.read;
|
|
195
216
|
if (read === undefined)
|
|
196
217
|
throw new Error("This sandbox cannot read files back.");
|
|
197
|
-
const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file
|
|
218
|
+
const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file}`, { format: "stream", requestTimeoutMs: 300_000 });
|
|
198
219
|
const parts = [];
|
|
199
220
|
for await (const chunk of stream)
|
|
200
221
|
parts.push(chunk);
|
|
201
|
-
|
|
222
|
+
return Buffer.concat(parts);
|
|
223
|
+
}
|
|
224
|
+
/** The bytes of one file of the runner's run directory, checked against its listed digest. */
|
|
225
|
+
export async function readRemoteFile(runner, runId, file) {
|
|
226
|
+
const bytes = await readRemoteBytes(runner, runId, file.path);
|
|
202
227
|
const digest = createHash("sha256").update(bytes).digest("hex");
|
|
203
228
|
if (digest !== file.sha256)
|
|
204
229
|
throw new Error(`${file.path} changed or arrived damaged while it was copied from the runner.`);
|
|
@@ -25,7 +25,8 @@ study's input and admission estimate without credentials, a provider request,
|
|
|
25
25
|
or a new analysis artifact. It prints the expected cost, the worst case and the
|
|
26
26
|
cap; a refused dry run says why, and names the `--max-cost` that admits it.
|
|
27
27
|
|
|
28
|
-
The default provider is `openai`; its default model is `gpt-6-
|
|
28
|
+
The default provider is `openai`; its default model is `gpt-6.1-sol`, at medium reasoning effort
|
|
29
|
+
([reasoning effort](#reasoning-effort)). A request sends selected retained text and
|
|
29
30
|
captures to OpenAI, without tools, redirects, provider-side response storage, or
|
|
30
31
|
automatic retries. `--question` adds a reviewer question; it never changes the
|
|
31
32
|
participant assignment. For OpenAI API analysis, `--max-cost` is required, including for dry-run
|
|
@@ -118,6 +119,37 @@ the current merge instructions, billed $1.92, most of its input served from the
|
|
|
118
119
|
provider's prompt cache, and each of its 466 citations resolved. Two runs do not
|
|
119
120
|
calibrate the merge request's estimate.
|
|
120
121
|
|
|
122
|
+
## Reasoning effort
|
|
123
|
+
|
|
124
|
+
`review.analysis.reasoningEffort`, or `analyze --reasoning-effort <level>`, sets how long an API
|
|
125
|
+
analyst reasons before it answers. OpenAI receives it as `reasoning.effort`, and Gemini as
|
|
126
|
+
`generationConfig.thinkingConfig.thinkingLevel` in capitals. Each model takes the levels its
|
|
127
|
+
provider documents for it (OpenAI's model pages and Gemini's thinking guide, read 2026-10-21):
|
|
128
|
+
|
|
129
|
+
| Model | Levels |
|
|
130
|
+
| ----------------------------------------------------------------------- | ----------------------------------------------- |
|
|
131
|
+
| `gpt-6-astra`, `gpt-6.1-sol` | `low`, `medium`, `high`, `xhigh`, `max` |
|
|
132
|
+
| `gpt-6-luna`, `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna` | `none`, `low`, `medium`, `high`, `xhigh`, `max` |
|
|
133
|
+
| `gpt-5.5` | `none`, `low`, `medium`, `high`, `xhigh` |
|
|
134
|
+
| `gemini-3.8-flash` | `low`, `medium`, `high` |
|
|
135
|
+
|
|
136
|
+
Any other level is refused before a request, with a message naming the levels the model takes.
|
|
137
|
+
The Codex account analyst runs at its fixed low effort, and a declared effort with
|
|
138
|
+
`provider: codex` is refused.
|
|
139
|
+
|
|
140
|
+
The default analyst, an OpenAI analysis that names no model and declares no effort, is
|
|
141
|
+
`gpt-6.1-sol` at `medium`, and its resolved configuration writes `reasoningEffort: medium`. A study
|
|
142
|
+
or command that names a model but no effort runs that model at `high`; only the default analyst
|
|
143
|
+
runs at medium. That analysis's configuration has no `reasoningEffort` field, so its configuration
|
|
144
|
+
hash, and with it reuse of an earlier analysis, is what it was before the field existed. A declared effort is stored in the analysis artifact's
|
|
145
|
+
`config`, `high` included, and gives the configuration its own hash. An artifact whose config has
|
|
146
|
+
no `reasoningEffort` ran at `high`.
|
|
147
|
+
|
|
148
|
+
Admission expects the same output at every effort, from analyses billed at `high`. No billed
|
|
149
|
+
analysis at a lower effort calibrates a smaller expectation yet, so a lower effort's expected cost
|
|
150
|
+
is likely over its bill: on 24 one-participant Taskly analyses on 2026-10-22, `gpt-6.1-sol` at
|
|
151
|
+
medium wrote a mean 3,747 output tokens against the 13,000 admission expects. The worst case spends the whole output allowance at any effort.
|
|
152
|
+
|
|
121
153
|
## Gemini API analysis
|
|
122
154
|
|
|
123
155
|
`review.analysis.provider: gemini`, or `analyze --provider gemini`, sends the same request to
|
|
@@ -131,7 +163,8 @@ humanish analyze --run latest --provider gemini --model gemini-3.8-flash --max-c
|
|
|
131
163
|
|
|
132
164
|
The request goes to `generativelanguage.googleapis.com/v1beta/models/<model>:generateContent`
|
|
133
165
|
with the key in the `x-goog-api-key` header. It carries the same instructions, packet and
|
|
134
|
-
captures, with each capture at `MEDIA_RESOLUTION_HIGH`,
|
|
166
|
+
captures, with each capture at `MEDIA_RESOLUTION_HIGH`, the analysis's reasoning effort as its
|
|
167
|
+
thinking level (`HIGH` unless declared), one candidate,
|
|
135
168
|
the standard service tier, no tools and `store: false`. The key is `GEMINI_API_KEY` when set,
|
|
136
169
|
else `GOOGLE_API_KEY`. Google's client libraries prefer `GOOGLE_API_KEY`; humanish does not, so a
|
|
137
170
|
`GOOGLE_API_KEY` exported for another Google API never replaces the stored Gemini key. Key
|
|
@@ -10,8 +10,12 @@ Every other participant, including a Claude Code `local-agent` participant, gets
|
|
|
10
10
|
analyst. humanish never moves a study to the other analyst because a key is missing: the account
|
|
11
11
|
that pays for analysis is the one the study names or its participant implies.
|
|
12
12
|
|
|
13
|
-
The OpenAI default is `gpt-6-
|
|
14
|
-
|
|
13
|
+
The OpenAI default is `gpt-6.1-sol` at medium reasoning effort (`review.analysis.reasoningEffort`
|
|
14
|
+
sets another), a separate $10 cap on its
|
|
15
|
+
expected cost and a 600-second timeout. A study or command that names a model but no effort runs
|
|
16
|
+
that model at high; only the default analyst, with no model named, runs at medium. On the same
|
|
17
|
+
recordings it matched `gpt-6-astra` at high, the earlier default, at about one seventh of the cost
|
|
18
|
+
([analysis models on fixed recordings](https://github.com/danielgwilson/humanish/blob/main/docs/evidence/benchmark/2026-10-22-analysis-models.md)). When no output limit is specified,
|
|
15
19
|
humanish selects 32,768 tokens if the exact input's expected cost still fits that
|
|
16
20
|
cap; otherwise it keeps the established 16,384-token allowance. This preserves
|
|
17
21
|
previously admitted studies without increasing their spending limit. To customize it:
|
|
@@ -21,7 +25,9 @@ review:
|
|
|
21
25
|
analysis:
|
|
22
26
|
maxCostUsd: 10
|
|
23
27
|
# Optional overrides. Omit maxOutputTokens for budget-aware selection.
|
|
24
|
-
model
|
|
28
|
+
# A named model without reasoningEffort runs at high.
|
|
29
|
+
model: gpt-6.1-sol
|
|
30
|
+
reasoningEffort: medium
|
|
25
31
|
timeoutMs: 600000
|
|
26
32
|
# maxOutputTokens: 32768
|
|
27
33
|
# question: Where did participants need to recover?
|
|
@@ -41,9 +47,12 @@ the cap admits no analysis at all, it says so instead. A study of more than 16 p
|
|
|
41
47
|
analysed in cohorts of at most 16, one request each, and one more request merges their reports;
|
|
42
48
|
the range covers every request and the analysis line names them (see
|
|
43
49
|
[runs with more than 16 participants](../contracts/study-analysis.md#runs-with-more-than-16-participants)).
|
|
44
|
-
The default $10 cap admits
|
|
45
|
-
participant
|
|
46
|
-
|
|
50
|
+
The default $10 cap admits the default analyst up to the evidence limits: a single request at
|
|
51
|
+
$0.76 expected for one participant and $0.91 for 16, and 17 to 40 participants at $0.92 to $3.24
|
|
52
|
+
from no evidence to the limits. Admission expects 13,000 output tokens for one participant at any
|
|
53
|
+
effort; on 24 Taskly analyses `gpt-6.1-sol` at medium wrote a mean 3,747 and cost a mean $0.06.
|
|
54
|
+
`gpt-6-astra` at high expects $3.79 for one participant at the limits and $4.60 to $7.05 for 17 to
|
|
55
|
+
40 with no evidence, so a run that keeps more evidence can be refused, and its skip names the
|
|
47
56
|
`--max-cost` that admits it. Use `analyze --dry-run --max-cost <usd>` on retained
|
|
48
57
|
evidence to inspect the expected cost, the worst case and the selected token
|
|
49
58
|
allowance before deliberately choosing a larger budget. Explicit
|
|
@@ -89,7 +98,8 @@ review:
|
|
|
89
98
|
maxCostUsd: 2
|
|
90
99
|
```
|
|
91
100
|
|
|
92
|
-
Gemini analysis sends the same evidence packet and captures to Google, at high thinking level
|
|
101
|
+
Gemini analysis sends the same evidence packet and captures to Google, at the high thinking level
|
|
102
|
+
unless the study declares a `reasoningEffort`,
|
|
93
103
|
with each capture at high media resolution. It reads `GEMINI_API_KEY`, which `humanish keys set
|
|
94
104
|
gemini` stores, and `GOOGLE_API_KEY` only when `GEMINI_API_KEY` is unset. Key discovery treats
|
|
95
105
|
the two names as one key: a project's `.humanish/local/provider.env` or the user store never adds
|
package/package.json
CHANGED
package/skills/humanish/SKILL.md
CHANGED
|
@@ -98,7 +98,8 @@ exact returned path, not a basename that could resolve to another manifest.
|
|
|
98
98
|
Analysis is separate from the participant. A study that declares no analyst
|
|
99
99
|
gets one from its participant: a Codex `local-agent` participant, local or
|
|
100
100
|
hosted, defaults to a separate Codex account analyst; every other participant,
|
|
101
|
-
Claude Code included, defaults to the OpenAI API
|
|
101
|
+
Claude Code included, defaults to the OpenAI API (`gpt-6.1-sol` at medium
|
|
102
|
+
reasoning effort) with a $10 admission budget.
|
|
102
103
|
Without `OPENAI_API_KEY` that OpenAI analysis is skipped, and the run output
|
|
103
104
|
names the next command: `analyze --provider codex` when Codex is signed in. To
|
|
104
105
|
select the restricted Codex ChatGPT account analyst on any supported study,
|
|
@@ -118,6 +119,12 @@ and a missing Gemini key never falls back to OpenAI. On a free-tier key, Google
|
|
|
118
119
|
may use the content to improve its products; tell the user before a study with
|
|
119
120
|
private evidence runs on one.
|
|
120
121
|
|
|
122
|
+
`review.analysis.reasoningEffort` (or `analyze --reasoning-effort <level>`)
|
|
123
|
+
sets an OpenAI or Gemini analyst's reasoning effort. When unset it is `medium`
|
|
124
|
+
for the default analyst, which names no model, and `high` for a named model.
|
|
125
|
+
Each model takes only the levels its provider documents, and `analyze` refuses
|
|
126
|
+
the others before any request with the levels the model takes.
|
|
127
|
+
|
|
121
128
|
When post-run analysis is refused for its cost, the run output and `humanish
|
|
122
129
|
review` give its expected cost, its worst case and the command that runs it,
|
|
123
130
|
`humanish analyze --run <id> --max-cost <n>`. Show the user those costs and run
|