humanish 0.142.0 → 0.144.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +1 -1
- package/dist/analysis/automatic-config.d.ts +12 -1
- package/dist/analysis/automatic-config.js +35 -6
- package/dist/analysis/execute.d.ts +1 -11
- package/dist/analysis/execute.js +4 -21
- package/dist/analysis/gemini-wire.d.ts +2 -1
- package/dist/analysis/gemini-wire.js +3 -2
- package/dist/analysis/models.d.ts +12 -0
- package/dist/analysis/models.js +47 -0
- package/dist/analysis/openai-wire.d.ts +2 -1
- package/dist/analysis/openai-wire.js +3 -2
- package/dist/analysis/provider.d.ts +4 -0
- package/dist/analysis/types.d.ts +4 -0
- package/dist/analysis/validation.js +4 -3
- package/dist/cli/commands/analyze.js +5 -3
- package/dist/cli/commands/run-command.js +2 -1
- package/dist/cli/commands/study-remote.d.ts +34 -2
- package/dist/cli/commands/study-remote.js +103 -52
- package/dist/cli/commands/study-route-run.d.ts +3 -2
- package/dist/cli/commands/study-route-run.js +7 -2
- package/dist/cli/commands/study-run.js +19 -9
- package/dist/cli/findings.d.ts +2 -1
- package/dist/cli/findings.js +31 -9
- package/dist/cli/io.d.ts +2 -0
- package/dist/remote/launch.d.ts +17 -2
- package/dist/remote/launch.js +108 -29
- package/dist/remote/plan.d.ts +2 -0
- package/dist/remote/plan.js +12 -0
- package/dist/remote/runner.d.ts +21 -0
- package/dist/remote/runner.js +42 -5
- package/dist/run/results.d.ts +9 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -0
- package/dist/study/parse/execution.js +5 -0
- package/dist/study/types.d.ts +6 -0
- package/dist/tui/launch.js +3 -0
- package/docs/contracts/study-analysis.md +35 -2
- package/docs/product/automatic-analysis.md +17 -7
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +16 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,27 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.144.0: Hosted studies run remotely by default (2026-10-22)
|
|
12
|
+
|
|
13
|
+
humanish 0.144.0 runs a hosted computer-use study remotely by default. A study on
|
|
14
|
+
`e2b-desktop` or `modal-desktop` with a public `app-url` runs its own process in a sandbox in your
|
|
15
|
+
E2B or Modal account, so it keeps going when this machine sleeps; the command follows it, serves
|
|
16
|
+
the live Observer here and copies the evidence back. Before the runner starts, the command names
|
|
17
|
+
the keys it sends. Pass `--local` for one run, or set `execution.remote: false` in a study, to keep
|
|
18
|
+
it on this machine.
|
|
19
|
+
|
|
20
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.144.0)
|
|
21
|
+
|
|
22
|
+
## 0.143.0: Analysis costs about one seventh as much: gpt-6.1-sol at medium effort is the default (2026-10-22)
|
|
23
|
+
|
|
24
|
+
humanish 0.143.0 analyzes studies with `gpt-6.1-sol` at medium reasoning effort by default, in
|
|
25
|
+
place of `gpt-6-astra` at high. On the same recordings it found what `gpt-6-astra` found, invented
|
|
26
|
+
nothing, and cost about one seventh as much: $0.06 against $0.41 per analysis on the Taskly
|
|
27
|
+
benchmark. A study can set the analysis effort with `review.analysis.reasoningEffort`, and one
|
|
28
|
+
that names its model keeps it at high effort with an unchanged configuration hash.
|
|
29
|
+
|
|
30
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.143.0)
|
|
31
|
+
|
|
11
32
|
## 0.142.0: Remote runs, Gemini and GPT-6 analysis models, and a view-only live stream (2026-10-21)
|
|
12
33
|
|
|
13
34
|
humanish 0.142.0 can run a study remotely: `humanish run <study> --remote` puts the study's own
|
package/README.md
CHANGED
|
@@ -249,7 +249,7 @@ coverage limits, estimated cost controls, corrections and sharing behavior.
|
|
|
249
249
|
|
|
250
250
|
Supported live studies generate findings automatically after participants finish.
|
|
251
251
|
A Codex participant's run is analyzed by the same Codex account; every other run uses
|
|
252
|
-
`gpt-6-
|
|
252
|
+
`gpt-6.1-sol` at medium reasoning effort and a separate $10 cap on the expected cost.
|
|
253
253
|
This is additional to participant execution costs, and is not a hard provider
|
|
254
254
|
billing cap. To disable the extra request:
|
|
255
255
|
|
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
import type { StudyRoute } from "../study/plan.js";
|
|
2
|
+
import { type ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
3
|
import type { AnalysisConfig } from "./types.js";
|
|
3
4
|
export declare const DEFAULT_ANALYSIS_TIMEOUT_MS = 600000;
|
|
4
|
-
|
|
5
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
6
|
+
export declare const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
7
|
+
/**
|
|
8
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
9
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
10
|
+
*/
|
|
11
|
+
export declare const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
5
12
|
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
6
13
|
* stable Flash model. */
|
|
7
14
|
export declare const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
@@ -15,10 +22,12 @@ export type StudyAnalysis = StudyAnalysisSettings & ({
|
|
|
15
22
|
provider?: "openai" | "gemini";
|
|
16
23
|
maxCostUsd: number;
|
|
17
24
|
maxOutputTokens?: number;
|
|
25
|
+
reasoningEffort?: ReasoningEffort;
|
|
18
26
|
} | {
|
|
19
27
|
provider: "codex";
|
|
20
28
|
maxCostUsd?: null;
|
|
21
29
|
maxOutputTokens?: null;
|
|
30
|
+
reasoningEffort?: never;
|
|
22
31
|
});
|
|
23
32
|
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
24
33
|
export declare const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
@@ -43,6 +52,8 @@ export interface AutomaticAnalysisBudget {
|
|
|
43
52
|
provider?: "openai" | "gemini" | "codex";
|
|
44
53
|
billing?: "api-estimate" | "account-unknown";
|
|
45
54
|
model: string;
|
|
55
|
+
/** The reasoning effort the analysis runs at. */
|
|
56
|
+
effort: ReasoningEffort;
|
|
46
57
|
maxCostUsd: number | null;
|
|
47
58
|
trigger: "default" | "explicit";
|
|
48
59
|
/** The participants the expected cost range is for. */
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange,
|
|
2
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, } from "./execute.js";
|
|
3
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
4
|
+
import { isReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
5
|
import { admissionRule } from "./admission.js";
|
|
4
6
|
import { analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
5
7
|
import { plural } from "../run/text.js";
|
|
6
8
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
7
9
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
8
10
|
const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
9
|
-
|
|
11
|
+
/** The OpenAI analyst a study or command gets when it names no model. */
|
|
12
|
+
export const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
|
|
13
|
+
/**
|
|
14
|
+
* The effort the default OpenAI analyst runs at when no effort is declared. A named model without
|
|
15
|
+
* an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_ANALYSIS_EFFORT = "medium";
|
|
10
18
|
/** The model a Gemini analysis uses when the study or command names none: Google's current
|
|
11
19
|
* stable Flash model. */
|
|
12
20
|
export const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
|
|
@@ -17,6 +25,7 @@ const FIELDS = new Set([
|
|
|
17
25
|
"question",
|
|
18
26
|
"timeoutMs",
|
|
19
27
|
"maxOutputTokens",
|
|
28
|
+
"reasoningEffort",
|
|
20
29
|
]);
|
|
21
30
|
const validTimeout = (value) => typeof value === "number" &&
|
|
22
31
|
Number.isSafeInteger(value) &&
|
|
@@ -59,7 +68,7 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
59
68
|
Object.keys(raw).some((key) => !FIELDS.has(key))) {
|
|
60
69
|
return {
|
|
61
70
|
ok: false,
|
|
62
|
-
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs and
|
|
71
|
+
message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs, maxOutputTokens and reasoningEffort.",
|
|
63
72
|
};
|
|
64
73
|
}
|
|
65
74
|
const value = raw;
|
|
@@ -70,6 +79,11 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
70
79
|
return { ok: false, message: "review.analysis.provider must be openai, gemini or codex." };
|
|
71
80
|
}
|
|
72
81
|
if (value.provider === "codex") {
|
|
82
|
+
if (value.reasoningEffort !== undefined)
|
|
83
|
+
return {
|
|
84
|
+
ok: false,
|
|
85
|
+
message: "Codex account analysis runs at its fixed low reasoning effort, so it takes no reasoningEffort. Omit it, or choose provider openai or gemini to set one.",
|
|
86
|
+
};
|
|
73
87
|
const model = value.model === undefined ? CODEX_ANALYSIS_MODEL : value.model;
|
|
74
88
|
const timeoutMs = value.timeoutMs === undefined ? DEFAULT_ANALYSIS_TIMEOUT_MS : value.timeoutMs;
|
|
75
89
|
const question = value.question === undefined ? null : value.question;
|
|
@@ -127,18 +141,32 @@ export function resolveAutomaticAnalysis(raw, participant) {
|
|
|
127
141
|
message: `review.analysis requires a positive maxCostUsd up to 1000, a supported ${vendor === "gemini" ? "Gemini" : "OpenAI"} analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
|
|
128
142
|
};
|
|
129
143
|
}
|
|
144
|
+
// Only the default model gets the default effort, written out because an omitted effort is high.
|
|
145
|
+
const reasoningEffort = value.reasoningEffort === undefined && value.model === undefined && vendor === "openai"
|
|
146
|
+
? DEFAULT_ANALYSIS_EFFORT
|
|
147
|
+
: value.reasoningEffort;
|
|
148
|
+
const efforts = analysisEfforts(model, vendor);
|
|
149
|
+
if (reasoningEffort !== undefined &&
|
|
150
|
+
(!isReasoningEffort(reasoningEffort) || !efforts.includes(reasoningEffort)))
|
|
151
|
+
return {
|
|
152
|
+
ok: false,
|
|
153
|
+
message: `review.analysis.reasoningEffort for ${model} must be ${efforts.slice(0, -1).join(", ")} or ${efforts.at(-1)}, the efforts its provider documents for it.`,
|
|
154
|
+
};
|
|
130
155
|
if (question !== null && scanEncodedText(question).sensitive)
|
|
131
156
|
return SENSITIVE_QUESTION;
|
|
132
157
|
return {
|
|
133
158
|
ok: true,
|
|
134
159
|
config: {
|
|
135
|
-
// An omitted provider
|
|
160
|
+
// An omitted provider, or an omitted effort on a named model, stays omitted, so the
|
|
161
|
+
// configuration keeps the hash it had before the field existed. An omitted provider is the
|
|
162
|
+
// OpenAI contract.
|
|
136
163
|
...(value.provider === undefined ? {} : { provider: vendor }),
|
|
137
164
|
model,
|
|
138
165
|
maxCostUsd,
|
|
139
166
|
timeoutMs,
|
|
140
167
|
maxOutputTokens,
|
|
141
168
|
question: question,
|
|
169
|
+
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
142
170
|
},
|
|
143
171
|
...(value.maxOutputTokens === undefined ? { preferLargerOutput: true } : {}),
|
|
144
172
|
};
|
|
@@ -164,6 +192,7 @@ export function automaticAnalysisBudget(raw, route, participants, participant) {
|
|
|
164
192
|
? { provider: "gemini" }
|
|
165
193
|
: {}),
|
|
166
194
|
model: config.model,
|
|
195
|
+
effort: analysisEffort(config),
|
|
167
196
|
maxCostUsd: config.maxCostUsd,
|
|
168
197
|
trigger: raw === undefined ? "default" : "explicit",
|
|
169
198
|
...(range === undefined || participants === undefined ? {} : { participants, ...range }),
|
|
@@ -180,7 +209,7 @@ export function analysisRequestsText(requests) {
|
|
|
180
209
|
}
|
|
181
210
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
182
211
|
if (budget.provider === "codex")
|
|
183
|
-
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
212
|
+
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} at ${budget.effort} effort · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
184
213
|
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
185
214
|
// The participants, and the requests that analyse them when there is more than one.
|
|
186
215
|
const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
|
|
@@ -194,5 +223,5 @@ export function formatAutomaticAnalysisBudget(budget) {
|
|
|
194
223
|
? `refused before it starts if ${rule}`
|
|
195
224
|
: `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
|
|
196
225
|
const where = budget.provider === "gemini" ? " on Gemini, which sends the evidence to Google" : "";
|
|
197
|
-
return `After live runs: ${budget.trigger} analysis${where} · ${budget.model}${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
226
|
+
return `After live runs: ${budget.trigger} analysis${where} · ${budget.model} at ${budget.effort} effort${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
198
227
|
}
|
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
|
|
2
|
-
import { type
|
|
2
|
+
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
4
|
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
5
|
-
export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
|
|
6
|
-
/**
|
|
7
|
-
* Gemini models that completed a live analysis. Each model's page under
|
|
8
|
-
* ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
|
|
9
|
-
* output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
|
|
10
|
-
* once a live analysis on each completes.
|
|
11
|
-
*/
|
|
12
|
-
export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
|
|
13
|
-
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
14
|
-
export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
|
|
15
5
|
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
16
6
|
* returns at most 32,768 output tokens from one generateContent request. */
|
|
17
7
|
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -6,6 +6,7 @@ import { modelRate } from "../run/pricing.js";
|
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
7
7
|
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
8
8
|
import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
|
|
9
|
+
import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
|
|
9
10
|
import { createAnalysisProvider, } from "./provider.js";
|
|
10
11
|
import { hashAnalysisValue, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
|
|
11
12
|
import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
@@ -13,27 +14,6 @@ import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema,
|
|
|
13
14
|
import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
|
|
14
15
|
import { mapWithConcurrency } from "../run/concurrency.js";
|
|
15
16
|
export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
|
|
16
|
-
export const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
|
|
17
|
-
"gpt-6-astra",
|
|
18
|
-
"gpt-6.1-sol",
|
|
19
|
-
"gpt-6-luna",
|
|
20
|
-
"gpt-5.5",
|
|
21
|
-
"gpt-5.6",
|
|
22
|
-
"gpt-5.6-sol",
|
|
23
|
-
"gpt-5.6-terra",
|
|
24
|
-
"gpt-5.6-luna",
|
|
25
|
-
]);
|
|
26
|
-
const SUPPORTED_MODELS = new Set(SUPPORTED_ANALYSIS_MODELS);
|
|
27
|
-
/**
|
|
28
|
-
* Gemini models that completed a live analysis. Each model's page under
|
|
29
|
-
* ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
|
|
30
|
-
* output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
|
|
31
|
-
* once a live analysis on each completes.
|
|
32
|
-
*/
|
|
33
|
-
export const GEMINI_ANALYSIS_MODELS = Object.freeze(["gemini-3.8-flash"]);
|
|
34
|
-
const GEMINI_MODELS = new Set(GEMINI_ANALYSIS_MODELS);
|
|
35
|
-
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
36
|
-
export const isSupportedAnalysisModel = (model, provider = "openai") => (provider === "gemini" ? GEMINI_MODELS : SUPPORTED_MODELS).has(model);
|
|
37
17
|
/** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
|
|
38
18
|
* returns at most 32,768 output tokens from one generateContent request. */
|
|
39
19
|
export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
|
|
@@ -192,6 +172,8 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
192
172
|
config.provider !== "openai" &&
|
|
193
173
|
config.provider !== "gemini") ||
|
|
194
174
|
!isSupportedAnalysisModel(config.model, config.provider) ||
|
|
175
|
+
(config.reasoningEffort !== undefined &&
|
|
176
|
+
!analysisEfforts(config.model, config.provider).includes(config.reasoningEffort)) ||
|
|
195
177
|
!Number.isFinite(config.maxCostUsd) ||
|
|
196
178
|
config.maxCostUsd <= 0 ||
|
|
197
179
|
config.maxCostUsd > 1000 ||
|
|
@@ -547,6 +529,7 @@ export async function runAnalysis(input, config, options) {
|
|
|
547
529
|
images: request.images,
|
|
548
530
|
schema: request.schema,
|
|
549
531
|
maxOutputTokens: config.maxOutputTokens,
|
|
532
|
+
reasoningEffort: analysisEffort(config),
|
|
550
533
|
timeoutMs: config.timeoutMs,
|
|
551
534
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
552
535
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AnalysisWire } from "./provider.js";
|
|
2
2
|
/**
|
|
3
|
-
* One stateless request: no tools,
|
|
3
|
+
* One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
|
|
4
|
+
* ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
|
|
4
5
|
* service tier, and `store: false` so the request stays out of the project's request logs. Each
|
|
5
6
|
* capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
|
|
6
7
|
* image analysis, after a text part naming its evidence key.
|
|
@@ -116,7 +116,8 @@ function parseGenerateContent(raw) {
|
|
|
116
116
|
}
|
|
117
117
|
}
|
|
118
118
|
/**
|
|
119
|
-
* One stateless request: no tools,
|
|
119
|
+
* One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
|
|
120
|
+
* ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
|
|
120
121
|
* service tier, and `store: false` so the request stays out of the project's request logs. Each
|
|
121
122
|
* capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
|
|
122
123
|
* image analysis, after a text part naming its evidence key.
|
|
@@ -152,7 +153,7 @@ export const geminiWire = {
|
|
|
152
153
|
generationConfig: {
|
|
153
154
|
candidateCount: 1,
|
|
154
155
|
maxOutputTokens: request.maxOutputTokens,
|
|
155
|
-
thinkingConfig: { thinkingLevel:
|
|
156
|
+
thinkingConfig: { thinkingLevel: request.reasoningEffort.toUpperCase() },
|
|
156
157
|
responseFormat: {
|
|
157
158
|
text: { mimeType: "APPLICATION_JSON", schema },
|
|
158
159
|
},
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
|
+
import type { AnalysisApiVendor } from "./provider.js";
|
|
3
|
+
import type { AnalysisConfig } from "./types.js";
|
|
4
|
+
export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
|
|
5
|
+
export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
|
|
6
|
+
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
7
|
+
export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
|
|
8
|
+
/** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
|
|
9
|
+
export declare const analysisEfforts: (model: string, provider?: AnalysisApiVendor) => readonly ReasoningEffort[];
|
|
10
|
+
/** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
|
|
11
|
+
* and the Codex account's fixed effort. */
|
|
12
|
+
export declare function analysisEffort(config: AnalysisConfig): ReasoningEffort;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// The models each API analyst runs, and the reasoning efforts each model accepts. Config parsing,
|
|
2
|
+
// admission, the CLI help and the wire formats all read this one table.
|
|
3
|
+
/**
|
|
4
|
+
* `reasoning.effort` values from each model's page under developers.openai.com/api/docs/models
|
|
5
|
+
* (read 2026-10-21). None of them lists `minimal`; gpt-6-astra and gpt-6.1-sol also refuse `none`.
|
|
6
|
+
*/
|
|
7
|
+
const OPENAI_MODELS = {
|
|
8
|
+
"gpt-6-astra": ["low", "medium", "high", "xhigh", "max"],
|
|
9
|
+
"gpt-6.1-sol": ["low", "medium", "high", "xhigh", "max"],
|
|
10
|
+
"gpt-6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
11
|
+
"gpt-5.5": ["none", "low", "medium", "high", "xhigh"],
|
|
12
|
+
"gpt-5.6": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
13
|
+
"gpt-5.6-sol": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
14
|
+
"gpt-5.6-terra": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
15
|
+
"gpt-5.6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Gemini models that completed a live analysis, with the `thinkingLevel` values the thinking guide
|
|
19
|
+
* lists for each (ai.google.dev/gemini-api/docs/generate-content/thinking, read 2026-10-21):
|
|
20
|
+
* gemini-3.8-flash answers `minimal` with an error. Each model's page under
|
|
21
|
+
* ai.google.dev/gemini-api/docs/models also lists image input and JSON-schema output for
|
|
22
|
+
* gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this table once a
|
|
23
|
+
* live analysis on each completes.
|
|
24
|
+
*/
|
|
25
|
+
const GEMINI_MODELS = {
|
|
26
|
+
"gemini-3.8-flash": ["low", "medium", "high"],
|
|
27
|
+
};
|
|
28
|
+
const MODELS = { openai: OPENAI_MODELS, gemini: GEMINI_MODELS };
|
|
29
|
+
export const SUPPORTED_ANALYSIS_MODELS = Object.freeze(Object.keys(OPENAI_MODELS));
|
|
30
|
+
export const GEMINI_ANALYSIS_MODELS = Object.freeze(Object.keys(GEMINI_MODELS));
|
|
31
|
+
/** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
|
|
32
|
+
export const isSupportedAnalysisModel = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model);
|
|
33
|
+
/** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
|
|
34
|
+
export const analysisEfforts = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model) ? MODELS[provider][model] : [];
|
|
35
|
+
/**
|
|
36
|
+
* The effort an API analysis runs at when its configuration declares none. Every saved analysis
|
|
37
|
+
* without `reasoningEffort` ran at this effort, so it stays high: a different default must be
|
|
38
|
+
* written into the configuration it resolves, where it also changes the configuration's hash.
|
|
39
|
+
*/
|
|
40
|
+
const UNDECLARED_EFFORT = "high";
|
|
41
|
+
/** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
|
|
42
|
+
* and the Codex account's fixed effort. */
|
|
43
|
+
export function analysisEffort(config) {
|
|
44
|
+
if (config.provider === "codex")
|
|
45
|
+
return config.identity.reasoningEffort;
|
|
46
|
+
return config.reasoningEffort ?? UNDECLARED_EFFORT;
|
|
47
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AnalysisProviderResult, AnalysisWire } from "./provider.js";
|
|
2
2
|
/** Text and usage wire shapes reuse the captured closing-report contract; refusal fails closed. */
|
|
3
3
|
export declare function parseAnalysisResponse(raw: unknown): AnalysisProviderResult;
|
|
4
|
-
/** One stateless request: no stored response, no tools,
|
|
4
|
+
/** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
|
|
5
|
+
* schema. */
|
|
5
6
|
export declare const openaiWire: AnalysisWire;
|
|
@@ -67,7 +67,8 @@ export function parseAnalysisResponse(raw) {
|
|
|
67
67
|
return { ...base, status: "failed", errorCode: "invalid_response" };
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
|
-
/** One stateless request: no stored response, no tools,
|
|
70
|
+
/** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
|
|
71
|
+
* schema. */
|
|
71
72
|
export const openaiWire = {
|
|
72
73
|
url: () => OPENAI_RESPONSES_URL,
|
|
73
74
|
headers: (apiKey) => ({ Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }),
|
|
@@ -91,7 +92,7 @@ export const openaiWire = {
|
|
|
91
92
|
tool_choice: "none",
|
|
92
93
|
truncation: "disabled",
|
|
93
94
|
service_tier: "default",
|
|
94
|
-
reasoning: { effort:
|
|
95
|
+
reasoning: { effort: request.reasoningEffort },
|
|
95
96
|
max_output_tokens: request.maxOutputTokens,
|
|
96
97
|
text: {
|
|
97
98
|
format: { type: "json_schema", name: "study_analysis", strict: true, schema: request.schema },
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Dispatcher } from "undici";
|
|
2
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
2
3
|
/** Deliberately separate from the stateful computer-use actor: one request, no tools or retries. */
|
|
3
4
|
export interface AnalysisProviderRequest {
|
|
4
5
|
model: string;
|
|
@@ -10,6 +11,9 @@ export interface AnalysisProviderRequest {
|
|
|
10
11
|
}[];
|
|
11
12
|
schema: Record<string, unknown>;
|
|
12
13
|
maxOutputTokens: number | null;
|
|
14
|
+
/** The effort the analysis runs at (models.ts analysisEffort). The Codex account analyst runs
|
|
15
|
+
* its fixed low effort, which this then names. */
|
|
16
|
+
reasoningEffort: ReasoningEffort;
|
|
13
17
|
timeoutMs: number;
|
|
14
18
|
signal?: AbortSignal;
|
|
15
19
|
}
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
|
|
2
2
|
import type { CuaEndpointSource } from "../actors/goal-source.js";
|
|
3
|
+
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
4
|
import type { AutomaticAnalysisView } from "./job.js";
|
|
4
5
|
import type { RunAnalysisCost } from "../run/run-cost.js";
|
|
5
6
|
/** Independent interpretation of retained evidence; never a participant or harness verdict. */
|
|
@@ -160,6 +161,9 @@ interface OpenAIAnalysisConfig {
|
|
|
160
161
|
maxCostUsd: number;
|
|
161
162
|
timeoutMs: number;
|
|
162
163
|
maxOutputTokens: number;
|
|
164
|
+
/** Present only when the study or command declared it, so an omitted effort keeps the hash of
|
|
165
|
+
* every configuration before the field. An analysis without it ran at high (models.ts). */
|
|
166
|
+
reasoningEffort?: ReasoningEffort | undefined;
|
|
163
167
|
}
|
|
164
168
|
/** The Gemini API analyst: the same billed, capped request as OpenAI's, sent to Google. */
|
|
165
169
|
interface GeminiAnalysisConfig extends Omit<OpenAIAnalysisConfig, "provider"> {
|
|
@@ -3,6 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
5
|
import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
|
|
6
|
+
import { REASONING_EFFORTS } from "../actors/reasoning-effort.js";
|
|
6
7
|
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
7
8
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
8
9
|
const text = (max) => z
|
|
@@ -204,6 +205,7 @@ const apiAnalysisConfigSchema = z.object({
|
|
|
204
205
|
maxCostUsd: z.number().positive().max(1000),
|
|
205
206
|
timeoutMs: z.number().int().positive().max(3600000),
|
|
206
207
|
maxOutputTokens: z.number().int().positive().max(128000),
|
|
208
|
+
reasoningEffort: z.enum(REASONING_EFFORTS).optional(),
|
|
207
209
|
});
|
|
208
210
|
const analysisArtifactSchema = z
|
|
209
211
|
.object({
|
|
@@ -361,8 +363,7 @@ export function checkAnalysisResult(input, value) {
|
|
|
361
363
|
if (!distinct(observation.evidenceIds) ||
|
|
362
364
|
refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
363
365
|
errors.add("ANALYSIS_OBSERVATION_REFERENCE_INVALID");
|
|
364
|
-
if (observation.basis === "visual" &&
|
|
365
|
-
!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
|
|
366
|
+
if (observation.basis === "visual" && !refs.some((ref) => ref?.capture))
|
|
366
367
|
errors.add("ANALYSIS_VISUAL_WITHOUT_CAPTURE");
|
|
367
368
|
if (observation.basis === "action" &&
|
|
368
369
|
!refs.some((ref) => ref &&
|
|
@@ -394,7 +395,7 @@ export function checkAnalysisResult(input, value) {
|
|
|
394
395
|
const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
|
|
395
396
|
if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
396
397
|
errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
|
|
397
|
-
if (!refs.some((ref) => ref?.capture
|
|
398
|
+
if (!refs.some((ref) => ref?.capture))
|
|
398
399
|
errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
|
|
399
400
|
// Seen by means a cited capture of that participant shows the problem.
|
|
400
401
|
const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
|
|
1
|
+
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
5
5
|
import { admissionRule } from "../../analysis/admission.js";
|
|
6
|
-
import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS
|
|
6
|
+
import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS } from "../../analysis/models.js";
|
|
7
7
|
import { CODEX_ANALYSIS_MODEL } from "../../analysis/codex-config.js";
|
|
8
8
|
import { listAnalyses } from "../../analysis/store.js";
|
|
9
9
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
@@ -34,7 +34,8 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
34
34
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
35
35
|
.option("--provider <name>", "Analyst: openai (default), gemini (sends the evidence and captures to Google), or restricted codex account. No provider fallback.")
|
|
36
36
|
.option("--max-cost <usd>", `Required for OpenAI and Gemini, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
|
|
37
|
-
.option("--model <id>", `OpenAI analysis model
|
|
37
|
+
.option("--model <id>", `OpenAI analysis model: ${SUPPORTED_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_ANALYSIS_MODEL} at ${DEFAULT_ANALYSIS_EFFORT} effort when omitted. Gemini analysis model: ${GEMINI_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_GEMINI_ANALYSIS_MODEL} when omitted. Codex account analysis runs ${CODEX_ANALYSIS_MODEL} at low effort.`)
|
|
38
|
+
.option("--reasoning-effort <level>", `OpenAI reasoning.effort or Gemini thinking level, from none, minimal, low, medium, high, xhigh and max; each model takes only the levels its provider documents for it, and others are refused before any request. When omitted: ${DEFAULT_ANALYSIS_EFFORT} for the default OpenAI analyst, high with --model or --provider gemini. Unsupported for Codex.`)
|
|
38
39
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
39
40
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
40
41
|
.option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for admission-based sizing. Unsupported for Codex.")
|
|
@@ -84,6 +85,7 @@ async function handleAnalyze(io, options, command) {
|
|
|
84
85
|
...(options.maxOutputTokens === undefined
|
|
85
86
|
? {}
|
|
86
87
|
: { maxOutputTokens: Number(options.maxOutputTokens) }),
|
|
88
|
+
...(options.reasoningEffort === undefined ? {} : { reasoningEffort: options.reasoningEffort }),
|
|
87
89
|
});
|
|
88
90
|
if (!selected.ok || !selected.config) {
|
|
89
91
|
const refusal = await dryRunBundleRefusal(await resolvePhysicalCwd(options.cwd), options.run, options.dryRun === true);
|
|
@@ -20,7 +20,8 @@ export function addRunOptions(command) {
|
|
|
20
20
|
.option("--dotenv <path>", DOTENV_OPTION_DESCRIPTION)
|
|
21
21
|
.addOption(envFileAliasOption())
|
|
22
22
|
.option("--run-id <id>", "Explicit run id for deterministic fixture tests; refused when that run already exists.")
|
|
23
|
-
.
|
|
23
|
+
.addOption(new Option("--remote", "Require a remote run: the study's own process runs in a sandbox on its participants' provider, so it keeps going when this machine sleeps. A study that can is run this way by default; this refuses one that cannot, with the reason.").conflicts("local"))
|
|
24
|
+
.option("--local", "Run the study on this machine even when it could run remotely. It then stops if this machine sleeps.")
|
|
24
25
|
.addOption(ownerKeyOption())
|
|
25
26
|
.option("--json", JSON_OPTION_DESCRIPTION);
|
|
26
27
|
}
|
|
@@ -1,13 +1,45 @@
|
|
|
1
1
|
import type { Command } from "commander";
|
|
2
2
|
import { type RemoteOutcome } from "../../remote/launch.js";
|
|
3
|
+
import { type RemotePlan } from "../../remote/plan.js";
|
|
4
|
+
import type { RunResult } from "../../run/results.js";
|
|
3
5
|
import type { StudyConfig } from "../../study/types.js";
|
|
4
6
|
import { type CliIo, type StudyCommandOptions } from "../io.js";
|
|
7
|
+
type RemoteErrorCode = "HUMANISH_REMOTE_REFUSED" | "HUMANISH_REMOTE_FAILED";
|
|
8
|
+
/** Where a study's run executes, decided before anything is created. */
|
|
9
|
+
export type RunPlacement = {
|
|
10
|
+
kind: "local";
|
|
11
|
+
note?: string;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "remote";
|
|
14
|
+
plan: RemotePlan;
|
|
15
|
+
} | {
|
|
16
|
+
kind: "refused";
|
|
17
|
+
message: string;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Whether this run executes here or in a runner sandbox. A hosted study runs remotely by default
|
|
21
|
+
* when it can; `--local` or `execution.remote: false` keeps it here, and `--remote` or
|
|
22
|
+
* `execution.remote: true` requires a remote run, refused with the reason when the study cannot
|
|
23
|
+
* have one. `--local` and `--remote` win over the study's field.
|
|
24
|
+
*/
|
|
25
|
+
export declare function placeRun(args: {
|
|
26
|
+
options: StudyCommandOptions;
|
|
27
|
+
config: StudyConfig;
|
|
28
|
+
studyPath: string;
|
|
29
|
+
env?: Readonly<Record<string, string | undefined>>;
|
|
30
|
+
/** Whether this humanish is a built package a runner can install; read from the module when absent. */
|
|
31
|
+
packaged?: boolean;
|
|
32
|
+
}): RunPlacement;
|
|
33
|
+
/** A refused placement as the command's result. */
|
|
34
|
+
export declare function refuseRemote(command: Command, io: CliIo, cwd: string, message: string, code?: RemoteErrorCode, forwarded?: NonNullable<RunResult["remote"]>): void;
|
|
35
|
+
/** Launch `remote` in a runner sandbox and report how it ended. */
|
|
5
36
|
export declare function runRemoteStudy(args: {
|
|
6
37
|
command: Command;
|
|
7
38
|
io: CliIo;
|
|
8
39
|
options: StudyCommandOptions;
|
|
9
|
-
config: StudyConfig;
|
|
10
40
|
studyPath: string;
|
|
41
|
+
remote: RemotePlan;
|
|
11
42
|
}): Promise<void>;
|
|
12
43
|
/** The command's result and exit code for a remote run that ended, stopped or detached. */
|
|
13
|
-
export declare function reportRemoteOutcome(command: Command, io: CliIo, cwd: string, outcome: RemoteOutcome): void;
|
|
44
|
+
export declare function reportRemoteOutcome(command: Command, io: CliIo, cwd: string, outcome: RemoteOutcome, forwarded?: NonNullable<RunResult["remote"]>): void;
|
|
45
|
+
export {};
|