humanish 0.142.0 → 0.143.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -8,6 +8,16 @@ The Unreleased section holds the full notes for the next version until it is tag
8
8
 
9
9
  ## Unreleased
10
10
 
11
+ ## 0.143.0: Analysis costs about one seventh as much: gpt-6.1-sol at medium effort is the default (2026-10-22)
12
+
13
+ humanish 0.143.0 analyzes studies with `gpt-6.1-sol` at medium reasoning effort by default, in
14
+ place of `gpt-6-astra` at high. On the same recordings it found what `gpt-6-astra` found, invented
15
+ nothing, and cost about one seventh as much: $0.06 against $0.41 per analysis on the Taskly
16
+ benchmark. A study can set the analysis effort with `review.analysis.reasoningEffort`, and one
17
+ that names its model keeps it at high effort with an unchanged configuration hash.
18
+
19
+ [Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.143.0)
20
+
11
21
  ## 0.142.0: Remote runs, Gemini and GPT-6 analysis models, and a view-only live stream (2026-10-21)
12
22
 
13
23
  humanish 0.142.0 can run a study remotely: `humanish run <study> --remote` puts the study's own
package/README.md CHANGED
@@ -249,7 +249,7 @@ coverage limits, estimated cost controls, corrections and sharing behavior.
249
249
 
250
250
  Supported live studies generate findings automatically after participants finish.
251
251
  A Codex participant's run is analyzed by the same Codex account; every other run uses
252
- `gpt-6-astra` with high reasoning and a separate $10 cap on the expected cost.
252
+ `gpt-6.1-sol` at medium reasoning effort and a separate $10 cap on the expected cost.
253
253
  This is additional to participant execution costs, and is not a hard provider
254
254
  billing cap. To disable the extra request:
255
255
 
@@ -1,7 +1,14 @@
1
1
  import type { StudyRoute } from "../study/plan.js";
2
+ import { type ReasoningEffort } from "../actors/reasoning-effort.js";
2
3
  import type { AnalysisConfig } from "./types.js";
3
4
  export declare const DEFAULT_ANALYSIS_TIMEOUT_MS = 600000;
4
- export declare const DEFAULT_ANALYSIS_MODEL = "gpt-6-astra";
5
+ /** The OpenAI analyst a study or command gets when it names no model. */
6
+ export declare const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
7
+ /**
8
+ * The effort the default OpenAI analyst runs at when no effort is declared. A named model without
9
+ * an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
10
+ */
11
+ export declare const DEFAULT_ANALYSIS_EFFORT = "medium";
5
12
  /** The model a Gemini analysis uses when the study or command names none: Google's current
6
13
  * stable Flash model. */
7
14
  export declare const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
@@ -15,10 +22,12 @@ export type StudyAnalysis = StudyAnalysisSettings & ({
15
22
  provider?: "openai" | "gemini";
16
23
  maxCostUsd: number;
17
24
  maxOutputTokens?: number;
25
+ reasoningEffort?: ReasoningEffort;
18
26
  } | {
19
27
  provider: "codex";
20
28
  maxCostUsd?: null;
21
29
  maxOutputTokens?: null;
30
+ reasoningEffort?: never;
22
31
  });
23
32
  /** The spend cap on an OpenAI analysis the study did not declare. */
24
33
  export declare const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
@@ -43,6 +52,8 @@ export interface AutomaticAnalysisBudget {
43
52
  provider?: "openai" | "gemini" | "codex";
44
53
  billing?: "api-estimate" | "account-unknown";
45
54
  model: string;
55
+ /** The reasoning effort the analysis runs at. */
56
+ effort: ReasoningEffort;
46
57
  maxCostUsd: number | null;
47
58
  trigger: "default" | "explicit";
48
59
  /** The participants the expected cost range is for. */
@@ -1,12 +1,20 @@
1
1
  import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
2
- import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, isSupportedAnalysisModel, } from "./execute.js";
2
+ import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, } from "./execute.js";
3
+ import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
4
+ import { isReasoningEffort } from "../actors/reasoning-effort.js";
3
5
  import { admissionRule } from "./admission.js";
4
6
  import { analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
5
7
  import { plural } from "../run/text.js";
6
8
  import { scanEncodedText } from "../evidence/encoded-text.js";
7
9
  export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
8
10
  const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
9
- export const DEFAULT_ANALYSIS_MODEL = "gpt-6-astra";
11
+ /** The OpenAI analyst a study or command gets when it names no model. */
12
+ export const DEFAULT_ANALYSIS_MODEL = "gpt-6.1-sol";
13
+ /**
14
+ * The effort the default OpenAI analyst runs at when no effort is declared. A named model without
15
+ * an effort runs at high (models.ts), so a study that names its model keeps its configuration hash.
16
+ */
17
+ export const DEFAULT_ANALYSIS_EFFORT = "medium";
10
18
  /** The model a Gemini analysis uses when the study or command names none: Google's current
11
19
  * stable Flash model. */
12
20
  export const DEFAULT_GEMINI_ANALYSIS_MODEL = "gemini-3.8-flash";
@@ -17,6 +25,7 @@ const FIELDS = new Set([
17
25
  "question",
18
26
  "timeoutMs",
19
27
  "maxOutputTokens",
28
+ "reasoningEffort",
20
29
  ]);
21
30
  const validTimeout = (value) => typeof value === "number" &&
22
31
  Number.isSafeInteger(value) &&
@@ -59,7 +68,7 @@ export function resolveAutomaticAnalysis(raw, participant) {
59
68
  Object.keys(raw).some((key) => !FIELDS.has(key))) {
60
69
  return {
61
70
  ok: false,
62
- message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs and maxOutputTokens.",
71
+ message: "review.analysis must be false or a mapping containing only provider, maxCostUsd, model, question, timeoutMs, maxOutputTokens and reasoningEffort.",
63
72
  };
64
73
  }
65
74
  const value = raw;
@@ -70,6 +79,11 @@ export function resolveAutomaticAnalysis(raw, participant) {
70
79
  return { ok: false, message: "review.analysis.provider must be openai, gemini or codex." };
71
80
  }
72
81
  if (value.provider === "codex") {
82
+ if (value.reasoningEffort !== undefined)
83
+ return {
84
+ ok: false,
85
+ message: "Codex account analysis runs at its fixed low reasoning effort, so it takes no reasoningEffort. Omit it, or choose provider openai or gemini to set one.",
86
+ };
73
87
  const model = value.model === undefined ? CODEX_ANALYSIS_MODEL : value.model;
74
88
  const timeoutMs = value.timeoutMs === undefined ? DEFAULT_ANALYSIS_TIMEOUT_MS : value.timeoutMs;
75
89
  const question = value.question === undefined ? null : value.question;
@@ -127,18 +141,32 @@ export function resolveAutomaticAnalysis(raw, participant) {
127
141
  message: `review.analysis requires a positive maxCostUsd up to 1000, a supported ${vendor === "gemini" ? "Gemini" : "OpenAI"} analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
128
142
  };
129
143
  }
144
+ // Only the default model gets the default effort, written out because an omitted effort is high.
145
+ const reasoningEffort = value.reasoningEffort === undefined && value.model === undefined && vendor === "openai"
146
+ ? DEFAULT_ANALYSIS_EFFORT
147
+ : value.reasoningEffort;
148
+ const efforts = analysisEfforts(model, vendor);
149
+ if (reasoningEffort !== undefined &&
150
+ (!isReasoningEffort(reasoningEffort) || !efforts.includes(reasoningEffort)))
151
+ return {
152
+ ok: false,
153
+ message: `review.analysis.reasoningEffort for ${model} must be ${efforts.slice(0, -1).join(", ")} or ${efforts.at(-1)}, the efforts its provider documents for it.`,
154
+ };
130
155
  if (question !== null && scanEncodedText(question).sensitive)
131
156
  return SENSITIVE_QUESTION;
132
157
  return {
133
158
  ok: true,
134
159
  config: {
135
- // An omitted provider stays omitted: it is the OpenAI contract, and its hash predates the field.
160
+ // An omitted provider, or an omitted effort on a named model, stays omitted, so the
161
+ // configuration keeps the hash it had before the field existed. An omitted provider is the
162
+ // OpenAI contract.
136
163
  ...(value.provider === undefined ? {} : { provider: vendor }),
137
164
  model,
138
165
  maxCostUsd,
139
166
  timeoutMs,
140
167
  maxOutputTokens,
141
168
  question: question,
169
+ ...(reasoningEffort === undefined ? {} : { reasoningEffort }),
142
170
  },
143
171
  ...(value.maxOutputTokens === undefined ? { preferLargerOutput: true } : {}),
144
172
  };
@@ -164,6 +192,7 @@ export function automaticAnalysisBudget(raw, route, participants, participant) {
164
192
  ? { provider: "gemini" }
165
193
  : {}),
166
194
  model: config.model,
195
+ effort: analysisEffort(config),
167
196
  maxCostUsd: config.maxCostUsd,
168
197
  trigger: raw === undefined ? "default" : "explicit",
169
198
  ...(range === undefined || participants === undefined ? {} : { participants, ...range }),
@@ -180,7 +209,7 @@ export function analysisRequestsText(requests) {
180
209
  }
181
210
  export function formatAutomaticAnalysisBudget(budget) {
182
211
  if (budget.provider === "codex")
183
- return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
212
+ return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} at ${budget.effort} effort · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
184
213
  const rule = admissionRule(`$${budget.maxCostUsd}`);
185
214
  // The participants, and the requests that analyse them when there is more than one.
186
215
  const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
@@ -194,5 +223,5 @@ export function formatAutomaticAnalysisBudget(budget) {
194
223
  ? `refused before it starts if ${rule}`
195
224
  : `refused before it starts for ${who} even with no evidence (expected $${budget.refusedFromUsd.toFixed(2)}), since ${rule}`;
196
225
  const where = budget.provider === "gemini" ? " on Gemini, which sends the evidence to Google" : "";
197
- return `After live runs: ${budget.trigger} analysis${where} · ${budget.model}${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
226
+ return `After live runs: ${budget.trigger} analysis${where} · ${budget.model} at ${budget.effort} effort${range} · ${refused}; this is not a billing cap. Set review.analysis: false to disable.`;
198
227
  }
@@ -1,17 +1,7 @@
1
1
  import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
2
- import { type AnalysisApiVendor, type AnalysisFetch, type AnalysisProvider } from "./provider.js";
2
+ import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
3
3
  import type { RejectedAnalysisOutput } from "./diagnostics.js";
4
4
  export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
5
- export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
6
- /**
7
- * Gemini models that completed a live analysis. Each model's page under
8
- * ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
9
- * output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
10
- * once a live analysis on each completes.
11
- */
12
- export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
13
- /** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
14
- export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
15
5
  /** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
16
6
  * returns at most 32,768 output tokens from one generateContent request. */
17
7
  export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
@@ -6,6 +6,7 @@ import { modelRate } from "../run/pricing.js";
6
6
  import { containsSensitive } from "../evidence/redaction.js";
7
7
  import { scanEncodedText } from "../evidence/encoded-text.js";
8
8
  import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
9
+ import { analysisEffort, analysisEfforts, isSupportedAnalysisModel } from "./models.js";
9
10
  import { createAnalysisProvider, } from "./provider.js";
10
11
  import { hashAnalysisValue, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
11
12
  import { analysisCohorts, analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
@@ -13,27 +14,6 @@ import { checkMergedResponse, cohortInputs, mergePacket, mergeResultJsonSchema,
13
14
  import { checkProviderAnalysis, codexWarnings, httpStatusWarning, recordProviderUsage, } from "./responses.js";
14
15
  import { mapWithConcurrency } from "../run/concurrency.js";
15
16
  export const ANALYSIS_PROMPT_VERSION = "study-evidence-8";
16
- export const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
17
- "gpt-6-astra",
18
- "gpt-6.1-sol",
19
- "gpt-6-luna",
20
- "gpt-5.5",
21
- "gpt-5.6",
22
- "gpt-5.6-sol",
23
- "gpt-5.6-terra",
24
- "gpt-5.6-luna",
25
- ]);
26
- const SUPPORTED_MODELS = new Set(SUPPORTED_ANALYSIS_MODELS);
27
- /**
28
- * Gemini models that completed a live analysis. Each model's page under
29
- * ai.google.dev/gemini-api/docs/models (read 2026-10-21) also lists image input and JSON-schema
30
- * output for gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this list
31
- * once a live analysis on each completes.
32
- */
33
- export const GEMINI_ANALYSIS_MODELS = Object.freeze(["gemini-3.8-flash"]);
34
- const GEMINI_MODELS = new Set(GEMINI_ANALYSIS_MODELS);
35
- /** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
36
- export const isSupportedAnalysisModel = (model, provider = "openai") => (provider === "gemini" ? GEMINI_MODELS : SUPPORTED_MODELS).has(model);
37
17
  /** The API output limit when a study omits maxOutputTokens, and the most it may ask for. Gemini 4
38
18
  * returns at most 32,768 output tokens from one generateContent request. */
39
19
  export const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16_384;
@@ -192,6 +172,8 @@ export function estimateAnalysisAdmission(input, config) {
192
172
  config.provider !== "openai" &&
193
173
  config.provider !== "gemini") ||
194
174
  !isSupportedAnalysisModel(config.model, config.provider) ||
175
+ (config.reasoningEffort !== undefined &&
176
+ !analysisEfforts(config.model, config.provider).includes(config.reasoningEffort)) ||
195
177
  !Number.isFinite(config.maxCostUsd) ||
196
178
  config.maxCostUsd <= 0 ||
197
179
  config.maxCostUsd > 1000 ||
@@ -547,6 +529,7 @@ export async function runAnalysis(input, config, options) {
547
529
  images: request.images,
548
530
  schema: request.schema,
549
531
  maxOutputTokens: config.maxOutputTokens,
532
+ reasoningEffort: analysisEffort(config),
550
533
  timeoutMs: config.timeoutMs,
551
534
  ...(options.signal === undefined ? {} : { signal: options.signal }),
552
535
  });
@@ -1,6 +1,7 @@
1
1
  import type { AnalysisWire } from "./provider.js";
2
2
  /**
3
- * One stateless request: no tools, high thinking, JSON constrained to the schema, the standard
3
+ * One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
4
+ * ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
4
5
  * service tier, and `store: false` so the request stays out of the project's request logs. Each
5
6
  * capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
6
7
  * image analysis, after a text part naming its evidence key.
@@ -116,7 +116,8 @@ function parseGenerateContent(raw) {
116
116
  }
117
117
  }
118
118
  /**
119
- * One stateless request: no tools, high thinking, JSON constrained to the schema, the standard
119
+ * One stateless request: no tools, the analysis's reasoning effort as its thinking level (the
120
+ * ThinkingLevel enum spells the same names in capitals), JSON constrained to the schema, the standard
120
121
  * service tier, and `store: false` so the request stays out of the project's request logs. Each
121
122
  * capture goes at MEDIA_RESOLUTION_HIGH, the setting the media-resolution guide recommends for
122
123
  * image analysis, after a text part naming its evidence key.
@@ -152,7 +153,7 @@ export const geminiWire = {
152
153
  generationConfig: {
153
154
  candidateCount: 1,
154
155
  maxOutputTokens: request.maxOutputTokens,
155
- thinkingConfig: { thinkingLevel: "HIGH" },
156
+ thinkingConfig: { thinkingLevel: request.reasoningEffort.toUpperCase() },
156
157
  responseFormat: {
157
158
  text: { mimeType: "APPLICATION_JSON", schema },
158
159
  },
@@ -0,0 +1,12 @@
1
+ import type { ReasoningEffort } from "../actors/reasoning-effort.js";
2
+ import type { AnalysisApiVendor } from "./provider.js";
3
+ import type { AnalysisConfig } from "./types.js";
4
+ export declare const SUPPORTED_ANALYSIS_MODELS: readonly string[];
5
+ export declare const GEMINI_ANALYSIS_MODELS: readonly string[];
6
+ /** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
7
+ export declare const isSupportedAnalysisModel: (model: string, provider?: AnalysisApiVendor) => boolean;
8
+ /** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
9
+ export declare const analysisEfforts: (model: string, provider?: AnalysisApiVendor) => readonly ReasoningEffort[];
10
+ /** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
11
+ * and the Codex account's fixed effort. */
12
+ export declare function analysisEffort(config: AnalysisConfig): ReasoningEffort;
@@ -0,0 +1,47 @@
1
+ // The models each API analyst runs, and the reasoning efforts each model accepts. Config parsing,
2
+ // admission, the CLI help and the wire formats all read this one table.
3
+ /**
4
+ * `reasoning.effort` values from each model's page under developers.openai.com/api/docs/models
5
+ * (read 2026-10-21). None of them lists `minimal`; gpt-6-astra and gpt-6.1-sol also refuse `none`.
6
+ */
7
+ const OPENAI_MODELS = {
8
+ "gpt-6-astra": ["low", "medium", "high", "xhigh", "max"],
9
+ "gpt-6.1-sol": ["low", "medium", "high", "xhigh", "max"],
10
+ "gpt-6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
11
+ "gpt-5.5": ["none", "low", "medium", "high", "xhigh"],
12
+ "gpt-5.6": ["none", "low", "medium", "high", "xhigh", "max"],
13
+ "gpt-5.6-sol": ["none", "low", "medium", "high", "xhigh", "max"],
14
+ "gpt-5.6-terra": ["none", "low", "medium", "high", "xhigh", "max"],
15
+ "gpt-5.6-luna": ["none", "low", "medium", "high", "xhigh", "max"],
16
+ };
17
+ /**
18
+ * Gemini models that completed a live analysis, with the `thinkingLevel` values the thinking guide
19
+ * lists for each (ai.google.dev/gemini-api/docs/generate-content/thinking, read 2026-10-21):
20
+ * gemini-3.8-flash answers `minimal` with an error. Each model's page under
21
+ * ai.google.dev/gemini-api/docs/models also lists image input and JSON-schema output for
22
+ * gemini-3.5-flash-lite, gemini-3-flash-preview and gemini-4-argon; they join this table once a
23
+ * live analysis on each completes.
24
+ */
25
+ const GEMINI_MODELS = {
26
+ "gemini-3.8-flash": ["low", "medium", "high"],
27
+ };
28
+ const MODELS = { openai: OPENAI_MODELS, gemini: GEMINI_MODELS };
29
+ export const SUPPORTED_ANALYSIS_MODELS = Object.freeze(Object.keys(OPENAI_MODELS));
30
+ export const GEMINI_ANALYSIS_MODELS = Object.freeze(Object.keys(GEMINI_MODELS));
31
+ /** Whether the API analyst `provider` (omitted is OpenAI) analyzes with `model`. */
32
+ export const isSupportedAnalysisModel = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model);
33
+ /** The reasoning efforts `model` accepts from `provider`, least first; empty for an unknown model. */
34
+ export const analysisEfforts = (model, provider = "openai") => Object.hasOwn(MODELS[provider], model) ? MODELS[provider][model] : [];
35
+ /**
36
+ * The effort an API analysis runs at when its configuration declares none. Every saved analysis
37
+ * without `reasoningEffort` ran at this effort, so it stays high: a different default must be
38
+ * written into the configuration it resolves, where it also changes the configuration's hash.
39
+ */
40
+ const UNDECLARED_EFFORT = "high";
41
+ /** The effort an analysis runs at: the declared one, high for an API analysis that declares none,
42
+ * and the Codex account's fixed effort. */
43
+ export function analysisEffort(config) {
44
+ if (config.provider === "codex")
45
+ return config.identity.reasoningEffort;
46
+ return config.reasoningEffort ?? UNDECLARED_EFFORT;
47
+ }
@@ -1,5 +1,6 @@
1
1
  import type { AnalysisProviderResult, AnalysisWire } from "./provider.js";
2
2
  /** Text and usage wire shapes reuse the captured closing-report contract; refusal fails closed. */
3
3
  export declare function parseAnalysisResponse(raw: unknown): AnalysisProviderResult;
4
- /** One stateless request: no stored response, no tools, high reasoning effort, strict schema. */
4
+ /** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
5
+ * schema. */
5
6
  export declare const openaiWire: AnalysisWire;
@@ -67,7 +67,8 @@ export function parseAnalysisResponse(raw) {
67
67
  return { ...base, status: "failed", errorCode: "invalid_response" };
68
68
  }
69
69
  }
70
- /** One stateless request: no stored response, no tools, high reasoning effort, strict schema. */
70
+ /** One stateless request: no stored response, no tools, the analysis's reasoning effort, strict
71
+ * schema. */
71
72
  export const openaiWire = {
72
73
  url: () => OPENAI_RESPONSES_URL,
73
74
  headers: (apiKey) => ({ Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }),
@@ -91,7 +92,7 @@ export const openaiWire = {
91
92
  tool_choice: "none",
92
93
  truncation: "disabled",
93
94
  service_tier: "default",
94
- reasoning: { effort: "high" },
95
+ reasoning: { effort: request.reasoningEffort },
95
96
  max_output_tokens: request.maxOutputTokens,
96
97
  text: {
97
98
  format: { type: "json_schema", name: "study_analysis", strict: true, schema: request.schema },
@@ -1,4 +1,5 @@
1
1
  import { type Dispatcher } from "undici";
2
+ import type { ReasoningEffort } from "../actors/reasoning-effort.js";
2
3
  /** Deliberately separate from the stateful computer-use actor: one request, no tools or retries. */
3
4
  export interface AnalysisProviderRequest {
4
5
  model: string;
@@ -10,6 +11,9 @@ export interface AnalysisProviderRequest {
10
11
  }[];
11
12
  schema: Record<string, unknown>;
12
13
  maxOutputTokens: number | null;
14
+ /** The effort the analysis runs at (models.ts analysisEffort). The Codex account analyst runs
15
+ * its fixed low effort, which this then names. */
16
+ reasoningEffort: ReasoningEffort;
13
17
  timeoutMs: number;
14
18
  signal?: AbortSignal;
15
19
  }
@@ -1,5 +1,6 @@
1
1
  import type { ActorStatus, ActorCompletionReason, ActorStopCause, ParticipantDeclaredOutcome } from "../actors/contract.js";
2
2
  import type { CuaEndpointSource } from "../actors/goal-source.js";
3
+ import type { ReasoningEffort } from "../actors/reasoning-effort.js";
3
4
  import type { AutomaticAnalysisView } from "./job.js";
4
5
  import type { RunAnalysisCost } from "../run/run-cost.js";
5
6
  /** Independent interpretation of retained evidence; never a participant or harness verdict. */
@@ -160,6 +161,9 @@ interface OpenAIAnalysisConfig {
160
161
  maxCostUsd: number;
161
162
  timeoutMs: number;
162
163
  maxOutputTokens: number;
164
+ /** Present only when the study or command declared it, so an omitted effort keeps the hash of
165
+ * every configuration before the field. An analysis without it ran at high (models.ts). */
166
+ reasoningEffort?: ReasoningEffort | undefined;
163
167
  }
164
168
  /** The Gemini API analyst: the same billed, capped request as OpenAI's, sent to Google. */
165
169
  interface GeminiAnalysisConfig extends Omit<OpenAIAnalysisConfig, "provider"> {
@@ -3,6 +3,7 @@ import { createHash } from "node:crypto";
3
3
  import { z } from "zod";
4
4
  import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
5
5
  import { CUA_ENDPOINT_SOURCES } from "../actors/goal-source.js";
6
+ import { REASONING_EFFORTS } from "../actors/reasoning-effort.js";
6
7
  import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
7
8
  import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
8
9
  const text = (max) => z
@@ -204,6 +205,7 @@ const apiAnalysisConfigSchema = z.object({
204
205
  maxCostUsd: z.number().positive().max(1000),
205
206
  timeoutMs: z.number().int().positive().max(3600000),
206
207
  maxOutputTokens: z.number().int().positive().max(128000),
208
+ reasoningEffort: z.enum(REASONING_EFFORTS).optional(),
207
209
  });
208
210
  const analysisArtifactSchema = z
209
211
  .object({
@@ -361,8 +363,7 @@ export function checkAnalysisResult(input, value) {
361
363
  if (!distinct(observation.evidenceIds) ||
362
364
  refs.some((ref) => !ref || !included.has(ref.streamId)))
363
365
  errors.add("ANALYSIS_OBSERVATION_REFERENCE_INVALID");
364
- if (observation.basis === "visual" &&
365
- !refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
366
+ if (observation.basis === "visual" && !refs.some((ref) => ref?.capture))
366
367
  errors.add("ANALYSIS_VISUAL_WITHOUT_CAPTURE");
367
368
  if (observation.basis === "action" &&
368
369
  !refs.some((ref) => ref &&
@@ -394,7 +395,7 @@ export function checkAnalysisResult(input, value) {
394
395
  const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
395
396
  if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
396
397
  errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
397
- if (!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
398
+ if (!refs.some((ref) => ref?.capture))
398
399
  errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
399
400
  // Seen by means a cited capture of that participant shows the problem.
400
401
  const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
@@ -1,9 +1,9 @@
1
- import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
1
+ import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, DEFAULT_GEMINI_ANALYSIS_MODEL, } from "../../analysis/automatic-config.js";
2
2
  import { Command, Option } from "commander";
3
3
  import { forTerminal } from "../../routes/terminal/encoding.js";
4
4
  import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
5
5
  import { admissionRule } from "../../analysis/admission.js";
6
- import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS, } from "../../analysis/execute.js";
6
+ import { GEMINI_ANALYSIS_MODELS, SUPPORTED_ANALYSIS_MODELS } from "../../analysis/models.js";
7
7
  import { CODEX_ANALYSIS_MODEL } from "../../analysis/codex-config.js";
8
8
  import { listAnalyses } from "../../analysis/store.js";
9
9
  import { listAnalysisExecutions } from "../../analysis/store-executions.js";
@@ -34,7 +34,8 @@ export function registerAnalyzeCommand(parent, io) {
34
34
  .option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
35
35
  .option("--provider <name>", "Analyst: openai (default), gemini (sends the evidence and captures to Google), or restricted codex account. No provider fallback.")
36
36
  .option("--max-cost <usd>", `Required for OpenAI and Gemini, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
37
- .option("--model <id>", `OpenAI analysis model, at high effort: ${SUPPORTED_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_ANALYSIS_MODEL} when omitted. Gemini analysis model, at high thinking: ${GEMINI_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_GEMINI_ANALYSIS_MODEL} when omitted. Codex account analysis runs ${CODEX_ANALYSIS_MODEL} at low effort.`)
37
+ .option("--model <id>", `OpenAI analysis model: ${SUPPORTED_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_ANALYSIS_MODEL} at ${DEFAULT_ANALYSIS_EFFORT} effort when omitted. Gemini analysis model: ${GEMINI_ANALYSIS_MODELS.join(", ")}; ${DEFAULT_GEMINI_ANALYSIS_MODEL} when omitted. Codex account analysis runs ${CODEX_ANALYSIS_MODEL} at low effort.`)
38
+ .option("--reasoning-effort <level>", `OpenAI reasoning.effort or Gemini thinking level, from none, minimal, low, medium, high, xhigh and max; each model takes only the levels its provider documents for it, and others are refused before any request. When omitted: ${DEFAULT_ANALYSIS_EFFORT} for the default OpenAI analyst, high with --model or --provider gemini. Unsupported for Codex.`)
38
39
  .option("--question <text>", "Additional reviewer question; does not change participant instructions.")
39
40
  .option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
40
41
  .option("--max-output-tokens <n>", "OpenAI or Gemini response-token limit including reasoning, 256–32768. Omit for admission-based sizing. Unsupported for Codex.")
@@ -84,6 +85,7 @@ async function handleAnalyze(io, options, command) {
84
85
  ...(options.maxOutputTokens === undefined
85
86
  ? {}
86
87
  : { maxOutputTokens: Number(options.maxOutputTokens) }),
88
+ ...(options.reasoningEffort === undefined ? {} : { reasoningEffort: options.reasoningEffort }),
87
89
  });
88
90
  if (!selected.ok || !selected.config) {
89
91
  const refusal = await dryRunBundleRefusal(await resolvePhysicalCwd(options.cwd), options.run, options.dryRun === true);
@@ -158,10 +158,11 @@ interface LiveStudy {
158
158
  }
159
159
  /** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
160
160
  export declare function cwdFlag(cwd: string): string;
161
- /** An analysis provider and model, as a study or an analysis record names them. */
161
+ /** An analysis provider, model and effort, as a study or an analysis record names them. */
162
162
  interface Analyst {
163
163
  provider?: string | undefined;
164
164
  model?: string;
165
+ reasoningEffort?: string | undefined;
165
166
  }
166
167
  /** The findings view of a run's selected analysis, or why it has none. */
167
168
  export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
@@ -5,7 +5,7 @@
5
5
  // starts an analysis.
6
6
  import path from "node:path";
7
7
  import { costRefusal } from "../analysis/admission.js";
8
- import { DEFAULT_ANALYSIS_MAX_COST_USD, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
8
+ import { DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MAX_COST_USD, DEFAULT_ANALYSIS_MODEL, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
9
9
  import { ANALYSIS_USUALLY_TAKES } from "../analysis/analysis-limits.js";
10
10
  import { bySeverity } from "../analysis/finding-lead.js";
11
11
  import { loadAnalysis } from "../analysis/load.js";
@@ -35,15 +35,25 @@ const KEY_MISSING_REASONS = new Set([
35
35
  "AUTOMATIC_ANALYSIS_KEY_MISSING",
36
36
  "AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING",
37
37
  ]);
38
+ /** The effort an analysis configuration declares; Codex configurations declare none. */
39
+ const declaredEffort = (config) => "reasoningEffort" in config ? config.reasoningEffort : undefined;
38
40
  /** The `analyze` command that runs `analyst` on the run; undefined is the OpenAI default. */
39
41
  function analyzeCommand(source, analyst) {
40
42
  const run = cli(`analyze --run ${source.runId}${source.cwdFlag}`);
41
43
  if (analyst?.provider === "codex")
42
44
  return `${run} --provider codex`;
43
- const gemini = analyst?.provider === "gemini"
44
- ? ` --provider gemini${analyst.model === undefined ? "" : ` --model ${analyst.model}`}`
45
- : "";
46
- return `${run}${gemini} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
45
+ if (analyst?.provider === "gemini") {
46
+ const model = analyst.model === undefined ? "" : ` --model ${analyst.model}`;
47
+ return `${run} --provider gemini${model} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
48
+ }
49
+ // A named model or effort stays in the command, so the rerun is the same analyst. The default
50
+ // analyst needs neither: `analyze` without flags runs it.
51
+ const isDefault = (analyst?.model ?? DEFAULT_ANALYSIS_MODEL) === DEFAULT_ANALYSIS_MODEL &&
52
+ (analyst?.model === undefined || analyst.reasoningEffort === DEFAULT_ANALYSIS_EFFORT);
53
+ const flags = isDefault
54
+ ? ""
55
+ : ` --model ${analyst.model}${analyst.reasoningEffort === undefined ? "" : ` --reasoning-effort ${analyst.reasoningEffort}`}`;
56
+ return `${run}${flags} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
47
57
  }
48
58
  /** The cited items the analysis evidence resolves, each once, in citation order. */
49
59
  function citedEvidence(ids, analysis, runRoot) {
@@ -180,7 +190,7 @@ function missingKeyFindings(source, reason) {
180
190
  }));
181
191
  const opening = "The automatic analysis was skipped because OPENAI_API_KEY is not set.";
182
192
  if (source.codexSignedIn !== true)
183
- return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, undefined));
193
+ return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, source.analyst?.provider === "gemini" ? undefined : source.analyst));
184
194
  const file = source.study?.path === undefined ? "the study file" : source.study.path;
185
195
  return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, { provider: "codex" }));
186
196
  }
@@ -234,7 +244,11 @@ async function studyAnalyst(cwd, study) {
234
244
  return undefined;
235
245
  const analysis = resolveAutomaticAnalysis(resolved.config.review?.analysis, resolved.config.actor);
236
246
  return analysis.ok && analysis.config !== undefined
237
- ? { provider: analysis.config.provider, model: analysis.config.model }
247
+ ? {
248
+ provider: analysis.config.provider,
249
+ model: analysis.config.model,
250
+ reasoningEffort: declaredEffort(analysis.config),
251
+ }
238
252
  : undefined;
239
253
  }
240
254
  /** The findings view of a run's selected analysis, or why it has none. */
@@ -255,7 +269,11 @@ export function analysisFindings(source) {
255
269
  ? `The run's evidence changed after analysis ${analysis.id}, so its findings may not match the run.`
256
270
  : `Analysis ${analysis.id} is ${analysis.status}, with ${plural(result.findings.length, "finding")}.`,
257
271
  next: stale
258
- ? analyzeCommand(source, { provider: analysis.provider, model: analysis.config.model })
272
+ ? analyzeCommand(source, {
273
+ provider: analysis.provider,
274
+ model: analysis.config.model,
275
+ reasoningEffort: declaredEffort(analysis.config),
276
+ })
259
277
  : null,
260
278
  analysisId: analysis.id,
261
279
  status: analysis.status,
@@ -279,7 +297,11 @@ export function analysisFindings(source) {
279
297
  if (pending !== undefined)
280
298
  return pending;
281
299
  if (analysis && analysis.result === null)
282
- return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, { provider: analysis.provider, model: analysis.config.model }));
300
+ return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, {
301
+ provider: analysis.provider,
302
+ model: analysis.config.model,
303
+ reasoningEffort: declaredEffort(analysis.config),
304
+ }));
283
305
  if (job?.state === "failed" || job?.state === "cancelled") {
284
306
  const outcome = analysisOutcomeText(job);
285
307
  return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? { provider: "codex" } : undefined));
@@ -38,7 +38,10 @@ export declare function followRemoteRun(args: {
38
38
  signals: Signals;
39
39
  proxyUrl: string;
40
40
  }): Promise<RemoteOutcome>;
41
- /** Two sandbox journals as one: every line of each, in order, one per line. */
41
+ /**
42
+ * Two sandbox journals as one: every line of each, in order, once. A copy that runs again adds
43
+ * nothing it already added.
44
+ */
42
45
  export declare function joinJournals(...journals: string[]): string;
43
46
  export interface Signals {
44
47
  readonly requested: "stop" | "detach" | undefined;
@@ -2,7 +2,7 @@
2
2
  // participants' provider, follow it from here, and copy the finished run back. This machine only
3
3
  // plans, launches and watches, so it may sleep or lose its network while the study goes on.
4
4
  import { execFile } from "node:child_process";
5
- import { randomBytes } from "node:crypto";
5
+ import { createHash, randomBytes } from "node:crypto";
6
6
  import { mkdtemp, readFile, rm } from "node:fs/promises";
7
7
  import { tmpdir } from "node:os";
8
8
  import path from "node:path";
@@ -18,13 +18,19 @@ import { e2bConnection } from "../substrates/e2b/connection.js";
18
18
  import { acquireE2BShellSandbox } from "../substrates/e2b/sandbox.js";
19
19
  import { loadModalShellModule } from "../substrates/modal/sandbox.js";
20
20
  import { serveObserverProxy } from "./observer-proxy.js";
21
- import { installHumanish, listRemoteRun, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
21
+ import { installHumanish, listRemoteRun, listSettledEvidence, readRemoteBytes, readRemoteFile, readRemoteRun, RUNNER_OBSERVER_PORT, runnerOf, startRemoteRun, stopRemoteRun, uploadProject, } from "./runner.js";
22
22
  /** How often the launcher reads the runner's run while it follows it. */
23
23
  const POLL_MS = 5_000;
24
24
  /** Reads in a row the runner may leave unanswered before the launcher gives it up. */
25
25
  const UNANSWERED_LIMIT = 24;
26
26
  /** How long the run has to write its status record after it is started. */
27
27
  const START_DEADLINE_MS = 3 * 60_000;
28
+ /** How often the launcher copies the run's settled evidence while it goes on. */
29
+ const SYNC_EVERY_MS = 30_000;
30
+ /** How long a screenshot or recording must be unchanged before it is copied mid-run. */
31
+ const SETTLED_SECONDS = 10;
32
+ /** The largest file read back to compare digests: a recording's own limit, with room. */
33
+ const LARGEST_FILE_BYTES = 256 * 1024 * 1024;
28
34
  /** How long a stopped run has to end and write its result. */
29
35
  const STOP_DEADLINE_MS = 2 * 60_000;
30
36
  /** Launch, follow and copy back one remote run. */
@@ -117,7 +123,15 @@ async function acquireRunner(launch, paths) {
117
123
  export async function followRemoteRun(args) {
118
124
  const { launch, runner, paths, runId, signals } = args;
119
125
  const { say } = launch;
120
- const copy = () => copyRunBack(runner, paths, runId);
126
+ // Evidence already copied while the run went on, by path, with its size then.
127
+ const copied = new Map();
128
+ const copy = async () => {
129
+ const { files, alreadyHere } = await copyRunBack(runner, paths, runId);
130
+ if (alreadyHere > 0)
131
+ say(`${alreadyHere} of its ${files} files were already here.`);
132
+ return files;
133
+ };
134
+ let syncAt = Date.now() + SYNC_EVERY_MS;
121
135
  let state;
122
136
  let stopAt;
123
137
  let unanswered = 0;
@@ -188,6 +202,11 @@ export async function followRemoteRun(args) {
188
202
  ...("files" in copied ? { files: copied.files } : {}),
189
203
  });
190
204
  }
205
+ if (state === "running" && Date.now() >= syncAt) {
206
+ syncAt = Date.now() + SYNC_EVERY_MS;
207
+ // Best effort: what this misses, the copy at the end takes.
208
+ await copySettledEvidence(runner, paths, runId, copied).catch(() => undefined);
209
+ }
191
210
  await new Promise((resolve) => setTimeout(resolve, POLL_MS));
192
211
  }
193
212
  }
@@ -198,9 +217,16 @@ export async function followRemoteRun(args) {
198
217
  */
199
218
  async function copyRunBack(runner, paths, runId) {
200
219
  const files = await listRemoteRun(runner, runId);
220
+ let alreadyHere = 0;
201
221
  for (const file of files) {
202
222
  if (file.path === "owner-key")
203
223
  continue;
224
+ // Already here with the same bytes, copied while the run went on or by an earlier attempt.
225
+ if (file.path !== SANDBOX_RECEIPTS_ARTIFACT &&
226
+ (await localDigest(paths, file.path)) === file.sha256) {
227
+ alreadyHere += 1;
228
+ continue;
229
+ }
204
230
  const bytes = await readRemoteFile(runner, runId, file);
205
231
  if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
206
232
  const local = await readContainedRegularFile(paths, file.path, 16 * 1024 * 1024);
@@ -211,13 +237,32 @@ async function copyRunBack(runner, paths, runId) {
211
237
  await writeContainedOutputFile(paths, file.path, bytes);
212
238
  }
213
239
  await pointLatestRunAt(paths, runId);
214
- return files.length;
240
+ return { files: files.length, alreadyHere };
241
+ }
242
+ /**
243
+ * Copy the run's screenshots and recordings that have settled and are not here yet, while the run
244
+ * goes on, so the copy at its end is only what is left.
245
+ */
246
+ async function copySettledEvidence(runner, paths, runId, copied) {
247
+ for (const file of await listSettledEvidence(runner, runId, SETTLED_SECONDS)) {
248
+ if (copied.get(file.path) === file.size)
249
+ continue;
250
+ await writeContainedOutputFile(paths, file.path, await readRemoteBytes(runner, runId, file.path));
251
+ copied.set(file.path, file.size);
252
+ }
215
253
  }
216
- /** Two sandbox journals as one: every line of each, in order, one per line. */
254
+ async function localDigest(paths, file) {
255
+ const read = await readContainedRegularFile(paths, file, LARGEST_FILE_BYTES);
256
+ return read.status === "read" ? createHash("sha256").update(read.bytes).digest("hex") : undefined;
257
+ }
258
+ /**
259
+ * Two sandbox journals as one: every line of each, in order, once. A copy that runs again adds
260
+ * nothing it already added.
261
+ */
217
262
  export function joinJournals(...journals) {
218
- const lines = journals
219
- .flatMap((journal) => journal.split("\n"))
220
- .filter((line) => line.trim() !== "");
263
+ const lines = [
264
+ ...new Set(journals.flatMap((journal) => journal.split("\n")).filter((line) => line.trim() !== "")),
265
+ ];
221
266
  return lines.length === 0 ? "" : `${lines.join("\n")}\n`;
222
267
  }
223
268
  /**
@@ -59,6 +59,22 @@ export interface RemoteFile {
59
59
  export declare function listRemoteRun(runner: Runner, runId: string): Promise<RemoteFile[]>;
60
60
  /** sha256sum lines, `<hex> ./<path>`, as RemoteFiles with paths relative to the directory. */
61
61
  export declare function parseDigests(output: string): RemoteFile[];
62
+ /**
63
+ * The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
64
+ * with their sizes. A run writes each of them once, so one that has settled can be copied while
65
+ * the run goes on; files whose name starts with a dot are still being written.
66
+ */
67
+ export declare function listSettledEvidence(runner: Runner, runId: string, settledSeconds: number): Promise<{
68
+ path: string;
69
+ size: number;
70
+ }[]>;
71
+ /** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
72
+ export declare function parseSizes(output: string): {
73
+ path: string;
74
+ size: number;
75
+ }[];
76
+ /** The bytes of one file of the runner's run directory, as they are now. */
77
+ export declare function readRemoteBytes(runner: Runner, runId: string, file: string): Promise<Uint8Array>;
62
78
  /** The bytes of one file of the runner's run directory, checked against its listed digest. */
63
79
  export declare function readRemoteFile(runner: Runner, runId: string, file: RemoteFile): Promise<Uint8Array>;
64
80
  /**
@@ -189,16 +189,41 @@ export function parseDigests(output) {
189
189
  }
190
190
  return files;
191
191
  }
192
- /** The bytes of one file of the runner's run directory, checked against its listed digest. */
193
- export async function readRemoteFile(runner, runId, file) {
192
+ /**
193
+ * The screenshots and recordings of the runner's run that have not changed for `settledSeconds`,
194
+ * with their sizes. A run writes each of them once, so one that has settled can be copied while
195
+ * the run goes on; files whose name starts with a dot are still being written.
196
+ */
197
+ export async function listSettledEvidence(runner, runId, settledSeconds) {
198
+ const dir = `${RUNNER_PROJECT}/.humanish/runs/${runId}`;
199
+ const listed = await runner.shell.run(`cd ${shellQuote(dir)} 2>/dev/null && find screenshots recordings -type f ! -name '.*' ! -newermt '${settledSeconds} seconds ago' -printf '%p\\t%s\\n' 2>/dev/null`, { timeoutMs: 120_000 });
200
+ return parseSizes(listed.stdout);
201
+ }
202
+ /** `find -printf '%p\t%s\n'` lines as paths and sizes; a path that leaves the directory is dropped. */
203
+ export function parseSizes(output) {
204
+ const files = [];
205
+ for (const line of output.split("\n")) {
206
+ const match = /^((?:screenshots|recordings)\/[^\t]+)\t(\d+)$/.exec(line);
207
+ if (match === null || match[1].split("/").includes(".."))
208
+ continue;
209
+ files.push({ path: match[1], size: Number(match[2]) });
210
+ }
211
+ return files;
212
+ }
213
+ /** The bytes of one file of the runner's run directory, as they are now. */
214
+ export async function readRemoteBytes(runner, runId, file) {
194
215
  const read = runner.sandbox.files.read;
195
216
  if (read === undefined)
196
217
  throw new Error("This sandbox cannot read files back.");
197
- const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file.path}`, { format: "stream", requestTimeoutMs: 300_000 });
218
+ const stream = await read.call(runner.sandbox.files, `${RUNNER_PROJECT}/.humanish/runs/${runId}/${file}`, { format: "stream", requestTimeoutMs: 300_000 });
198
219
  const parts = [];
199
220
  for await (const chunk of stream)
200
221
  parts.push(chunk);
201
- const bytes = Buffer.concat(parts);
222
+ return Buffer.concat(parts);
223
+ }
224
+ /** The bytes of one file of the runner's run directory, checked against its listed digest. */
225
+ export async function readRemoteFile(runner, runId, file) {
226
+ const bytes = await readRemoteBytes(runner, runId, file.path);
202
227
  const digest = createHash("sha256").update(bytes).digest("hex");
203
228
  if (digest !== file.sha256)
204
229
  throw new Error(`${file.path} changed or arrived damaged while it was copied from the runner.`);
@@ -25,7 +25,8 @@ study's input and admission estimate without credentials, a provider request,
25
25
  or a new analysis artifact. It prints the expected cost, the worst case and the
26
26
  cap; a refused dry run says why, and names the `--max-cost` that admits it.
27
27
 
28
- The default provider is `openai`; its default model is `gpt-6-astra`, with high reasoning effort. A request sends selected retained text and
28
+ The default provider is `openai`; its default model is `gpt-6.1-sol`, at medium reasoning effort
29
+ ([reasoning effort](#reasoning-effort)). A request sends selected retained text and
29
30
  captures to OpenAI, without tools, redirects, provider-side response storage, or
30
31
  automatic retries. `--question` adds a reviewer question; it never changes the
31
32
  participant assignment. For OpenAI API analysis, `--max-cost` is required, including for dry-run
@@ -118,6 +119,37 @@ the current merge instructions, billed $1.92, most of its input served from the
118
119
  provider's prompt cache, and each of its 466 citations resolved. Two runs do not
119
120
  calibrate the merge request's estimate.
120
121
 
122
+ ## Reasoning effort
123
+
124
+ `review.analysis.reasoningEffort`, or `analyze --reasoning-effort <level>`, sets how long an API
125
+ analyst reasons before it answers. OpenAI receives it as `reasoning.effort`, and Gemini as
126
+ `generationConfig.thinkingConfig.thinkingLevel` in capitals. Each model takes the levels its
127
+ provider documents for it (OpenAI's model pages and Gemini's thinking guide, read 2026-10-21):
128
+
129
+ | Model | Levels |
130
+ | ----------------------------------------------------------------------- | ----------------------------------------------- |
131
+ | `gpt-6-astra`, `gpt-6.1-sol` | `low`, `medium`, `high`, `xhigh`, `max` |
132
+ | `gpt-6-luna`, `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna` | `none`, `low`, `medium`, `high`, `xhigh`, `max` |
133
+ | `gpt-5.5` | `none`, `low`, `medium`, `high`, `xhigh` |
134
+ | `gemini-3.8-flash` | `low`, `medium`, `high` |
135
+
136
+ Any other level is refused before a request, with a message naming the levels the model takes.
137
+ The Codex account analyst runs at its fixed low effort, and a declared effort with
138
+ `provider: codex` is refused.
139
+
140
+ The default analyst, an OpenAI analysis that names no model and declares no effort, is
141
+ `gpt-6.1-sol` at `medium`, and its resolved configuration writes `reasoningEffort: medium`. A study
142
+ or command that names a model but no effort runs that model at `high`; only the default analyst
143
+ runs at medium. That analysis's configuration has no `reasoningEffort` field, so its configuration
144
+ hash, and with it reuse of an earlier analysis, is what it was before the field existed. A declared effort is stored in the analysis artifact's
145
+ `config`, `high` included, and gives the configuration its own hash. An artifact whose config has
146
+ no `reasoningEffort` ran at `high`.
147
+
148
+ Admission expects the same output at every effort, from analyses billed at `high`. No billed
149
+ analysis at a lower effort calibrates a smaller expectation yet, so a lower effort's expected cost
150
+ is likely over its bill: on 24 one-participant Taskly analyses on 2026-10-22, `gpt-6.1-sol` at
151
+ medium wrote a mean 3,747 output tokens against the 13,000 admission expects. The worst case spends the whole output allowance at any effort.
152
+
121
153
  ## Gemini API analysis
122
154
 
123
155
  `review.analysis.provider: gemini`, or `analyze --provider gemini`, sends the same request to
@@ -131,7 +163,8 @@ humanish analyze --run latest --provider gemini --model gemini-3.8-flash --max-c
131
163
 
132
164
  The request goes to `generativelanguage.googleapis.com/v1beta/models/<model>:generateContent`
133
165
  with the key in the `x-goog-api-key` header. It carries the same instructions, packet and
134
- captures, with each capture at `MEDIA_RESOLUTION_HIGH`, thinking level `HIGH`, one candidate,
166
+ captures, with each capture at `MEDIA_RESOLUTION_HIGH`, the analysis's reasoning effort as its
167
+ thinking level (`HIGH` unless declared), one candidate,
135
168
  the standard service tier, no tools and `store: false`. The key is `GEMINI_API_KEY` when set,
136
169
  else `GOOGLE_API_KEY`. Google's client libraries prefer `GOOGLE_API_KEY`; humanish does not, so a
137
170
  `GOOGLE_API_KEY` exported for another Google API never replaces the stored Gemini key. Key
@@ -10,8 +10,12 @@ Every other participant, including a Claude Code `local-agent` participant, gets
10
10
  analyst. humanish never moves a study to the other analyst because a key is missing: the account
11
11
  that pays for analysis is the one the study names or its participant implies.
12
12
 
13
- The OpenAI default is `gpt-6-astra` with high reasoning effort, a separate $10 cap on its
14
- expected cost and a 600-second timeout. When no output limit is specified,
13
+ The OpenAI default is `gpt-6.1-sol` at medium reasoning effort (`review.analysis.reasoningEffort`
14
+ sets another), a separate $10 cap on its
15
+ expected cost and a 600-second timeout. A study or command that names a model but no effort runs
16
+ that model at high; only the default analyst, with no model named, runs at medium. On the same
17
+ recordings it matched `gpt-6-astra` at high, the earlier default, at about one seventh of the cost
18
+ ([analysis models on fixed recordings](https://github.com/danielgwilson/humanish/blob/main/docs/evidence/benchmark/2026-10-22-analysis-models.md)). When no output limit is specified,
15
19
  humanish selects 32,768 tokens if the exact input's expected cost still fits that
16
20
  cap; otherwise it keeps the established 16,384-token allowance. This preserves
17
21
  previously admitted studies without increasing their spending limit. To customize it:
@@ -21,7 +25,9 @@ review:
21
25
  analysis:
22
26
  maxCostUsd: 10
23
27
  # Optional overrides. Omit maxOutputTokens for budget-aware selection.
24
- model: gpt-6-astra
28
+ # A named model without reasoningEffort runs at high.
29
+ model: gpt-6.1-sol
30
+ reasoningEffort: medium
25
31
  timeoutMs: 600000
26
32
  # maxOutputTokens: 32768
27
33
  # question: Where did participants need to recover?
@@ -41,9 +47,12 @@ the cap admits no analysis at all, it says so instead. A study of more than 16 p
41
47
  analysed in cohorts of at most 16, one request each, and one more request merges their reports;
42
48
  the range covers every request and the analysis line names them (see
43
49
  [runs with more than 16 participants](../contracts/study-analysis.md#runs-with-more-than-16-participants)).
44
- The default $10 cap admits a single request at the evidence limits ($3.79 expected for one
45
- participant, $4.54 for 16) and the analysis of 17 to 40 participants from $4.60 to $7.05 with no
46
- evidence; a run that keeps more evidence can still be refused, and its skip names the
50
+ The default $10 cap admits the default analyst up to the evidence limits: a single request at
51
+ $0.76 expected for one participant and $0.91 for 16, and 17 to 40 participants at $0.92 to $3.24
52
+ from no evidence to the limits. Admission expects 13,000 output tokens for one participant at any
53
+ effort; on 24 Taskly analyses `gpt-6.1-sol` at medium wrote a mean 3,747 and cost a mean $0.06.
54
+ `gpt-6-astra` at high expects $3.79 for one participant at the limits and $4.60 to $7.05 for 17 to
55
+ 40 with no evidence, so a run that keeps more evidence can be refused, and its skip names the
47
56
  `--max-cost` that admits it. Use `analyze --dry-run --max-cost <usd>` on retained
48
57
  evidence to inspect the expected cost, the worst case and the selected token
49
58
  allowance before deliberately choosing a larger budget. Explicit
@@ -89,7 +98,8 @@ review:
89
98
  maxCostUsd: 2
90
99
  ```
91
100
 
92
- Gemini analysis sends the same evidence packet and captures to Google, at high thinking level,
101
+ Gemini analysis sends the same evidence packet and captures to Google, at the high thinking level
102
+ unless the study declares a `reasoningEffort`,
93
103
  with each capture at high media resolution. It reads `GEMINI_API_KEY`, which `humanish keys set
94
104
  gemini` stores, and `GOOGLE_API_KEY` only when `GEMINI_API_KEY` is unset. Key discovery treats
95
105
  the two names as one key: a project's `.humanish/local/provider.env` or the user store never adds
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "humanish",
3
- "version": "0.142.0",
3
+ "version": "0.143.0",
4
4
  "description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
5
5
  "keywords": [
6
6
  "cli",
@@ -98,7 +98,8 @@ exact returned path, not a basename that could resolve to another manifest.
98
98
  Analysis is separate from the participant. A study that declares no analyst
99
99
  gets one from its participant: a Codex `local-agent` participant, local or
100
100
  hosted, defaults to a separate Codex account analyst; every other participant,
101
- Claude Code included, defaults to the OpenAI API with a $10 admission budget.
101
+ Claude Code included, defaults to the OpenAI API (`gpt-6.1-sol` at medium
102
+ reasoning effort) with a $10 admission budget.
102
103
  Without `OPENAI_API_KEY` that OpenAI analysis is skipped, and the run output
103
104
  names the next command: `analyze --provider codex` when Codex is signed in. To
104
105
  select the restricted Codex ChatGPT account analyst on any supported study,
@@ -118,6 +119,12 @@ and a missing Gemini key never falls back to OpenAI. On a free-tier key, Google
118
119
  may use the content to improve its products; tell the user before a study with
119
120
  private evidence runs on one.
120
121
 
122
+ `review.analysis.reasoningEffort` (or `analyze --reasoning-effort <level>`)
123
+ sets an OpenAI or Gemini analyst's reasoning effort. When unset it is `medium`
124
+ for the default analyst, which names no model, and `high` for a named model.
125
+ Each model takes only the levels its provider documents, and `analyze` refuses
126
+ the others before any request with the levels the model takes.
127
+
121
128
  When post-run analysis is refused for its cost, the run output and `humanish
122
129
  review` give its expected cost, its worst case and the command that runs it,
123
130
  `humanish analyze --run <id> --max-cost <n>`. Show the user those costs and run