humanish 0.113.0 → 0.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/CONTEXT.md +1 -0
- package/README.md +1 -1
- package/dist/actors/computer-use/openai-conversation.js +9 -7
- package/dist/actors/contract.d.ts +4 -2
- package/dist/analysis/admission.d.ts +37 -0
- package/dist/analysis/admission.js +60 -0
- package/dist/analysis/automatic-completion.d.ts +0 -6
- package/dist/analysis/automatic-completion.js +3 -3
- package/dist/analysis/automatic-config.d.ts +10 -2
- package/dist/analysis/automatic-config.js +29 -14
- package/dist/analysis/automatic.js +7 -0
- package/dist/analysis/execute.d.ts +19 -5
- package/dist/analysis/execute.js +58 -27
- package/dist/analysis/image-tokens.d.ts +2 -0
- package/dist/analysis/image-tokens.js +1 -1
- package/dist/analysis/job.d.ts +9 -1
- package/dist/analysis/job.js +11 -2
- package/dist/analysis/service.js +6 -3
- package/dist/cli/commands/analyze.js +7 -2
- package/dist/cli/commands/notes.d.ts +3 -0
- package/dist/cli/commands/notes.js +144 -0
- package/dist/cli/commands/observe.js +1 -1
- package/dist/cli/commands/study-run.js +2 -1
- package/dist/cli/findings.js +4 -1
- package/dist/cli/io.d.ts +1 -0
- package/dist/cli/io.js +11 -12
- package/dist/cli/program.js +2 -0
- package/dist/evidence/encoded-text.d.ts +15 -2
- package/dist/evidence/encoded-text.js +68 -4
- package/dist/evidence/secret-scrub.d.ts +1 -1
- package/dist/evidence/secret-scrub.js +23 -2
- package/dist/feedback/draft.d.ts +19 -0
- package/dist/feedback/draft.js +51 -1
- package/dist/feedback/export-bundle.js +2 -2
- package/dist/feedback/export.js +47 -3
- package/dist/feedback/feedback.js +42 -2
- package/dist/observer/artifact.d.ts +5 -0
- package/dist/observer/artifact.js +18 -2
- package/dist/observer/notes-route.d.ts +21 -0
- package/dist/observer/notes-route.js +140 -0
- package/dist/observer/pinned-files.d.ts +8 -1
- package/dist/observer/pinned-files.js +32 -3
- package/dist/observer/render.js +38 -14
- package/dist/observer/run-routes.d.ts +8 -2
- package/dist/observer/run-routes.js +47 -6
- package/dist/observer/serve.d.ts +3 -0
- package/dist/observer/serve.js +28 -3
- package/dist/observer-app.html +7 -6
- package/dist/run/contained-output.d.ts +7 -0
- package/dist/run/contained-output.js +35 -1
- package/dist/run/note-files.d.ts +73 -0
- package/dist/run/note-files.js +184 -0
- package/dist/run/notes.d.ts +37 -0
- package/dist/run/notes.js +198 -0
- package/dist/run/participant-caption.d.ts +9 -0
- package/dist/run/participant-caption.js +19 -0
- package/dist/study/doctor.js +1 -1
- package/dist/study/preflight-result.js +2 -1
- package/dist/study/summary.js +2 -2
- package/docs/contracts/errors.md +2 -2
- package/docs/contracts/run-bundle.md +57 -0
- package/docs/contracts/schemas.md +5 -2
- package/docs/contracts/study-analysis.md +26 -8
- package/docs/principles/invariants-and-defaults.md +2 -2
- package/docs/product/automatic-analysis.md +22 -16
- package/package.json +2 -2
- package/skills/humanish/SKILL.md +5 -0
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,28 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.114.0: Reviewer notes, expected analysis cost, percent-encoded UTF-8 in verify (2026-10-08)
|
|
12
|
+
|
|
13
|
+
humanish 0.114.0 lets a reviewer note a moment of a recorded run, admits post-run analysis on its
|
|
14
|
+
expected cost, and reads percent escapes as UTF-8 when it checks a run for secrets. Pause the study
|
|
15
|
+
timeline in an Observer served on 127.0.0.1 and choose "Add a note at 02:31", or run
|
|
16
|
+
`humanish notes <run> --add --at 02:31 "text"`. Each note is saved as its own file under `notes/`
|
|
17
|
+
in the run directory, marked on the study timeline and listed under "Reviewer notes" in Findings,
|
|
18
|
+
and `verify`, HTML export and feedback drafts read it. Analysis admission now prices input at 3
|
|
19
|
+
bytes per token and output at 12,000 tokens plus 1,000 per participant, and runs an analysis when
|
|
20
|
+
that expected cost plus 10% is within the cap, so the default $3 cap no longer refuses analyses
|
|
21
|
+
billed at $1 to $2. A refusal names the expected cost, the worst case, the cap and the
|
|
22
|
+
`humanish analyze --max-cost` command that admits it, and `study check` and live starts give the
|
|
23
|
+
expected cost range for the study's participant count. In `analyze --json`,
|
|
24
|
+
`admission.estimatedCostUsd` is now the expected cost. OpenAI computer-use traces list the
|
|
25
|
+
impressions request in `conversation.requests` with `kind: impressions`. `verify` and bundle
|
|
26
|
+
export also read percent escapes as UTF-8, so a percent-encoded password that starts with a
|
|
27
|
+
non-ASCII letter grades its run blocked, and the known-value scrub finds a percent-encoded value
|
|
28
|
+
with non-ASCII characters. The scrub also checks its matches against redaction markers by binary
|
|
29
|
+
search, so 1 MiB of markers and values takes well under a second instead of about 5 s.
|
|
30
|
+
|
|
31
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.114.0)
|
|
32
|
+
|
|
11
33
|
## 0.113.0: Participant impressions, mission and persona warnings, local desktop capacity, key status, update notice (2026-10-07)
|
|
12
34
|
|
|
13
35
|
humanish 0.113.0 asks computer-use participants what they thought at the end of a session, warns
|
package/CONTEXT.md
CHANGED
|
@@ -41,6 +41,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
41
41
|
| analysis | An independent model reading of a finished run's retained evidence, kept apart from participant feedback. Supported live runs request it automatically, and `humanish analyze` runs it on demand. It records findings and observations as `humanish.study-analysis.v1`. | `src/analysis/`; `runAutomaticAnalysis` in `src/analysis/automatic.ts`; `analyzeRun` in `src/analysis/service.ts`; `ANALYSIS_SCHEMA` in `src/analysis/types.ts` |
|
|
42
42
|
| finding | A result of analysis, with citations to evidence in the run. Each finding groups the observations that support it. | `src/analysis/` |
|
|
43
43
|
| observation | One claim in analysis with its basis, evidence ids and limitation. `concernReviews[]` gives each material concern's observation a disposition: `finding`, `context` or `unsupported`. | `src/analysis/` |
|
|
44
|
+
| reviewer note | Free text a person adds at a moment of a recorded run, for one participant or the whole run, kept in the run directory as its own file, `notes/<id>.json` (`humanish.run-note.v1`). It is not participant feedback or analysis. `humanish notes` and an Observer served on loopback add notes. | `addRunNote` in `src/run/notes.ts`; `readRunNotes` in `src/run/note-files.ts`; `createNotesWriter` in `src/observer/notes-route.ts` |
|
|
44
45
|
|
|
45
46
|
## Read these field spellings as the concept they name
|
|
46
47
|
|
package/README.md
CHANGED
|
@@ -219,7 +219,7 @@ The OpenAI API default stays unchanged. See the [analysis contract](docs/contrac
|
|
|
219
219
|
coverage limits, estimated cost controls, corrections and sharing behavior.
|
|
220
220
|
|
|
221
221
|
Supported live studies generate findings automatically after participants finish,
|
|
222
|
-
using a separate $3
|
|
222
|
+
using a separate $3 cap on the expected cost with `gpt-6-astra` and high reasoning.
|
|
223
223
|
This is additional to participant execution costs, and is not a hard provider
|
|
224
224
|
billing cap. To disable the extra request:
|
|
225
225
|
|
|
@@ -73,7 +73,7 @@ export class OpenAiConversation {
|
|
|
73
73
|
const sent = this.newItems(req);
|
|
74
74
|
return {
|
|
75
75
|
body: () => closing === undefined ? this.turnBody(req, sent) : this.closingBody(req, sent, closing),
|
|
76
|
-
accept: (reply) => this.accept(reply, closing
|
|
76
|
+
accept: (reply) => this.accept(reply, closing, sent),
|
|
77
77
|
};
|
|
78
78
|
}
|
|
79
79
|
/**
|
|
@@ -171,10 +171,10 @@ export class OpenAiConversation {
|
|
|
171
171
|
accept(reply, closing, sent) {
|
|
172
172
|
// Recorded before the reply is accepted: accepting can cut the carried history, and the record
|
|
173
173
|
// counts what this request carried.
|
|
174
|
-
if (
|
|
175
|
-
this.evidence.requested(this.mode, this.carried, sent);
|
|
174
|
+
if (closing !== "report")
|
|
175
|
+
this.evidence.requested(this.mode, this.carried, sent, closing);
|
|
176
176
|
const parsed = parseOpenAiResponse(reply);
|
|
177
|
-
if (
|
|
177
|
+
if (closing === undefined && parsed.turn.interruption === "output_limit")
|
|
178
178
|
return parsed.turn;
|
|
179
179
|
if (parsed.turn.responseId !== undefined)
|
|
180
180
|
this.lastResponseId = parsed.turn.responseId;
|
|
@@ -206,10 +206,11 @@ class ConversationRecord {
|
|
|
206
206
|
// Counted like requests[], which includes replies set aside at their output limit.
|
|
207
207
|
this.switchedAtRequest = this.requests.length + 1;
|
|
208
208
|
}
|
|
209
|
-
/** A participant request went out with `sent` after `conversation`. */
|
|
210
|
-
requested(mode, conversation, sent) {
|
|
209
|
+
/** A participant turn or impressions request went out with `sent` after `conversation`. */
|
|
210
|
+
requested(mode, conversation, sent, kind) {
|
|
211
|
+
const marker = kind === undefined ? {} : { kind };
|
|
211
212
|
if (mode !== "explicit_context") {
|
|
212
|
-
this.requests.push({ mode: "threaded" });
|
|
213
|
+
this.requests.push({ mode: "threaded", ...marker });
|
|
213
214
|
return;
|
|
214
215
|
}
|
|
215
216
|
const carried = conversation.size();
|
|
@@ -217,6 +218,7 @@ class ConversationRecord {
|
|
|
217
218
|
this.summarizedTurns = carried.notedTurns;
|
|
218
219
|
this.requests.push({
|
|
219
220
|
mode: "explicit_context",
|
|
221
|
+
...marker,
|
|
220
222
|
carriedExchanges: carried.exchanges,
|
|
221
223
|
carriedScreenshots: carried.screenshots,
|
|
222
224
|
estimatedInputTokens: carried.estimatedTokens + estimateTokens(sent),
|
|
@@ -261,11 +261,13 @@ export interface ActorConversation {
|
|
|
261
261
|
*/
|
|
262
262
|
summarizedTurns: number;
|
|
263
263
|
/**
|
|
264
|
-
* Per participant request, in order: its mode and, in explicit_context,
|
|
265
|
-
* estimated input. `tokenUsage.turns` has each request's billed input.
|
|
264
|
+
* Per participant turn or impressions request, in order: its mode and, in explicit_context,
|
|
265
|
+
* what it carried and its estimated input. `tokenUsage.turns` has each request's billed input.
|
|
266
266
|
*/
|
|
267
267
|
requests: Array<{
|
|
268
268
|
mode: "threaded" | "explicit_context";
|
|
269
|
+
/** Present for the impressions-only request; absent for participant turns and older bundles. */
|
|
270
|
+
kind?: "impressions";
|
|
269
271
|
carriedExchanges?: number;
|
|
270
272
|
carriedScreenshots?: number;
|
|
271
273
|
estimatedInputTokens?: number;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { ModelRate } from "../run/pricing.js";
|
|
2
|
+
/** Admission compares the expected cost times this margin with the cap. */
|
|
3
|
+
export declare const ADMISSION_MARGIN = 1.1;
|
|
4
|
+
export interface AnalysisRequestSize {
|
|
5
|
+
/** UTF-8 bytes of the instructions, the evidence packet and the result schema. */
|
|
6
|
+
textBytes: number;
|
|
7
|
+
imageTokens: number;
|
|
8
|
+
participants: number;
|
|
9
|
+
/** The request's output limit. The worst case spends all of it. */
|
|
10
|
+
outputAllowance: number;
|
|
11
|
+
}
|
|
12
|
+
export interface AnalysisCostEstimate {
|
|
13
|
+
inputTokens: number;
|
|
14
|
+
expectedOutputTokens: number;
|
|
15
|
+
expectedCostUsd: number;
|
|
16
|
+
worstCaseCostUsd: number;
|
|
17
|
+
/** The expected cost with the margin, never above the worst case: what admission compares. */
|
|
18
|
+
admittedCostUsd: number;
|
|
19
|
+
}
|
|
20
|
+
export declare function estimateAnalysisCost(rate: ModelRate, size: AnalysisRequestSize): AnalysisCostEstimate;
|
|
21
|
+
/** The numbers a cost refusal reports. */
|
|
22
|
+
export interface RefusedAnalysisCost {
|
|
23
|
+
expectedCostUsd: number;
|
|
24
|
+
worstCaseCostUsd: number;
|
|
25
|
+
maxCostUsd: number;
|
|
26
|
+
}
|
|
27
|
+
/** The --max-cost that admits a refused analysis: its worst case, rounded up to a whole dollar. */
|
|
28
|
+
export declare function admittingMaxCost(cost: RefusedAnalysisCost): number;
|
|
29
|
+
/** Why admission refused an analysis, in plain words. */
|
|
30
|
+
export declare function costRefusalText(cost: RefusedAnalysisCost): string;
|
|
31
|
+
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
32
|
+
export declare function refusedCost(admission: {
|
|
33
|
+
error: string | null;
|
|
34
|
+
estimatedCostUsd: number | null;
|
|
35
|
+
worstCaseCostUsd: number | null;
|
|
36
|
+
maxCostUsd: number | null;
|
|
37
|
+
}): RefusedAnalysisCost | undefined;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// The admission cost model for one study analysis request: the expected cost, the worst case, and
|
|
2
|
+
// the figure admission compares with the cap.
|
|
3
|
+
/**
|
|
4
|
+
* UTF-8 bytes per input token for the instructions, evidence packet and schema. On 148 billed
|
|
5
|
+
* gpt-6-astra analyses the evidence packet ran 3.1 to 3.3 bytes per token and the instructions
|
|
6
|
+
* more, so 3 stays above every billed input. Chinese, Japanese and Korean text is 3 bytes per
|
|
7
|
+
* character in UTF-8, so it also counts at least a token per character.
|
|
8
|
+
*/
|
|
9
|
+
const BYTES_PER_INPUT_TOKEN = 3;
|
|
10
|
+
/** Request framing on top of the text and the images. */
|
|
11
|
+
const FRAMING_TOKENS = 2048;
|
|
12
|
+
/**
|
|
13
|
+
* The output an analysis is expected to write, reasoning included. The most any billed analysis
|
|
14
|
+
* used was 12,511 tokens for one participant and 16,722 for eight.
|
|
15
|
+
*/
|
|
16
|
+
const EXPECTED_OUTPUT_TOKENS = 12_000;
|
|
17
|
+
const EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT = 1_000;
|
|
18
|
+
/** Admission compares the expected cost times this margin with the cap. */
|
|
19
|
+
export const ADMISSION_MARGIN = 1.1;
|
|
20
|
+
/** Rounded up to the micro-dollar, so a small positive boundary never rounds into admission. */
|
|
21
|
+
const roundUp = (usd) => Math.ceil(usd * 1e6) / 1e6;
|
|
22
|
+
/** One request's cost. Input is priced at the highest input rate the model bills, cache writes
|
|
23
|
+
* included, and the long-context tier re-prices the whole request past its threshold. */
|
|
24
|
+
function requestCost(rate, inputTokens, outputTokens) {
|
|
25
|
+
const long = rate.longContext !== undefined && inputTokens > rate.longContext.thresholdInputTokens;
|
|
26
|
+
const inputRate = Math.max(rate.inputUsdPerToken, rate.cacheWriteUsdPerToken ?? 0, rate.cachedInputUsdPerToken ?? 0);
|
|
27
|
+
return roundUp(inputTokens * inputRate * (long ? rate.longContext.inputMultiplier : 1) +
|
|
28
|
+
outputTokens * rate.outputUsdPerToken * (long ? rate.longContext.outputMultiplier : 1));
|
|
29
|
+
}
|
|
30
|
+
export function estimateAnalysisCost(rate, size) {
|
|
31
|
+
const inputTokens = Math.ceil(size.textBytes / BYTES_PER_INPUT_TOKEN) + FRAMING_TOKENS + size.imageTokens;
|
|
32
|
+
const expectedOutputTokens = Math.min(size.outputAllowance, EXPECTED_OUTPUT_TOKENS + EXPECTED_OUTPUT_TOKENS_PER_PARTICIPANT * size.participants);
|
|
33
|
+
const expectedCostUsd = requestCost(rate, inputTokens, expectedOutputTokens);
|
|
34
|
+
const worstCaseCostUsd = requestCost(rate, inputTokens, size.outputAllowance);
|
|
35
|
+
return {
|
|
36
|
+
inputTokens,
|
|
37
|
+
expectedOutputTokens,
|
|
38
|
+
expectedCostUsd,
|
|
39
|
+
worstCaseCostUsd,
|
|
40
|
+
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/** The --max-cost that admits a refused analysis: its worst case, rounded up to a whole dollar. */
|
|
44
|
+
export function admittingMaxCost(cost) {
|
|
45
|
+
return Math.max(1, Math.ceil(cost.worstCaseCostUsd));
|
|
46
|
+
}
|
|
47
|
+
/** Why admission refused an analysis, in plain words. */
|
|
48
|
+
export function costRefusalText(cost) {
|
|
49
|
+
return `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${Math.round((ADMISSION_MARGIN - 1) * 100)}% margin the expected cost is over the $${cost.maxCostUsd} cap, so no request was sent.`;
|
|
50
|
+
}
|
|
51
|
+
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
52
|
+
export function refusedCost(admission) {
|
|
53
|
+
const { error, estimatedCostUsd, worstCaseCostUsd, maxCostUsd } = admission;
|
|
54
|
+
return error === "analysis_budget_exceeded" &&
|
|
55
|
+
estimatedCostUsd !== null &&
|
|
56
|
+
worstCaseCostUsd !== null &&
|
|
57
|
+
maxCostUsd !== null
|
|
58
|
+
? { expectedCostUsd: estimatedCostUsd, worstCaseCostUsd, maxCostUsd }
|
|
59
|
+
: undefined;
|
|
60
|
+
}
|
|
@@ -35,10 +35,4 @@ export declare function completeAutomaticAnalysis<T extends {
|
|
|
35
35
|
/** The route's reason not to analyze this run; see AutomaticAnalysisDeps.refusal. */
|
|
36
36
|
refusal?: AutomaticAnalysisDeps["refusal"];
|
|
37
37
|
}): Promise<T & AutomaticAnalysisResult>;
|
|
38
|
-
/**
|
|
39
|
-
* True when an analysis the study never declared was refused because its conservative estimate is
|
|
40
|
-
* over the default cap. The author did not ask for it, so the refusal does not fail the run;
|
|
41
|
-
* `humanish analyze --max-cost` can still run it.
|
|
42
|
-
*/
|
|
43
|
-
export declare function defaultAnalysisOverBudget(result: AutomaticAnalysisResult): boolean;
|
|
44
38
|
export declare function automaticAnalysisSucceeded(result: AutomaticAnalysisResult): boolean;
|
|
@@ -85,11 +85,11 @@ export async function completeAutomaticAnalysis(result, finished, config, input,
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
/**
|
|
88
|
-
* True when an analysis the study never declared was refused because its
|
|
89
|
-
* over the default cap. The author did not ask for it, so the refusal does not fail the run;
|
|
88
|
+
* True when an analysis the study never declared was refused because its expected cost, with the
|
|
89
|
+
* admission margin, is over the default cap. The author did not ask for it, so the refusal does not fail the run;
|
|
90
90
|
* `humanish analyze --max-cost` can still run it.
|
|
91
91
|
*/
|
|
92
|
-
|
|
92
|
+
function defaultAnalysisOverBudget(result) {
|
|
93
93
|
const value = result.automaticAnalysis;
|
|
94
94
|
return (result.automaticAnalysisTrigger === "default" &&
|
|
95
95
|
value?.state === "skipped" &&
|
|
@@ -34,8 +34,16 @@ export interface AutomaticAnalysisBudget {
|
|
|
34
34
|
model: string;
|
|
35
35
|
maxCostUsd: number | null;
|
|
36
36
|
trigger: "default" | "explicit";
|
|
37
|
+
/** The participants the expected cost range is for. */
|
|
38
|
+
participants?: number;
|
|
39
|
+
/** The expected cost, from a run that keeps no evidence to one at the evidence limits. */
|
|
40
|
+
expectedCostUsd?: {
|
|
41
|
+
low: number;
|
|
42
|
+
high: number;
|
|
43
|
+
};
|
|
37
44
|
}
|
|
38
|
-
/** Metadata only: resolving the future live-run budget never reads keys or dispatches.
|
|
39
|
-
|
|
45
|
+
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
46
|
+
* participant count it also gives the expected cost range for that many participants. */
|
|
47
|
+
export declare function automaticAnalysisBudget(raw: unknown, route: StudyRoute, participants?: number): AutomaticAnalysisBudget | undefined;
|
|
40
48
|
export declare function formatAutomaticAnalysisBudget(budget: AutomaticAnalysisBudget): string;
|
|
41
49
|
export {};
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { CODEX_ANALYSIS_MODEL, codexAnalysisIdentity } from "./codex-config.js";
|
|
2
|
-
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, isSupportedAnalysisModel, } from "./execute.js";
|
|
2
|
+
import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysisCostRange, isSupportedAnalysisModel, } from "./execute.js";
|
|
3
|
+
import { ADMISSION_MARGIN } from "./admission.js";
|
|
4
|
+
import { plural } from "../run/text.js";
|
|
3
5
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
4
6
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
5
7
|
const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
@@ -111,25 +113,38 @@ export function resolveAutomaticAnalysis(raw) {
|
|
|
111
113
|
...(value.maxOutputTokens === undefined ? { preferLargerOutput: true } : {}),
|
|
112
114
|
};
|
|
113
115
|
}
|
|
114
|
-
/** Metadata only: resolving the future live-run budget never reads keys or dispatches.
|
|
115
|
-
|
|
116
|
+
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
117
|
+
* participant count it also gives the expected cost range for that many participants. */
|
|
118
|
+
export function automaticAnalysisBudget(raw, route, participants) {
|
|
116
119
|
// The synthetic preview has no participants, so nothing is analyzed after it.
|
|
117
120
|
if (route === "preview")
|
|
118
121
|
return undefined;
|
|
119
122
|
const resolved = resolveAutomaticAnalysis(raw);
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
123
|
+
if (!resolved.ok || !resolved.config)
|
|
124
|
+
return undefined;
|
|
125
|
+
const { config } = resolved;
|
|
126
|
+
const range = participants === undefined || config.provider === "codex"
|
|
127
|
+
? undefined
|
|
128
|
+
: analysisCostRange(resolved.preferLargerOutput
|
|
129
|
+
? { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS }
|
|
130
|
+
: config, participants);
|
|
131
|
+
return {
|
|
132
|
+
...(config.provider === "codex"
|
|
133
|
+
? { provider: "codex", billing: "account-unknown" }
|
|
134
|
+
: {}),
|
|
135
|
+
model: config.model,
|
|
136
|
+
maxCostUsd: config.maxCostUsd,
|
|
137
|
+
trigger: raw === undefined ? "default" : "explicit",
|
|
138
|
+
...(range === undefined || participants === undefined
|
|
139
|
+
? {}
|
|
140
|
+
: { participants, expectedCostUsd: range }),
|
|
141
|
+
};
|
|
130
142
|
}
|
|
131
143
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
132
144
|
if (budget.provider === "codex")
|
|
133
145
|
return `After live runs: Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
134
|
-
|
|
146
|
+
const range = budget.expectedCostUsd === undefined || budget.participants === undefined
|
|
147
|
+
? ""
|
|
148
|
+
: ` · expected $${budget.expectedCostUsd.low.toFixed(2)} to $${budget.expectedCostUsd.high.toFixed(2)} for ${plural(budget.participants, "participant")}, depending on how much evidence the run keeps`;
|
|
149
|
+
return `After live runs: ${budget.trigger} analysis · ${budget.model}${range} · refused before it starts if the expected cost plus a ${Math.round((ADMISSION_MARGIN - 1) * 100)}% margin is over $${budget.maxCostUsd}; this is not a billing cap. Set review.analysis: false to disable.`;
|
|
135
150
|
}
|
|
@@ -5,6 +5,7 @@ import { refreshObserver, analyzeRun, readCompletedAnalysisSource, resolveAnalys
|
|
|
5
5
|
import { preferLargerAnalysisOutput, ANALYSIS_PROMPT_VERSION } from "./execute.js";
|
|
6
6
|
import { captureEvidence } from "./evidence.js";
|
|
7
7
|
import { hashAnalysisValue } from "./validation.js";
|
|
8
|
+
import { refusedCost } from "./admission.js";
|
|
8
9
|
import { claimAutomaticAnalysis, readAutomaticAnalysisPrepared, requestAutomaticAnalysisCancellationPrepared, } from "./job.js";
|
|
9
10
|
import { ANALYSIS_ID_PATTERN } from "./types.js";
|
|
10
11
|
import { readAnalysisVersion } from "./store.js";
|
|
@@ -105,6 +106,11 @@ async function readAutomaticSource(cwd, prepared, configInput, deps, hasKey) {
|
|
|
105
106
|
return skipped("AUTOMATIC_ANALYSIS_SOURCE_UNAVAILABLE");
|
|
106
107
|
}
|
|
107
108
|
}
|
|
109
|
+
/** The cost of an analysis admission refused for its cost, kept so review and Observer can say it. */
|
|
110
|
+
function withRefusedCost(result) {
|
|
111
|
+
const cost = result?.admission && refusedCost(result.admission);
|
|
112
|
+
return cost ? { admission: cost } : {};
|
|
113
|
+
}
|
|
108
114
|
/**
|
|
109
115
|
* Bind the job's terminal metadata to the exact safely published execution. Reads never promote a
|
|
110
116
|
* completed-looking sidecar without rechecking these bindings. Null when storage failed.
|
|
@@ -130,6 +136,7 @@ async function persistOutcome(job, prepared, outcome) {
|
|
|
130
136
|
receiptSha256: hashAnalysisValue(receipt),
|
|
131
137
|
}),
|
|
132
138
|
...(entry?.analysis ? { analysisSha256: hashAnalysisValue(entry.analysis) } : {}),
|
|
139
|
+
...withRefusedCost(outcome.result),
|
|
133
140
|
});
|
|
134
141
|
const persisted = await readAutomaticAnalysisPrepared(prepared);
|
|
135
142
|
return persisted?.state === "unknown"
|
|
@@ -9,9 +9,14 @@ export declare const MAX_ANALYSIS_OUTPUT_TOKENS = 32768;
|
|
|
9
9
|
export interface AnalysisAdmission {
|
|
10
10
|
allowed: boolean;
|
|
11
11
|
error: string | null;
|
|
12
|
+
/** The input tokens admission priced. */
|
|
12
13
|
inputTokenAllowance: number | null;
|
|
13
14
|
outputTokenAllowance: number | null;
|
|
15
|
+
/** The expected cost. Admission compares it, with a margin, with maxCostUsd. */
|
|
14
16
|
estimatedCostUsd: number | null;
|
|
17
|
+
/** The cost if the analyst spends its whole output allowance. */
|
|
18
|
+
worstCaseCostUsd: number | null;
|
|
19
|
+
maxCostUsd: number | null;
|
|
15
20
|
ratesAsOf: string | null;
|
|
16
21
|
}
|
|
17
22
|
export interface AnalysisProgress {
|
|
@@ -23,15 +28,24 @@ export interface AnalysisProgress {
|
|
|
23
28
|
}
|
|
24
29
|
/**
|
|
25
30
|
* Local admission estimate, not a provider-enforced billed-spend guarantee. No token-count API
|
|
26
|
-
* call, credential, or network access.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* assumptions.
|
|
31
|
+
* call, credential, or network access. admission.ts holds the cost model: text at a calibrated
|
|
32
|
+
* bytes-per-token ratio, the expected output, and the worst case that spends the whole output
|
|
33
|
+
* allowance. Each high-detail image is priced from its PNG size by the vision guide's patch
|
|
34
|
+
* formula (see image-tokens.ts), or at the 3,000-token ceiling when its size or the model's sizing
|
|
35
|
+
* is unknown. Unknown models/rates fail closed rather than inheriting those assumptions.
|
|
31
36
|
* Known-sensitive decoded text denies admission with analysis_input_sensitive or
|
|
32
37
|
* analysis_question_sensitive. Neither error includes rejected input values.
|
|
33
38
|
*/
|
|
34
39
|
export declare function estimateAnalysisAdmission(input: AnalysisInput, config: AnalysisConfig): AnalysisAdmission;
|
|
40
|
+
/**
|
|
41
|
+
* The expected cost of an analysis for this many participants before any evidence exists: from a
|
|
42
|
+
* packet with no evidence to one at the evidence limits with every capture at the image-token
|
|
43
|
+
* ceiling. Undefined for Codex and for a model without rates.
|
|
44
|
+
*/
|
|
45
|
+
export declare function analysisCostRange(config: AnalysisConfig, participants: number): {
|
|
46
|
+
low: number;
|
|
47
|
+
high: number;
|
|
48
|
+
} | undefined;
|
|
35
49
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
36
50
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
37
51
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
package/dist/analysis/execute.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { validCodexAnalysisConfig } from "./codex-config.js";
|
|
2
|
-
import { highDetailImageTokens } from "./image-tokens.js";
|
|
2
|
+
import { HIGH_DETAIL_IMAGE_TOKEN_CEILING, highDetailImageTokens } from "./image-tokens.js";
|
|
3
|
+
import { estimateAnalysisCost } from "./admission.js";
|
|
3
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
5
|
import { estimateActorCost, MODEL_RATES } from "../run/pricing.js";
|
|
5
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
@@ -128,11 +129,11 @@ function inputError(input) {
|
|
|
128
129
|
}
|
|
129
130
|
/**
|
|
130
131
|
* Local admission estimate, not a provider-enforced billed-spend guarantee. No token-count API
|
|
131
|
-
* call, credential, or network access.
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* assumptions.
|
|
132
|
+
* call, credential, or network access. admission.ts holds the cost model: text at a calibrated
|
|
133
|
+
* bytes-per-token ratio, the expected output, and the worst case that spends the whole output
|
|
134
|
+
* allowance. Each high-detail image is priced from its PNG size by the vision guide's patch
|
|
135
|
+
* formula (see image-tokens.ts), or at the 3,000-token ceiling when its size or the model's sizing
|
|
136
|
+
* is unknown. Unknown models/rates fail closed rather than inheriting those assumptions.
|
|
136
137
|
* Known-sensitive decoded text denies admission with analysis_input_sensitive or
|
|
137
138
|
* analysis_question_sensitive. Neither error includes rejected input values.
|
|
138
139
|
*/
|
|
@@ -143,6 +144,8 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
143
144
|
inputTokenAllowance: 0,
|
|
144
145
|
outputTokenAllowance: 0,
|
|
145
146
|
estimatedCostUsd: null,
|
|
147
|
+
worstCaseCostUsd: null,
|
|
148
|
+
maxCostUsd: null,
|
|
146
149
|
ratesAsOf: null,
|
|
147
150
|
});
|
|
148
151
|
if ((config.provider === "codex"
|
|
@@ -174,6 +177,8 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
174
177
|
inputTokenAllowance: null,
|
|
175
178
|
outputTokenAllowance: null,
|
|
176
179
|
estimatedCostUsd: null,
|
|
180
|
+
worstCaseCostUsd: null,
|
|
181
|
+
maxCostUsd: null,
|
|
177
182
|
ratesAsOf: null,
|
|
178
183
|
};
|
|
179
184
|
const rate = MODEL_RATES[config.model];
|
|
@@ -184,32 +189,56 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
184
189
|
!Number.isFinite(rate.outputUsdPerToken) ||
|
|
185
190
|
rate.outputUsdPerToken < 0)
|
|
186
191
|
return denied("analysis_rate_unknown");
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
const estimate = inputTokenAllowance * inputRate * (long ? rate.longContext.inputMultiplier : 1) +
|
|
197
|
-
config.maxOutputTokens *
|
|
198
|
-
rate.outputUsdPerToken *
|
|
199
|
-
(long ? rate.longContext.outputMultiplier : 1);
|
|
200
|
-
// Round upward for admission; rounding a small positive boundary down could admit an overrun.
|
|
201
|
-
const estimatedCostUsd = Math.ceil(estimate * 1e6) / 1e6;
|
|
202
|
-
if (!Number.isFinite(estimatedCostUsd))
|
|
192
|
+
const cost = estimateAnalysisCost(rate, {
|
|
193
|
+
textBytes: Buffer.byteLength(instructions(config)) +
|
|
194
|
+
Buffer.byteLength(evidenceText(input)) +
|
|
195
|
+
Buffer.byteLength(JSON.stringify(analysisResultJsonSchema)),
|
|
196
|
+
imageTokens: input.images.reduce((sum, image) => sum + highDetailImageTokens(config.model, image.dataUrl), 0),
|
|
197
|
+
participants: input.participants.length,
|
|
198
|
+
outputAllowance: config.maxOutputTokens,
|
|
199
|
+
});
|
|
200
|
+
if (!Number.isFinite(cost.worstCaseCostUsd))
|
|
203
201
|
return denied("analysis_rate_unknown");
|
|
202
|
+
const allowed = cost.admittedCostUsd <= config.maxCostUsd;
|
|
204
203
|
return {
|
|
205
|
-
allowed
|
|
206
|
-
error:
|
|
207
|
-
inputTokenAllowance,
|
|
204
|
+
allowed,
|
|
205
|
+
error: allowed ? null : "analysis_budget_exceeded",
|
|
206
|
+
inputTokenAllowance: cost.inputTokens,
|
|
208
207
|
outputTokenAllowance: config.maxOutputTokens,
|
|
209
|
-
estimatedCostUsd,
|
|
208
|
+
estimatedCostUsd: cost.expectedCostUsd,
|
|
209
|
+
worstCaseCostUsd: cost.worstCaseCostUsd,
|
|
210
|
+
maxCostUsd: config.maxCostUsd,
|
|
210
211
|
ratesAsOf: rate.asOf,
|
|
211
212
|
};
|
|
212
213
|
}
|
|
214
|
+
/** Structure each evidence entry adds to the packet: ids, kind, timestamps and flags. The largest
|
|
215
|
+
* retained packet averaged 200 bytes. */
|
|
216
|
+
const ENTRY_STRUCTURE_BYTES = 256;
|
|
217
|
+
/**
|
|
218
|
+
* The expected cost of an analysis for this many participants before any evidence exists: from a
|
|
219
|
+
* packet with no evidence to one at the evidence limits with every capture at the image-token
|
|
220
|
+
* ceiling. Undefined for Codex and for a model without rates.
|
|
221
|
+
*/
|
|
222
|
+
export function analysisCostRange(config, participants) {
|
|
223
|
+
if (config.provider === "codex")
|
|
224
|
+
return undefined;
|
|
225
|
+
const rate = MODEL_RATES[config.model];
|
|
226
|
+
if (!rate || rate.placeholder)
|
|
227
|
+
return undefined;
|
|
228
|
+
const prompt = Buffer.byteLength(instructions(config)) +
|
|
229
|
+
Buffer.byteLength(JSON.stringify(analysisResultJsonSchema));
|
|
230
|
+
const size = {
|
|
231
|
+
participants: Math.min(participants, EVIDENCE_LIMITS.participants),
|
|
232
|
+
outputAllowance: config.maxOutputTokens,
|
|
233
|
+
};
|
|
234
|
+
const low = estimateAnalysisCost(rate, { ...size, textBytes: prompt, imageTokens: 0 });
|
|
235
|
+
const high = estimateAnalysisCost(rate, {
|
|
236
|
+
...size,
|
|
237
|
+
textBytes: prompt + EVIDENCE_LIMITS.textBytes + EVIDENCE_LIMITS.evidence * ENTRY_STRUCTURE_BYTES,
|
|
238
|
+
imageTokens: EVIDENCE_LIMITS.captures * HIGH_DETAIL_IMAGE_TOKEN_CEILING,
|
|
239
|
+
});
|
|
240
|
+
return { low: low.expectedCostUsd, high: high.expectedCostUsd };
|
|
241
|
+
}
|
|
213
242
|
/** Only for an omitted output limit. Preserve the established allowance when
|
|
214
243
|
* more reasoning/report space would refuse a study its declared budget admits.
|
|
215
244
|
* This is one pre-dispatch choice, never a fallback request or a budget increase. */
|
|
@@ -464,9 +493,11 @@ function settleCompletedOutput(artifact, input, config, admission, response, onR
|
|
|
464
493
|
artifact.result = checked.result;
|
|
465
494
|
artifact.status = input.coverage.complete ? "complete" : "partial";
|
|
466
495
|
artifact.error = null;
|
|
496
|
+
// A bill above the expected cost is a normal outcome. Only one above the worst case or the cap
|
|
497
|
+
// shows the estimate was wrong.
|
|
467
498
|
if (config.provider !== "codex" &&
|
|
468
499
|
((response.usage?.output ?? 0) > config.maxOutputTokens ||
|
|
469
|
-
(artifact.usage.estimatedCostUsd ?? 0) > (admission.
|
|
500
|
+
(artifact.usage.estimatedCostUsd ?? 0) > (admission.worstCaseCostUsd ?? config.maxCostUsd) ||
|
|
470
501
|
(artifact.usage.estimatedCostUsd ?? 0) > config.maxCostUsd)) {
|
|
471
502
|
artifact.status = "partial";
|
|
472
503
|
artifact.error = "analysis_admission_estimate_exceeded";
|
|
@@ -16,7 +16,7 @@ const HIGH_DETAIL_SIZING = Object.freeze({
|
|
|
16
16
|
"gpt-5.5": { patchBudget: 2500, multiplierTenths: 12 },
|
|
17
17
|
});
|
|
18
18
|
/** 2,500 patches × 1.2: what any image costs on these models when its size is unknown. */
|
|
19
|
-
const HIGH_DETAIL_IMAGE_TOKEN_CEILING = 3000;
|
|
19
|
+
export const HIGH_DETAIL_IMAGE_TOKEN_CEILING = 3000;
|
|
20
20
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
21
21
|
/** Width and height from a PNG data URL's IHDR chunk; undefined for anything else. */
|
|
22
22
|
function pngDimensions(dataUrl) {
|
package/dist/analysis/job.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AnalyzeResult } from "./service.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
4
|
+
import type { RefusedAnalysisCost } from "./admission.js";
|
|
4
5
|
export declare const AUTOMATIC_ANALYSIS_DIRECTORY = "analysis-automatic";
|
|
5
6
|
export declare const AUTOMATIC_ANALYSIS_STALE_MS = 15000;
|
|
6
7
|
/** Execution metadata only. Never a participant outcome or permission to dispatch. */
|
|
@@ -11,6 +12,8 @@ export interface AutomaticAnalysisView {
|
|
|
11
12
|
/** Safe stable code, not provider text. */
|
|
12
13
|
reason: string | null;
|
|
13
14
|
updatedAt: string;
|
|
15
|
+
/** What the analysis would have cost, when admission refused it for its cost. */
|
|
16
|
+
admission?: RefusedAnalysisCost;
|
|
14
17
|
}
|
|
15
18
|
export interface AutomaticAnalysisOutcome {
|
|
16
19
|
state: "complete" | "partial" | "failed" | "cancelled" | "skipped" | "unknown";
|
|
@@ -69,6 +72,11 @@ declare const jobSchema: z.ZodObject<{
|
|
|
69
72
|
receiptSha256: z.ZodNullable<z.ZodString>;
|
|
70
73
|
startedAt: z.ZodNullable<z.ZodISODateTime>;
|
|
71
74
|
finishedAt: z.ZodNullable<z.ZodISODateTime>;
|
|
75
|
+
admission: z.ZodOptional<z.ZodObject<{
|
|
76
|
+
expectedCostUsd: z.ZodNumber;
|
|
77
|
+
worstCaseCostUsd: z.ZodNumber;
|
|
78
|
+
maxCostUsd: z.ZodNumber;
|
|
79
|
+
}, z.core.$strict>>;
|
|
72
80
|
}, z.core.$strict>;
|
|
73
81
|
type JobRecord = z.infer<typeof jobSchema>;
|
|
74
82
|
/** Also guards direct HTML projection callers, not only the contained-file reader. */
|
|
@@ -86,7 +94,7 @@ export declare function readAutomaticAnalysisPrepared(prepared: PreparedRunArtif
|
|
|
86
94
|
export interface AutomaticAnalysisJob {
|
|
87
95
|
readonly attemptId: string;
|
|
88
96
|
/** Serialized, awaited durability boundary. A failed write must prevent dispatch. */
|
|
89
|
-
update(update: Partial<Pick<JobRecord, "state" | "analysisId" | "reason" | "sourceRunSha256" | "inputDigest" | "startedAt" | "analysisSha256" | "receiptSha256">>): Promise<void>;
|
|
97
|
+
update(update: Partial<Pick<JobRecord, "state" | "analysisId" | "reason" | "sourceRunSha256" | "inputDigest" | "startedAt" | "analysisSha256" | "receiptSha256" | "admission">>): Promise<void>;
|
|
90
98
|
touch(): Promise<void>;
|
|
91
99
|
cancellationRequested(): Promise<boolean>;
|
|
92
100
|
}
|
package/dist/analysis/job.js
CHANGED
|
@@ -52,6 +52,12 @@ const reasons = [
|
|
|
52
52
|
const id = z.string().regex(ANALYSIS_ID_PATTERN);
|
|
53
53
|
const date = z.iso.datetime();
|
|
54
54
|
const digest = z.string().regex(SHA256_HEX_PATTERN);
|
|
55
|
+
const usd = z.number().min(0).max(1e9);
|
|
56
|
+
const refusedCostSchema = z.strictObject({
|
|
57
|
+
expectedCostUsd: usd,
|
|
58
|
+
worstCaseCostUsd: usd,
|
|
59
|
+
maxCostUsd: usd,
|
|
60
|
+
});
|
|
55
61
|
const jobSchema = z.strictObject({
|
|
56
62
|
schema: z.literal(AUTOMATIC_ANALYSIS_SCHEMA),
|
|
57
63
|
runId: id,
|
|
@@ -70,6 +76,7 @@ const jobSchema = z.strictObject({
|
|
|
70
76
|
receiptSha256: digest.nullable(),
|
|
71
77
|
startedAt: date.nullable(),
|
|
72
78
|
finishedAt: date.nullable(),
|
|
79
|
+
admission: refusedCostSchema.optional(),
|
|
73
80
|
});
|
|
74
81
|
const cancelSchema = z.strictObject({
|
|
75
82
|
schema: z.literal(CANCEL_SCHEMA),
|
|
@@ -111,11 +118,12 @@ export function projectAutomaticAnalysisView(value) {
|
|
|
111
118
|
if (value === undefined)
|
|
112
119
|
return undefined;
|
|
113
120
|
const parsed = jobSchema
|
|
114
|
-
.pick({ state: true, analysisId: true, reason: true, updatedAt: true })
|
|
121
|
+
.pick({ state: true, analysisId: true, reason: true, updatedAt: true, admission: true })
|
|
115
122
|
.safeParse(value);
|
|
116
123
|
if (!parsed.success || containsSensitive(JSON.stringify(parsed.data)))
|
|
117
124
|
return unknown();
|
|
118
|
-
|
|
125
|
+
const { admission, ...view } = parsed.data;
|
|
126
|
+
return admission === undefined ? view : { ...view, admission };
|
|
119
127
|
}
|
|
120
128
|
async function bindJob(prepared) {
|
|
121
129
|
await validatePreparedRunRootIdentity(prepared);
|
|
@@ -162,6 +170,7 @@ export async function readAutomaticAnalysisPrepared(prepared, now = Date.now())
|
|
|
162
170
|
analysisId: record.analysisId,
|
|
163
171
|
reason: record.reason,
|
|
164
172
|
updatedAt: record.updatedAt,
|
|
173
|
+
...(record.admission === undefined ? {} : { admission: record.admission }),
|
|
165
174
|
};
|
|
166
175
|
const updated = Date.parse(record.updatedAt);
|
|
167
176
|
if (updated > now || (pending(record.state) && now - updated > AUTOMATIC_ANALYSIS_STALE_MS)) {
|