@atmin.ai/review 0.1.0-alpha.2 → 0.1.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEPENDENCIES.md +20 -0
- package/README.md +246 -18
- package/dist/ablation.d.ts +30 -0
- package/dist/ablation.js +90 -0
- package/dist/assessment.d.ts +4 -0
- package/dist/assessment.js +61 -5
- package/dist/callees.d.ts +14 -0
- package/dist/callees.js +73 -0
- package/dist/claim-result.d.ts +64 -0
- package/dist/claim-result.js +206 -0
- package/dist/claim-run.d.ts +38 -0
- package/dist/claim-run.js +227 -0
- package/dist/claim.d.ts +71 -0
- package/dist/claim.js +163 -0
- package/dist/cli.js +103 -11
- package/dist/code-review-runner.d.ts +25 -0
- package/dist/code-review-runner.js +280 -0
- package/dist/contracts.d.ts +395 -5
- package/dist/contracts.js +49 -11
- package/dist/cost-report.d.ts +4 -2
- package/dist/evidence.d.ts +34 -0
- package/dist/evidence.js +57 -0
- package/dist/failure-excerpt.d.ts +5 -0
- package/dist/failure-excerpt.js +63 -0
- package/dist/github/api.d.ts +40 -1
- package/dist/github/api.js +108 -16
- package/dist/github/billing.d.ts +32 -0
- package/dist/github/billing.js +215 -0
- package/dist/github/checks.js +11 -5
- package/dist/github/cli.js +76 -13
- package/dist/github/config.d.ts +4 -0
- package/dist/github/config.js +16 -1
- package/dist/github/dashboard-view.d.ts +205 -0
- package/dist/github/dashboard-view.js +261 -0
- package/dist/github/dashboard.d.ts +11 -0
- package/dist/github/dashboard.js +551 -0
- package/dist/github/inline.d.ts +11 -0
- package/dist/github/inline.js +107 -0
- package/dist/github/outcomes.d.ts +17 -0
- package/dist/github/outcomes.js +127 -0
- package/dist/github/repositories.d.ts +110 -0
- package/dist/github/repositories.js +263 -0
- package/dist/github/runner-api.d.ts +4 -0
- package/dist/github/runner-api.js +200 -0
- package/dist/github/runner.d.ts +16 -3
- package/dist/github/runner.js +108 -21
- package/dist/github/runners.d.ts +56 -0
- package/dist/github/runners.js +127 -0
- package/dist/github/settings.d.ts +34 -0
- package/dist/github/settings.js +76 -0
- package/dist/github/site.d.ts +2 -0
- package/dist/github/site.js +48 -0
- package/dist/github/store.d.ts +16 -3
- package/dist/github/store.js +97 -7
- package/dist/github/task.js +10 -4
- package/dist/github/webhook.d.ts +6 -2
- package/dist/github/webhook.js +78 -57
- package/dist/github/worker.d.ts +27 -5
- package/dist/github/worker.js +215 -51
- package/dist/investigation.d.ts +112 -6
- package/dist/investigation.js +268 -76
- package/dist/investigator.d.ts +278 -0
- package/dist/investigator.js +375 -0
- package/dist/jev.d.ts +34 -0
- package/dist/jev.js +224 -0
- package/dist/lifecycle.d.ts +41 -0
- package/dist/lifecycle.js +180 -0
- package/dist/models/claude-cli.d.ts +11 -0
- package/dist/models/claude-cli.js +107 -0
- package/dist/openai-model.js +7 -0
- package/dist/openrouter-model.js +48 -7
- package/dist/policy.d.ts +42 -0
- package/dist/policy.js +62 -0
- package/dist/rating.d.ts +21 -0
- package/dist/rating.js +65 -0
- package/dist/render-claim.d.ts +4 -0
- package/dist/render-claim.js +61 -0
- package/dist/render.d.ts +19 -3
- package/dist/render.js +95 -21
- package/dist/run.js +6 -3
- package/dist/runner-cli.d.ts +2 -0
- package/dist/runner-cli.js +9 -0
- package/dist/snapshot.d.ts +19 -3
- package/dist/snapshot.js +161 -13
- package/dist/symbolic.d.ts +37 -0
- package/dist/symbolic.js +360 -0
- package/dist/titles.d.ts +13 -0
- package/dist/titles.js +79 -0
- package/dist/trace.d.ts +8 -0
- package/dist/trace.js +87 -0
- package/dist/verification.d.ts +22 -0
- package/dist/verification.js +136 -0
- package/package.json +10 -6
package/dist/callees.js
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { declaresSymbol } from './claim.js';
|
|
2
|
+
import { definitionOf } from './symbolic.js';
|
|
3
|
+
// The definitions, at head, of functions the added lines call and the change does not
|
|
4
|
+
// define. A change can be right on its own lines and wrong in what unchanged code does
|
|
5
|
+
// with its values: on mason-v1 #4590 (2026-09-24) new code threw plain errors into a
|
|
6
|
+
// catch-all mapper 780 lines away that labels every one "vendor unavailable, retry", and
|
|
7
|
+
// in seven runs the model never opened that file. Resolved by code, not left to the model
|
|
8
|
+
// to go looking. Calls are taken by name or on `this`/`self` only: `x.map(` names a method
|
|
9
|
+
// of a value nothing here can type, and a guess would put unrelated code in front of it.
|
|
10
|
+
// A definition that does not fit is named, never silently dropped.
|
|
11
|
+
export const MAX_CALLED_CODE_BYTES = 24 * 1024;
|
|
12
|
+
const CALL = /(?<![\w$])([A-Za-z_$][\w$]*)\s*\(/g;
|
|
13
|
+
const NOT_CALLS = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'typeof', 'await',
|
|
14
|
+
'super', 'import', 'require', 'def', 'elif', 'not', 'and', 'or', 'in', 'lambda', 'assert', 'print', 'with']);
|
|
15
|
+
const SOURCE = /\.(?:[cm]?[jt]sx?|py)$/;
|
|
16
|
+
export const TEST = /(?:^|\/)(?:__tests__|tests?)\/|\.(?:test|spec)\.[^/]+$|(?:^|\/)test_[^/]+\.py$/;
|
|
17
|
+
// Added lines per file, from a unified diff. A deleted file has no head side to call from.
|
|
18
|
+
function addedLines(diff) {
|
|
19
|
+
const added = new Map();
|
|
20
|
+
let path = null;
|
|
21
|
+
for (const line of diff.split('\n')) {
|
|
22
|
+
if (line.startsWith('+++ ')) {
|
|
23
|
+
path = line.startsWith('+++ b/') ? line.slice(6) : null;
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
if (line.startsWith('diff --git ')) {
|
|
27
|
+
path = null;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (path && line.startsWith('+'))
|
|
31
|
+
added.set(path, [...(added.get(path) ?? []), line.slice(1)]);
|
|
32
|
+
}
|
|
33
|
+
return added;
|
|
34
|
+
}
|
|
35
|
+
export function calledCode(diff, head) {
|
|
36
|
+
const added = addedLines(diff);
|
|
37
|
+
const addedText = [...added.values()].flat();
|
|
38
|
+
// name -> the first changed file that calls it, which is where resolution looks first.
|
|
39
|
+
const calls = new Map();
|
|
40
|
+
for (const [path, lines] of added) {
|
|
41
|
+
if (!SOURCE.test(path) || TEST.test(path))
|
|
42
|
+
continue;
|
|
43
|
+
for (const line of lines) {
|
|
44
|
+
for (const match of line.matchAll(CALL)) {
|
|
45
|
+
const before = line.slice(0, match.index);
|
|
46
|
+
if (/\.\s*$/.test(before) && !/\b(?:this|self)\.\s*$/.test(before))
|
|
47
|
+
continue;
|
|
48
|
+
const name = match[1];
|
|
49
|
+
if (!NOT_CALLS.has(name) && !calls.has(name))
|
|
50
|
+
calls.set(name, path);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
const called = [];
|
|
55
|
+
const omitted = [];
|
|
56
|
+
let bytes = 0;
|
|
57
|
+
for (const [symbol, from] of calls) {
|
|
58
|
+
// Defined by the change itself: the model already has it in the diff.
|
|
59
|
+
if (addedText.some(line => declaresSymbol(line, symbol)))
|
|
60
|
+
continue;
|
|
61
|
+
const definition = definitionOf(head, symbol, from, path => SOURCE.test(path) && !TEST.test(path));
|
|
62
|
+
if (!definition)
|
|
63
|
+
continue;
|
|
64
|
+
const size = Buffer.byteLength(definition.text);
|
|
65
|
+
if (bytes + size > MAX_CALLED_CODE_BYTES) {
|
|
66
|
+
omitted.push(symbol);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
bytes += size;
|
|
70
|
+
called.push({ symbol, ...definition });
|
|
71
|
+
}
|
|
72
|
+
return { called, omitted };
|
|
73
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { type ClaimReview, type IncrementalScope } from './claim-run.js';
|
|
2
|
+
import { type Titling } from './titles.js';
|
|
3
|
+
import { type Claim } from './claim.js';
|
|
4
|
+
import { type Packet, type Result } from './contracts.js';
|
|
5
|
+
import type { Chain } from './evidence.js';
|
|
6
|
+
import type { Model, Profile } from './investigation.js';
|
|
7
|
+
export interface ClaimRunSummary {
|
|
8
|
+
claims: Claim[];
|
|
9
|
+
chains: Chain[];
|
|
10
|
+
verdict: string;
|
|
11
|
+
rule: string;
|
|
12
|
+
limitations: string[];
|
|
13
|
+
complete: boolean;
|
|
14
|
+
stopReason: string | null;
|
|
15
|
+
model: string;
|
|
16
|
+
scope?: string;
|
|
17
|
+
titles?: Record<string, string>;
|
|
18
|
+
}
|
|
19
|
+
export declare function claimResult(packet: Packet, repository: string, run: ClaimRunSummary): Result;
|
|
20
|
+
export interface PreviousReview {
|
|
21
|
+
headSha: string;
|
|
22
|
+
mergeBaseSha: string;
|
|
23
|
+
claims: Claim[];
|
|
24
|
+
titles?: Record<string, string>;
|
|
25
|
+
}
|
|
26
|
+
export declare function incrementalScope(directory: string, packet: Packet): {
|
|
27
|
+
scope?: IncrementalScope;
|
|
28
|
+
note: string | null;
|
|
29
|
+
titles?: Record<string, string>;
|
|
30
|
+
};
|
|
31
|
+
export declare function previousReview(artifact: string): PreviousReview | null;
|
|
32
|
+
export declare function claimReceipt(profile: Profile, review: ClaimReview, startedAt: string, finishedAt: string, titling?: Titling): {
|
|
33
|
+
schemaVersion: number;
|
|
34
|
+
pipeline: string;
|
|
35
|
+
profile: Profile;
|
|
36
|
+
startedAt: string;
|
|
37
|
+
finishedAt: string;
|
|
38
|
+
stopReason: string | null;
|
|
39
|
+
providerFailure: import("./provider-error.js").ProviderFailure | null;
|
|
40
|
+
toolCalls: number;
|
|
41
|
+
toolErrors: {
|
|
42
|
+
tool: string;
|
|
43
|
+
reason: string;
|
|
44
|
+
detail?: string;
|
|
45
|
+
}[];
|
|
46
|
+
calls: ({
|
|
47
|
+
inputTokens: number;
|
|
48
|
+
outputTokens: number;
|
|
49
|
+
cachedInputTokens: null;
|
|
50
|
+
reservedUsd: number;
|
|
51
|
+
meteredUsd: number | null;
|
|
52
|
+
model: "gpt-5.4-2026-03-05" | "gpt-6-luna" | "gpt-5.6-sol" | "claude-opus-5-5" | "claude-sonnet-5" | "claude-haiku-4-5-20251001" | "claude-fable-5-1" | "cohere/north-mini-code:free" | "deepseek/deepseek-v3.2" | "anthropic/claude-sonnet-5" | "openai/gpt-6-luna";
|
|
53
|
+
purpose?: never;
|
|
54
|
+
} | {
|
|
55
|
+
purpose: string;
|
|
56
|
+
inputTokens: number;
|
|
57
|
+
outputTokens: number;
|
|
58
|
+
cachedInputTokens: null;
|
|
59
|
+
reservedUsd: number;
|
|
60
|
+
meteredUsd: number | null;
|
|
61
|
+
model: "gpt-5.4-2026-03-05" | "gpt-6-luna" | "gpt-5.6-sol" | "claude-opus-5-5" | "claude-sonnet-5" | "claude-haiku-4-5-20251001" | "claude-fable-5-1" | "cohere/north-mini-code:free" | "deepseek/deepseek-v3.2" | "anthropic/claude-sonnet-5" | "openai/gpt-6-luna";
|
|
62
|
+
})[];
|
|
63
|
+
};
|
|
64
|
+
export declare function runClaimReviewAsResult(directory: string, profile: Profile, signal?: AbortSignal, injectedModel?: Model): Promise<ClaimReview>;
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { charges, modelFor, runClaimReview } from './claim-run.js';
|
|
4
|
+
import { titleClaims } from './titles.js';
|
|
5
|
+
import { parseLocation } from './claim.js';
|
|
6
|
+
import { parseResult } from './contracts.js';
|
|
7
|
+
import { git, loadReview, validateAnchor } from './snapshot.js';
|
|
8
|
+
// The claim pipeline, written into the Result and receipt shapes the GitHub worker and
|
|
9
|
+
// the `review` command already publish from. This is the bridge that makes what ships
|
|
10
|
+
// the pipeline that was measured (2026-09-24: Luna, rung 3 on, P3 withheld) without
|
|
11
|
+
// rewriting publication, freshness, checks or inline comments, which do not depend on
|
|
12
|
+
// how the findings were produced.
|
|
13
|
+
const CATEGORY = {
|
|
14
|
+
injection_risk: 'security', hardcoded_secret: 'security', auth_bypass: 'security',
|
|
15
|
+
race_condition: 'reliability', data_loss: 'data-integrity', contract_break: 'correctness',
|
|
16
|
+
error_handling_gap: 'reliability', resource_leak: 'reliability',
|
|
17
|
+
};
|
|
18
|
+
const clip = (value, limit = 16000) => value.length <= limit ? value : `${value.slice(0, limit - 1)}…`;
|
|
19
|
+
export function claimResult(packet, repository, run) {
|
|
20
|
+
const byId = new Map(run.claims.map(claim => [claim.claimId, claim]));
|
|
21
|
+
const changed = new Map(packet.changedFiles.map(file => [file.path, file]));
|
|
22
|
+
const completed = run.complete && run.stopReason === 'finished';
|
|
23
|
+
const evidence = [];
|
|
24
|
+
const findings = [];
|
|
25
|
+
const limitations = [...run.limitations];
|
|
26
|
+
// The whole diff is in the claim pass's first turn (under the 128 KB limit), so every
|
|
27
|
+
// changed text file was in front of the model. That is what "reviewed" means here, and
|
|
28
|
+
// only for a run that finished: a run that stopped early claims no coverage at all.
|
|
29
|
+
const text = packet.changedFiles.filter(file => file.kind === 'text');
|
|
30
|
+
if (completed && text.length)
|
|
31
|
+
evidence.push({ id: 'change-diff', kind: 'source-reasoning', provenance: 'declared',
|
|
32
|
+
summary: 'The complete change diff for this path was in the claim pass context.',
|
|
33
|
+
anchors: text.map(file => ({ path: file.path, side: file.change === 'deleted' ? 'base' : 'head', line: null })) });
|
|
34
|
+
const outside = [];
|
|
35
|
+
// One finding per line: two confirmed claims on the same line are one defect seen twice
|
|
36
|
+
// (the main and failure-path passes word it differently, so their claim ids differ).
|
|
37
|
+
// The most severe is shown and carries the other's evidence; the summary counts the merge.
|
|
38
|
+
const onLine = new Map();
|
|
39
|
+
let merged = 0;
|
|
40
|
+
const confirmed = run.chains.filter(chain => chain.verdict === 'confirmed')
|
|
41
|
+
.sort((a, b) => (byId.get(a.claimId)?.severity ?? 'P4').localeCompare(byId.get(b.claimId)?.severity ?? 'P4'));
|
|
42
|
+
confirmed.forEach((chain, index) => {
|
|
43
|
+
const claim = byId.get(chain.claimId);
|
|
44
|
+
if (!claim)
|
|
45
|
+
return;
|
|
46
|
+
let anchor;
|
|
47
|
+
try {
|
|
48
|
+
const { path, line } = parseLocation(claim.location);
|
|
49
|
+
anchor = { path, side: changed.get(path)?.change === 'deleted' ? 'base' : 'head', line };
|
|
50
|
+
validateAnchor(repository, packet, anchor);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
anchor = undefined;
|
|
54
|
+
}
|
|
55
|
+
// A finding may sit outside the changed files, as a caller the change broke, but its
|
|
56
|
+
// line must exist. A confirmed claim whose location does not resolve is still
|
|
57
|
+
// reported, as a limitation, rather than dropped.
|
|
58
|
+
if (!anchor) {
|
|
59
|
+
outside.push(`${claim.severity} ${claim.type} at ${claim.location}: ${claim.description}`);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const id = `claim-${index + 1}`;
|
|
63
|
+
const established = chain.propositions.filter(item => item.status === 'established');
|
|
64
|
+
evidence.push({ id, kind: 'source-reasoning', provenance: 'declared', anchors: [anchor],
|
|
65
|
+
summary: clip(`Verified at ${chain.verifierConfidence} confidence. ${established.length} of ${chain.propositions.length} proposition(s) established: `
|
|
66
|
+
+ established.map(item => `${item.proposition} [${item.settledBy ?? 'unsettled'}]`).join('; ')) });
|
|
67
|
+
const same = onLine.get(`${anchor.path}:${anchor.line}`);
|
|
68
|
+
if (same) {
|
|
69
|
+
same.evidenceIds.push(id);
|
|
70
|
+
merged++;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const finding = { id, priority: claim.severity, kind: claim.severity === 'P4' ? 'improvement' : 'defect',
|
|
74
|
+
category: CATEGORY[claim.type], title: clip(run.titles?.[claim.claimId] ?? claim.description, 400), trigger: clip(claim.suspectedCondition),
|
|
75
|
+
consequence: clip(claim.description),
|
|
76
|
+
priorityReason: `Rated ${claim.severity} by the reviewer when it made the claim; type ${claim.type}.`,
|
|
77
|
+
counterEvidence: clip(`Each proposition was checked against the frozen revision: ${chain.propositions
|
|
78
|
+
.map(item => `${item.proposition} (${item.status})`).join('; ')}`),
|
|
79
|
+
...(claim.shouldBe ? { suggestion: clip(claim.shouldBe.text) } : {}),
|
|
80
|
+
anchor, evidenceIds: [id] };
|
|
81
|
+
findings.push(finding);
|
|
82
|
+
onLine.set(`${anchor.path}:${anchor.line}`, finding);
|
|
83
|
+
});
|
|
84
|
+
for (const item of outside)
|
|
85
|
+
limitations.push(clip(`Confirmed, but its location does not resolve at this revision: ${item}`));
|
|
86
|
+
const withheld = run.chains.filter(chain => chain.verdict === 'withheld').map(chain => byId.get(chain.claimId)?.location).filter(Boolean);
|
|
87
|
+
if (withheld.length)
|
|
88
|
+
limitations.push(`${withheld.length} confirmed minor (P3) finding(s) withheld: ${withheld.join(', ')}.`);
|
|
89
|
+
const counts = { refuted: 0, inconclusive: 0 };
|
|
90
|
+
for (const chain of run.chains)
|
|
91
|
+
if (chain.verdict === 'refuted' || chain.verdict === 'inconclusive')
|
|
92
|
+
counts[chain.verdict]++;
|
|
93
|
+
const result = {
|
|
94
|
+
schemaVersion: 1, headSha: packet.headSha, baseSha: packet.baseSha, policyHash: packet.policyHash,
|
|
95
|
+
status: completed ? 'completed' : 'partial',
|
|
96
|
+
reviewer: { name: 'atmin claim review', model: run.model, context: 'independent' },
|
|
97
|
+
summary: `${run.scope ? `${run.scope} ` : ''}Claim review: ${run.claims.length} claim(s) checked, ${findings.length + outside.length} confirmed and shown, ${merged ? `${merged} merged into a finding on the same line, ` : ''}${withheld.length} withheld as minor, `
|
|
98
|
+
+ `${counts.refuted} refuted, ${counts.inconclusive} inconclusive. Policy verdict ${run.verdict} (rule ${run.rule}).`,
|
|
99
|
+
coverage: packet.changedFiles.map(file => ({ path: file.path,
|
|
100
|
+
status: completed && file.kind === 'text' ? 'reviewed' : 'unreviewed', evidenceIds: completed && file.kind === 'text' ? ['change-diff'] : [] })),
|
|
101
|
+
validation: packet.policy.requiredChecks.map(name => ({ name, status: 'not-run', reason: 'Required validation has not run.', evidenceIds: [] })),
|
|
102
|
+
evidence, findings,
|
|
103
|
+
limitations: limitations.length ? limitations.map(item => clip(item)) : ['No repository code was executed.'],
|
|
104
|
+
};
|
|
105
|
+
return parseResult(result);
|
|
106
|
+
}
|
|
107
|
+
export function incrementalScope(directory, packet) {
|
|
108
|
+
const path = join(directory, 'previous.json');
|
|
109
|
+
if (!existsSync(path))
|
|
110
|
+
return { note: null };
|
|
111
|
+
const previous = JSON.parse(readFileSync(path, 'utf8'));
|
|
112
|
+
const sha = /^[a-f0-9]{40}$/;
|
|
113
|
+
if (!sha.test(previous.headSha) || !sha.test(previous.mergeBaseSha) || !Array.isArray(previous.claims)) {
|
|
114
|
+
return { note: 'Full review: the earlier review record was unreadable.' };
|
|
115
|
+
}
|
|
116
|
+
if (previous.mergeBaseSha !== packet.mergeBaseSha)
|
|
117
|
+
return { note: 'Full review: the merge base moved since the earlier review.' };
|
|
118
|
+
const repository = join(directory, 'source.git');
|
|
119
|
+
try {
|
|
120
|
+
git(repository, ['merge-base', '--is-ancestor', previous.headSha, packet.headSha]);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
return { note: 'Full review: the earlier reviewed head is not an ancestor of this one, as after a force-push.' };
|
|
124
|
+
}
|
|
125
|
+
const diff = git(repository, ['diff', '--no-ext-diff', '--no-textconv', '--no-renames', previous.headSha, packet.headSha, '--']);
|
|
126
|
+
const changed = git(repository, ['diff', '--no-ext-diff', '--no-textconv', '--no-renames', '--name-only', '-z', previous.headSha, packet.headSha, '--'])
|
|
127
|
+
.toString('utf8').split('\0').filter(Boolean);
|
|
128
|
+
const titles = Object.fromEntries(Object.entries(previous.titles ?? {}).filter(([, title]) => typeof title === 'string'));
|
|
129
|
+
return { scope: { since: previous.headSha, diff, carried: previous.claims, changed }, titles,
|
|
130
|
+
note: `Incremental review: new claims were sought only in the commits since ${previous.headSha.slice(0, 12)}; ${previous.claims.length} earlier finding(s) were re-checked against this head.` };
|
|
131
|
+
}
|
|
132
|
+
// Read from a finished claim run: only a completed review can stand in for the part of the
|
|
133
|
+
// change this one will not re-read, and only its confirmed and withheld claims carry.
|
|
134
|
+
export function previousReview(artifact) {
|
|
135
|
+
try {
|
|
136
|
+
const read = (name) => JSON.parse(readFileSync(join(artifact, name), 'utf8'));
|
|
137
|
+
const packet = read('packet.json'), result = read('result.json');
|
|
138
|
+
if (result.status !== 'completed' || !existsSync(join(artifact, 'claim-verification.json')))
|
|
139
|
+
return null;
|
|
140
|
+
const chains = (read('claim-verification.json').chains ?? []);
|
|
141
|
+
const kept = new Set(chains.filter(chain => chain.verdict === 'confirmed' || chain.verdict === 'withheld').map(chain => chain.claimId));
|
|
142
|
+
const claims = [...(existsSync(join(artifact, 'carried-claims.json')) ? read('carried-claims.json') : []), ...read('claims.json')];
|
|
143
|
+
const titles = existsSync(join(artifact, 'titles.json')) ? (read('titles.json').titles ?? {}) : {};
|
|
144
|
+
return { headSha: packet.headSha, mergeBaseSha: packet.mergeBaseSha, claims: claims.filter(claim => kept.has(claim.claimId)),
|
|
145
|
+
titles: Object.fromEntries(Object.entries(titles).filter(([id]) => kept.has(id))) };
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// The fields the dashboard reads from a receipt, and nothing it would misread. Spend is
|
|
152
|
+
// metered only when every request's charge is known, stopped runs included; otherwise it
|
|
153
|
+
// holds a reservation, and an unknown spend must not be shown as a known one.
|
|
154
|
+
export function claimReceipt(profile, review, startedAt, finishedAt, titling) {
|
|
155
|
+
const { investigation } = review;
|
|
156
|
+
return { schemaVersion: 1, pipeline: 'claims', profile, startedAt, finishedAt, stopReason: investigation.stopReason,
|
|
157
|
+
providerFailure: investigation.telemetry.failure, toolCalls: investigation.telemetry.toolCalls, toolErrors: investigation.toolErrors,
|
|
158
|
+
calls: [{ inputTokens: investigation.telemetry.inputTokens, outputTokens: investigation.telemetry.outputTokens, cachedInputTokens: null,
|
|
159
|
+
reservedUsd: investigation.spentUsd, meteredUsd: investigation.unsettledCalls ? null : investigation.spentUsd, model: profile.model },
|
|
160
|
+
// The titling call, when one was made: metered only when its charge is known.
|
|
161
|
+
...(titling && (titling.spentUsd || titling.failure) ? [{ purpose: 'titles', inputTokens: titling.inputTokens, outputTokens: titling.outputTokens,
|
|
162
|
+
cachedInputTokens: null, reservedUsd: titling.spentUsd, meteredUsd: titling.unsettled ? null : titling.spentUsd, model: profile.model }] : [])] };
|
|
163
|
+
}
|
|
164
|
+
// One claim review over a prepared snapshot, leaving result.json and receipt.json where
|
|
165
|
+
// the older engine left them. Rung 3 is Jev when TYPESAFE_API_KEY is set, which is the
|
|
166
|
+
// configuration measured; without it the run says so rather than failing.
|
|
167
|
+
export async function runClaimReviewAsResult(directory, profile, signal, injectedModel) {
|
|
168
|
+
const { packet, result: initial } = loadReview(directory);
|
|
169
|
+
// One run per snapshot, as with the older engine: a rerun must not reset a spend.
|
|
170
|
+
if (initial.status !== 'not-started' || existsSync(join(directory, 'claims.json'))) {
|
|
171
|
+
throw new Error('Run has existing investigation or spending evidence; prepare a new snapshot');
|
|
172
|
+
}
|
|
173
|
+
const jev = Boolean(process.env.TYPESAFE_API_KEY);
|
|
174
|
+
const deadline = AbortSignal.timeout(profile.deadlineMs);
|
|
175
|
+
const startedAt = new Date().toISOString();
|
|
176
|
+
const { scope, note, titles: earlierTitles = {} } = incrementalScope(directory, packet);
|
|
177
|
+
const review = await runClaimReview(directory, profile, injectedModel, signal ? AbortSignal.any([signal, deadline]) : deadline, undefined, jev ? 'jev' : 'none', {}, {}, scope);
|
|
178
|
+
// The claim record keeps its own name, because verification.json belongs to the local
|
|
179
|
+
// fix checks the worker may run next, which would overwrite it.
|
|
180
|
+
renameSync(join(directory, 'verification.json'), join(directory, 'claim-verification.json'));
|
|
181
|
+
// Titles for the confirmed findings only, from what the claim pass left of the budget.
|
|
182
|
+
const confirmed = new Set(review.verification.chains.filter(chain => chain.verdict === 'confirmed').map(chain => chain.claimId));
|
|
183
|
+
const titling = await titleClaims(review.claims.filter(claim => confirmed.has(claim.claimId) && !earlierTitles[claim.claimId]), modelFor(profile, injectedModel), charges(profile).costOf, charges(profile).charged, profile.maxUsd - review.investigation.spentUsd, signal ? AbortSignal.any([signal, deadline]) : deadline);
|
|
184
|
+
const persist = (name, value) => {
|
|
185
|
+
const temporary = join(directory, `${name}.pending`);
|
|
186
|
+
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx', flush: true });
|
|
187
|
+
renameSync(temporary, join(directory, name));
|
|
188
|
+
};
|
|
189
|
+
titling.titles = { ...Object.fromEntries(Object.entries(earlierTitles).filter(([id]) => confirmed.has(id))), ...titling.titles };
|
|
190
|
+
persist('titles.json', titling);
|
|
191
|
+
persist('receipt.json', claimReceipt(profile, review, startedAt, new Date().toISOString(), titling));
|
|
192
|
+
const limitations = [...review.investigation.limitations, ...review.verification.limitations];
|
|
193
|
+
if (!jev)
|
|
194
|
+
limitations.push('The cross-family rung was off (no TYPESAFE_API_KEY), so no finding reached high confidence through agreement.');
|
|
195
|
+
if (note)
|
|
196
|
+
limitations.push(note);
|
|
197
|
+
if (titling.failure)
|
|
198
|
+
limitations.push(`Finding titles were not written (${titling.failure}), so each finding is titled by its full description.`);
|
|
199
|
+
persist('result.json', claimResult(packet, join(directory, 'source.git'), {
|
|
200
|
+
claims: review.claims, chains: review.verification.chains, verdict: review.verification.decision.verdict,
|
|
201
|
+
rule: review.verification.decision.rule, limitations, complete: review.investigation.complete,
|
|
202
|
+
stopReason: review.investigation.stopReason, model: profile.model, titles: titling.titles,
|
|
203
|
+
...(scope ? { scope: `Incremental review since ${scope.since.slice(0, 12)}.` } : {})
|
|
204
|
+
}));
|
|
205
|
+
return review;
|
|
206
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { type ClaimInvestigation } from './investigator.js';
|
|
2
|
+
import { type CrossFamilyRung, type Verification, type VerifyOptions } from './lifecycle.js';
|
|
3
|
+
import { type RungContribution } from './ablation.js';
|
|
4
|
+
import { type Model, type ModelReply, type Profile } from './investigation.js';
|
|
5
|
+
import { type Claim } from './claim.js';
|
|
6
|
+
import type { Rung } from './evidence.js';
|
|
7
|
+
import type { Packet } from './contracts.js';
|
|
8
|
+
export interface ClaimReview {
|
|
9
|
+
claims: Claim[];
|
|
10
|
+
investigation: ClaimInvestigation;
|
|
11
|
+
verification: Verification;
|
|
12
|
+
}
|
|
13
|
+
export type CrossFamilySource = 'none' | 'jev';
|
|
14
|
+
export declare const MAX_DIFF_BYTES: number;
|
|
15
|
+
export interface IncrementalScope {
|
|
16
|
+
since: string;
|
|
17
|
+
diff: Buffer;
|
|
18
|
+
carried: Claim[];
|
|
19
|
+
changed: string[];
|
|
20
|
+
}
|
|
21
|
+
export declare function touchedBy(claim: Claim, changed: ReadonlySet<string>): boolean;
|
|
22
|
+
export declare const MAX_GUIDANCE_BYTES: number;
|
|
23
|
+
export declare function targetGuidance(repository: string, packet: Packet): {
|
|
24
|
+
guidance: {
|
|
25
|
+
path: string;
|
|
26
|
+
text: string;
|
|
27
|
+
}[];
|
|
28
|
+
omitted: string[];
|
|
29
|
+
};
|
|
30
|
+
export declare function charges(profile: Profile): {
|
|
31
|
+
costOf: (input: number, output: number) => number;
|
|
32
|
+
charged: (reply: ModelReply) => number | null;
|
|
33
|
+
};
|
|
34
|
+
export declare function modelFor(profile: Profile, injectedModel?: Model): Model;
|
|
35
|
+
export declare function runClaimReview(directory: string, profile: Profile, injectedModel?: Model, signal?: AbortSignal, crossFamily?: CrossFamilyRung, crossFamilySource?: CrossFamilySource, verify?: VerifyOptions, capture?: {
|
|
36
|
+
transcript?: boolean;
|
|
37
|
+
}, incremental?: IncrementalScope): Promise<ClaimReview>;
|
|
38
|
+
export declare function ablate(directory: string, rung?: Rung): RungContribution;
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { loadReview, sourceText } from './snapshot.js';
|
|
4
|
+
import { revisionFrom } from './symbolic.js';
|
|
5
|
+
import { investigateClaims } from './investigator.js';
|
|
6
|
+
import { verifyClaims } from './lifecycle.js';
|
|
7
|
+
import { contributionOf, recordedRung } from './ablation.js';
|
|
8
|
+
import { askJev } from './jev.js';
|
|
9
|
+
import { BALANCED } from './policy.js';
|
|
10
|
+
import { meteredCost, reservedCost, subscription } from './investigation.js';
|
|
11
|
+
import { openAIModel } from './openai-model.js';
|
|
12
|
+
import { openRouterModel } from './openrouter-model.js';
|
|
13
|
+
import { parseLocation } from './claim.js';
|
|
14
|
+
import { calledCode, MAX_CALLED_CODE_BYTES } from './callees.js';
|
|
15
|
+
import { failureExcerpt } from './failure-excerpt.js';
|
|
16
|
+
// The whole diff goes into every turn of the claim pass. 128 KB refused 29% of the PRs on
|
|
17
|
+
// the first real repository measured (mason-v1, 2026-09-24); 512 KB refuses 7%. A diff
|
|
18
|
+
// this size needs a profile whose input cap leaves room to read beside it: the OpenRouter
|
|
19
|
+
// adapter counts serialized bytes, not tokens, and the diff is escaped twice on the way,
|
|
20
|
+
// so a diff counts about 1.2 times its size (measured on a 471 KB mason-v1 diff). The
|
|
21
|
+
// production profile allows 1,000,000, under Luna's 1.05M-token window (`profiles/review-luna-openrouter.json`).
|
|
22
|
+
export const MAX_DIFF_BYTES = 512 * 1024;
|
|
23
|
+
// A carried claim is re-verified on the propositions it was recorded with, and a fix can
|
|
24
|
+
// remove a premise none of them states. Measured 2026-09-24 on a real push (mason-v1):
|
|
25
|
+
// the fix wrapped a read-then-delete in an advisory lock, every recorded proposition (the
|
|
26
|
+
// read, the comparison, the delete by name) stayed true, and the fixed race was confirmed
|
|
27
|
+
// again. So a claim whose file the push changed, or a file one of its checks names, is not
|
|
28
|
+
// carried on its own: the model is shown it and records it again only if it still holds.
|
|
29
|
+
// A fix made in a file the claim never names still escapes this.
|
|
30
|
+
export function touchedBy(claim, changed) {
|
|
31
|
+
const checked = claim.evidenceToCheck.flatMap(item => item.check && 'path' in item.check ? [item.check.path] : []);
|
|
32
|
+
return [parseLocation(claim.location).path, ...checked].some(path => changed.has(path));
|
|
33
|
+
}
|
|
34
|
+
// The target's own AGENTS.md files: the root one and one in each directory above a changed
|
|
35
|
+
// path, read from the target branch so a change cannot rewrite the rules it is reviewed
|
|
36
|
+
// against. The older engine loaded these (investigation.ts); the claim pipeline had not,
|
|
37
|
+
// and on mason-v1 #4590 (2026-09-24) 5 of CodeRabbit's 12 items came from house rules the
|
|
38
|
+
// claim pass was never shown. Shallowest first, so the root rules are the last to be cut;
|
|
39
|
+
// a file that does not fit is named, never silently dropped.
|
|
40
|
+
export const MAX_GUIDANCE_BYTES = 32 * 1024;
|
|
41
|
+
export function targetGuidance(repository, packet) {
|
|
42
|
+
const paths = new Set(['AGENTS.md']);
|
|
43
|
+
for (const { path } of packet.changedFiles) {
|
|
44
|
+
const parts = path.split('/');
|
|
45
|
+
parts.pop();
|
|
46
|
+
for (; parts.length; parts.pop())
|
|
47
|
+
paths.add(`${parts.join('/')}/AGENTS.md`);
|
|
48
|
+
}
|
|
49
|
+
const guidance = [];
|
|
50
|
+
const omitted = [];
|
|
51
|
+
let bytes = 0;
|
|
52
|
+
for (const path of [...paths].sort((a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b))) {
|
|
53
|
+
const text = sourceText(repository, packet.baseSha, path);
|
|
54
|
+
if (text === null)
|
|
55
|
+
continue;
|
|
56
|
+
const size = Buffer.byteLength(text);
|
|
57
|
+
if (bytes + size > MAX_GUIDANCE_BYTES) {
|
|
58
|
+
omitted.push(path);
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
bytes += size;
|
|
62
|
+
guidance.push({ path, text });
|
|
63
|
+
}
|
|
64
|
+
return { guidance, omitted };
|
|
65
|
+
}
|
|
66
|
+
// Both passes as one investigation: a claim either recorded counts once, the run is
|
|
67
|
+
// complete only if both were, and the second pass's limitations say which pass they are from.
|
|
68
|
+
function combined(main, focus, sites) {
|
|
69
|
+
const ids = new Set(main.claims.map(claim => claim.claimId));
|
|
70
|
+
const byName = { ...main.telemetry.toolCallsByName };
|
|
71
|
+
for (const [name, count] of Object.entries(focus.telemetry.toolCallsByName))
|
|
72
|
+
byName[name] = (byName[name] ?? 0) + count;
|
|
73
|
+
const transcript = main.transcript && focus.transcript ? [...main.transcript, ...focus.transcript] : undefined;
|
|
74
|
+
return {
|
|
75
|
+
claims: [...main.claims, ...focus.claims.filter(claim => !ids.has(claim.claimId))],
|
|
76
|
+
complete: main.complete && focus.complete,
|
|
77
|
+
limitations: [...main.limitations, ...focus.limitations.map(line => `Failure-path pass: ${line}`)],
|
|
78
|
+
toolErrors: [...main.toolErrors, ...focus.toolErrors],
|
|
79
|
+
stopReason: main.stopReason === 'finished' ? focus.stopReason : main.stopReason,
|
|
80
|
+
spentUsd: main.spentUsd + focus.spentUsd,
|
|
81
|
+
unsettledCalls: main.unsettledCalls + focus.unsettledCalls,
|
|
82
|
+
telemetry: {
|
|
83
|
+
turns: main.telemetry.turns + focus.telemetry.turns,
|
|
84
|
+
toolCalls: main.telemetry.toolCalls + focus.telemetry.toolCalls,
|
|
85
|
+
toolCallsByName: byName,
|
|
86
|
+
droppedTurns: main.telemetry.droppedTurns + focus.telemetry.droppedTurns,
|
|
87
|
+
inputTokens: main.telemetry.inputTokens + focus.telemetry.inputTokens,
|
|
88
|
+
cachedInputTokens: main.telemetry.cachedInputTokens + focus.telemetry.cachedInputTokens,
|
|
89
|
+
outputTokens: main.telemetry.outputTokens + focus.telemetry.outputTokens,
|
|
90
|
+
finishReason: focus.telemetry.finishReason ?? main.telemetry.finishReason,
|
|
91
|
+
failure: main.telemetry.failure ?? focus.telemetry.failure,
|
|
92
|
+
reads: [...main.telemetry.reads, ...focus.telemetry.reads],
|
|
93
|
+
retries: [...main.telemetry.retries, ...focus.telemetry.retries],
|
|
94
|
+
failurePathClaimIds: focus.claims.map(claim => claim.claimId),
|
|
95
|
+
failurePathSpentUsd: focus.spentUsd,
|
|
96
|
+
failurePathTurns: focus.telemetry.turns,
|
|
97
|
+
failurePathSites: sites,
|
|
98
|
+
},
|
|
99
|
+
...(transcript ? { transcript } : {}),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
const idle = () => ({ claims: [], complete: true, limitations: [], toolErrors: [], stopReason: 'finished',
|
|
103
|
+
spentUsd: 0, unsettledCalls: 0, telemetry: { turns: 0, toolCalls: 0, toolCallsByName: {}, droppedTurns: 0, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0,
|
|
104
|
+
finishReason: null, failure: null, reads: [], retries: [] } });
|
|
105
|
+
// Reservations use the rate card at its dearest; settlement uses what the call cost.
|
|
106
|
+
export function charges(profile) {
|
|
107
|
+
return {
|
|
108
|
+
costOf: (input, output) => reservedCost(profile, input, output),
|
|
109
|
+
charged: (reply) => meteredCost(profile, reply),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
// The model a profile names, unless a test or benchmark injects one.
|
|
113
|
+
export function modelFor(profile, injectedModel) {
|
|
114
|
+
if (subscription(profile) && !injectedModel) {
|
|
115
|
+
throw new Error('Local subscription experiments require a benchmark adapter (Codex or Claude); hosted execution is not supported');
|
|
116
|
+
}
|
|
117
|
+
return injectedModel ?? (profile.provider === 'openrouter' ? openRouterModel(profile) : openAIModel(profile));
|
|
118
|
+
}
|
|
119
|
+
export async function runClaimReview(directory, profile, injectedModel, signal, crossFamily, crossFamilySource = 'none', verify = {}, capture = {}, incremental) {
|
|
120
|
+
const { packet } = loadReview(directory);
|
|
121
|
+
const model = modelFor(profile, injectedModel);
|
|
122
|
+
const repository = join(directory, 'source.git');
|
|
123
|
+
// Both sides, because a claim about a regression is a claim about the difference.
|
|
124
|
+
const revisions = { head: revisionFrom(repository, packet.headSha), base: revisionFrom(repository, packet.mergeBaseSha) };
|
|
125
|
+
const sourceOf = (path) => sourceText(repository, packet.headSha, path);
|
|
126
|
+
const full = readFileSync(join(directory, 'change.diff'));
|
|
127
|
+
const diff = incremental ? incremental.diff : full;
|
|
128
|
+
if (diff.length > MAX_DIFF_BYTES)
|
|
129
|
+
throw new Error('Diff exceeds 512 KB investigation limit');
|
|
130
|
+
const changed = new Set(incremental?.changed ?? []);
|
|
131
|
+
const recheck = (incremental?.carried ?? []).filter(claim => touchedBy(claim, changed));
|
|
132
|
+
const { guidance, omitted } = targetGuidance(repository, packet);
|
|
133
|
+
const { called, omitted: uncalled } = calledCode(diff.toString('utf8'), revisions.head);
|
|
134
|
+
const guided = { ...(guidance.length ? { targetGuidance: guidance } : {}), ...(called.length ? { calledCode: called } : {}) };
|
|
135
|
+
const context = incremental
|
|
136
|
+
? { packet, ...guided, diff: diff.toString('utf8'), incrementalSince: incremental.since,
|
|
137
|
+
scope: `This diff holds only the commits pushed since an earlier review at ${incremental.since}. The whole change is listed in packet.changedFiles, and the earlier review's other findings are re-checked separately. Claim defects that these new commits introduce or expose; revision "base" still means the merge base.`
|
|
138
|
+
+ (recheck.length ? ' earlierFindings are findings from that review in files these commits changed. They are not carried forward on their own: read the new code, and record again, with the same type, location symbol and suspectedCondition, each one that still holds at the new head. Leave out any the new commits fixed.' : ''),
|
|
139
|
+
...(recheck.length ? { earlierFindings: recheck.map(({ type, location, description, suspectedCondition }) => ({ type, location, description, suspectedCondition })) } : {}) }
|
|
140
|
+
: { packet, ...guided, diff: diff.toString('utf8') };
|
|
141
|
+
const limits = (maxUsd) => ({
|
|
142
|
+
maxTurns: profile.maxTurns, maxToolCalls: profile.maxToolCalls,
|
|
143
|
+
maxInputTokens: profile.maxInputTokens, maxOutputTokens: profile.maxOutputTokens,
|
|
144
|
+
maxUsd, ...(capture.transcript ? { recordTranscript: true } : {}), ...charges(profile),
|
|
145
|
+
});
|
|
146
|
+
// Nothing new to read, as when only the target branch moved: no model is asked.
|
|
147
|
+
const main = incremental && !diff.length ? idle() : await investigateClaims(revisions, sourceOf, context, model, limits(profile.maxUsd), signal);
|
|
148
|
+
// A second pass on failure paths alone, within what the first left of the budget, shown
|
|
149
|
+
// only the diff around error handling; a change with none gets no second pass. See
|
|
150
|
+
// failurePathInstruction for why it is a pass of its own, failureExcerpt for the cost.
|
|
151
|
+
const failure = failureExcerpt(diff.toString('utf8'));
|
|
152
|
+
const failurePaths = !failure.excerpt ? idle()
|
|
153
|
+
: await investigateClaims(revisions, sourceOf, { ...context, diff: failure.excerpt }, model, { ...limits(profile.maxUsd - main.spentUsd), focus: 'failure_paths' }, signal);
|
|
154
|
+
const investigation = combined(main, failurePaths, failure.sites);
|
|
155
|
+
// Earlier claims first, so a claim the model records again keeps the earlier wording
|
|
156
|
+
// and checks; the verifier is then handed one list and cannot tell them apart.
|
|
157
|
+
const emitted = new Set(investigation.claims.map(claim => claim.claimId));
|
|
158
|
+
const carried = (incremental?.carried ?? []).filter(claim => !emitted.has(claim.claimId) && !touchedBy(claim, changed));
|
|
159
|
+
if (uncalled.length)
|
|
160
|
+
investigation.limitations.push(`Code the change calls over ${MAX_CALLED_CODE_BYTES / 1024} KB was left out, so these definitions were not shown: ${uncalled.join(', ')}.`);
|
|
161
|
+
if (omitted.length)
|
|
162
|
+
investigation.limitations.push(`Repository guidance over ${MAX_GUIDANCE_BYTES / 1024} KB was left out, so these rules were not shown: ${omitted.join(', ')}.`);
|
|
163
|
+
if (recheck.length)
|
|
164
|
+
investigation.limitations.push(`${recheck.length} earlier finding(s) were in files this push changed, so they were re-asked rather than carried; ${recheck.filter(claim => emitted.has(claim.claimId)).length} were recorded again.`);
|
|
165
|
+
const claims = [...carried, ...investigation.claims];
|
|
166
|
+
const persist = (name, value) => {
|
|
167
|
+
const temporary = join(directory, `${name}.pending`);
|
|
168
|
+
writeFileSync(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx', flush: true });
|
|
169
|
+
renameSync(temporary, join(directory, name));
|
|
170
|
+
};
|
|
171
|
+
persist('claims.json', investigation.claims);
|
|
172
|
+
if (incremental)
|
|
173
|
+
persist('carried-claims.json', carried);
|
|
174
|
+
// Opt-in, for the emission bench only: this is provider and repository text.
|
|
175
|
+
if (investigation.transcript)
|
|
176
|
+
persist('transcript.json', investigation.transcript);
|
|
177
|
+
// Written as its own file rather than folded into the verification, because it is about
|
|
178
|
+
// the run and not about any claim, and because a run that emits nothing still needs to
|
|
179
|
+
// leave an explanation behind. Counts and allowlisted enums only -- see ClaimTelemetry.
|
|
180
|
+
// Both are written before verification, because a verification that throws must not
|
|
181
|
+
// take the claims and the spend down with it: seen 2026-09-23, when a grep overflow did
|
|
182
|
+
// exactly that on a live run.
|
|
183
|
+
persist('telemetry.json', { ...investigation.telemetry, stopReason: investigation.stopReason,
|
|
184
|
+
complete: investigation.complete, claims: investigation.claims.length, spentUsd: investigation.spentUsd,
|
|
185
|
+
toolErrors: investigation.toolErrors, limitations: investigation.limitations });
|
|
186
|
+
// The verifier is handed the claims and the revision, and nothing else. Whatever the
|
|
187
|
+
// investigator believed does not travel with them.
|
|
188
|
+
let rung = crossFamily;
|
|
189
|
+
// A claim whose Jev call fails is skipped, which the lifecycle reads as unsettled, and
|
|
190
|
+
// that is the honest outcome. What was not honest was saying nothing about it: seen
|
|
191
|
+
// 2026-09-21, a run asked zero questions where seven claims needed them, every claim
|
|
192
|
+
// came out inconclusive, and the report gave no hint that the rung had never answered.
|
|
193
|
+
// The count is reported and the reason is not, because provider text can carry
|
|
194
|
+
// repository content.
|
|
195
|
+
let unreached = [];
|
|
196
|
+
if (!rung && crossFamilySource === 'jev' && claims.length) {
|
|
197
|
+
const asked = await askJev(claims, revisions, full.toString('utf8'), { signal, verify });
|
|
198
|
+
rung = recordedRung(asked.log);
|
|
199
|
+
unreached = [...new Set(asked.skippedClaims)];
|
|
200
|
+
}
|
|
201
|
+
const verification = verifyClaims(claims, revisions, BALANCED, rung, undefined, verify);
|
|
202
|
+
if (unreached.length) {
|
|
203
|
+
verification.limitations.push(unreached.length === claims.length
|
|
204
|
+
? `The cross-family rung answered nothing: all ${unreached.length} claim(s) failed to reach it, so any proposition only it could settle is unsettled.`
|
|
205
|
+
: `The cross-family rung did not reach ${unreached.length} of ${claims.length} claim(s), so any proposition only it could settle is unsettled for those.`);
|
|
206
|
+
}
|
|
207
|
+
persist('verification.json', { ...verification, stopReason: investigation.stopReason, spentUsd: investigation.spentUsd });
|
|
208
|
+
return { claims, investigation, verification };
|
|
209
|
+
}
|
|
210
|
+
// The same claims, verified again with the cross-family rung switched off. Emission is
|
|
211
|
+
// not repeated and the model is not re-asked — the recorded answers are replayed — so
|
|
212
|
+
// the difference is the rung's contribution and nothing else.
|
|
213
|
+
export function ablate(directory, rung = 'cross_family_llm') {
|
|
214
|
+
const { packet } = loadReview(directory);
|
|
215
|
+
const repository = join(directory, 'source.git');
|
|
216
|
+
const revisions = { head: revisionFrom(repository, packet.headSha), base: revisionFrom(repository, packet.mergeBaseSha) };
|
|
217
|
+
const read = (name) => {
|
|
218
|
+
const bytes = readFileSync(join(directory, name));
|
|
219
|
+
if (bytes.length > 16_000_000)
|
|
220
|
+
throw new Error(`${name} exceeds the 16 MB replay limit`);
|
|
221
|
+
return JSON.parse(bytes.toString('utf8'));
|
|
222
|
+
};
|
|
223
|
+
const verification = read('verification.json');
|
|
224
|
+
if (!Array.isArray(verification.crossFamilyLog))
|
|
225
|
+
throw new Error('This run predates cross-family recording; re-run claim-review to measure the rung');
|
|
226
|
+
return contributionOf(rung, read('claims.json'), revisions, BALANCED, verification.crossFamilyLog);
|
|
227
|
+
}
|