argus-reviewer-e2e 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -49
- package/action/action.yml +155 -61
- package/action/approval-review.mjs +393 -0
- package/action/bootstrap.mjs +102 -0
- package/action/cli.mjs +36 -0
- package/action/emit-review.mjs +336 -0
- package/action/runtime.mjs +154 -0
- package/action/sticky-comment.cjs +870 -0
- package/dist/api.d.ts +2 -0
- package/dist/api.js +4 -0
- package/dist/cache/store.d.ts +2 -0
- package/dist/cache/store.js +10 -3
- package/dist/cli.d.ts +137 -0
- package/dist/cli.js +677 -60
- package/dist/config.d.ts +67 -2
- package/dist/config.js +138 -7
- package/dist/debug.d.ts +1 -0
- package/dist/debug.js +9 -3
- package/dist/engine/loop.d.ts +12 -0
- package/dist/engine/loop.js +55 -6
- package/dist/evidence/ci.d.ts +3 -0
- package/dist/evidence/ci.js +3 -1
- package/dist/pipeline/budget.d.ts +11 -0
- package/dist/pipeline/budget.js +48 -0
- package/dist/pipeline/contracts.d.ts +12 -0
- package/dist/pipeline/contracts.js +16 -0
- package/dist/pipeline/verify.d.ts +27 -0
- package/dist/pipeline/verify.js +197 -0
- package/dist/probe/queue.d.ts +4 -1
- package/dist/probe/queue.js +46 -8
- package/dist/report/manifest.d.ts +87 -0
- package/dist/report/manifest.js +143 -0
- package/dist/report/run.d.ts +9 -0
- package/dist/report/run.js +6 -0
- package/dist/review/adjudicate.d.ts +63 -0
- package/dist/review/adjudicate.js +111 -0
- package/dist/review/secrets.d.ts +90 -0
- package/dist/review/secrets.js +224 -0
- package/dist/review/triage.d.ts +76 -0
- package/dist/review/triage.js +163 -0
- package/dist/trust.d.ts +50 -0
- package/dist/trust.js +103 -0
- package/dist/vision/cost.d.ts +14 -1
- package/dist/vision/cost.js +13 -0
- package/dist/vision/decisions.d.ts +95 -0
- package/dist/vision/decisions.js +232 -0
- package/package.json +9 -8
- package/action/sticky-comment.mjs +0 -404
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type LaneId, type RunIdentity, type RunManifest } from '../report/manifest.js';
|
|
2
|
+
import { type BudgetOptions } from './budget.js';
|
|
3
|
+
import { type LaneSelection } from './contracts.js';
|
|
4
|
+
export interface VerifyRunners {
|
|
5
|
+
review: () => Promise<number>;
|
|
6
|
+
flow?: (url: string) => Promise<number>;
|
|
7
|
+
app?: () => Promise<number>;
|
|
8
|
+
a0?: () => Promise<number>;
|
|
9
|
+
}
|
|
10
|
+
export interface VerifyInput {
|
|
11
|
+
cwd: string;
|
|
12
|
+
runId: string;
|
|
13
|
+
reportDir: string;
|
|
14
|
+
identity: RunIdentity;
|
|
15
|
+
selection: LaneSelection;
|
|
16
|
+
runners: VerifyRunners;
|
|
17
|
+
budgets?: Partial<Record<LaneId, BudgetOptions>>;
|
|
18
|
+
flowUrl?: string;
|
|
19
|
+
flowUnavailableReason?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface VerifyResult {
|
|
22
|
+
manifest: RunManifest;
|
|
23
|
+
exitCode: number;
|
|
24
|
+
}
|
|
25
|
+
/** Run selected lanes and return one stable manifest without owning subprocesses. */
|
|
26
|
+
export declare function runVerify(input: VerifyInput): Promise<VerifyResult>;
|
|
27
|
+
export declare function selectedManifestLanes(selection: LaneSelection): LaneId[];
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join, relative } from 'node:path';
|
|
3
|
+
import { aggregateLanes, addProviderUsage, emptyLane, emptyUsage, isHeadBindingConclusive, MANIFEST_SCHEMA_VERSION, } from '../report/manifest.js';
|
|
4
|
+
import { addProviderCalls, createBudget } from './budget.js';
|
|
5
|
+
import { selectedLanes } from './contracts.js';
|
|
6
|
+
async function readJson(path) {
|
|
7
|
+
try {
|
|
8
|
+
return JSON.parse(await readFile(path, 'utf8'));
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function relativeReport(cwd, reportDir, name) {
|
|
15
|
+
return relative(cwd, join(reportDir, name)) || name;
|
|
16
|
+
}
|
|
17
|
+
function laneStart(lane, at = new Date()) {
|
|
18
|
+
return { ...lane, startedAt: at.toISOString() };
|
|
19
|
+
}
|
|
20
|
+
function laneEnd(lane, at = new Date()) {
|
|
21
|
+
return { ...lane, finishedAt: at.toISOString() };
|
|
22
|
+
}
|
|
23
|
+
function reportStatus(code, report, inconclusive) {
|
|
24
|
+
if (report?.skipped === true)
|
|
25
|
+
return 'skipped';
|
|
26
|
+
if (inconclusive)
|
|
27
|
+
return 'inconclusive';
|
|
28
|
+
if (report?.ok === true && code === 0)
|
|
29
|
+
return 'passed';
|
|
30
|
+
return 'failed';
|
|
31
|
+
}
|
|
32
|
+
function providerUsageFromCalls(calls) {
|
|
33
|
+
const usage = emptyUsage('openrouter');
|
|
34
|
+
return calls === undefined ? usage : addProviderUsage(usage, calls);
|
|
35
|
+
}
|
|
36
|
+
function flowCalls(report) {
|
|
37
|
+
return report?.tests?.flatMap((test) => test.calls ?? []) ?? [];
|
|
38
|
+
}
|
|
39
|
+
function flowUsage(report) {
|
|
40
|
+
const calls = flowCalls(report);
|
|
41
|
+
const usage = providerUsageFromCalls(calls);
|
|
42
|
+
if (report?.totals !== undefined && calls.length === 0) {
|
|
43
|
+
return {
|
|
44
|
+
...usage,
|
|
45
|
+
calls: report.totals.visionCalls ?? 0,
|
|
46
|
+
tokens: 0,
|
|
47
|
+
costUsd: report.totals.visionCostUsd ?? 0,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
return usage;
|
|
51
|
+
}
|
|
52
|
+
function budgetFor(lane, input, report) {
|
|
53
|
+
const options = input.budgets?.[lane] ?? {};
|
|
54
|
+
let budget = createBudget(lane, options);
|
|
55
|
+
if (lane === 'review') {
|
|
56
|
+
budget = addProviderCalls(budget, report?.calls);
|
|
57
|
+
}
|
|
58
|
+
else if (lane === 'flow') {
|
|
59
|
+
budget = addProviderCalls(budget, flowCalls(report));
|
|
60
|
+
}
|
|
61
|
+
return budget;
|
|
62
|
+
}
|
|
63
|
+
function baseLane(lane, selected) {
|
|
64
|
+
return emptyLane(lane, selected);
|
|
65
|
+
}
|
|
66
|
+
/** Run selected lanes and return one stable manifest without owning subprocesses. */
|
|
67
|
+
export async function runVerify(input) {
|
|
68
|
+
const startedAt = new Date();
|
|
69
|
+
const lanes = Object.fromEntries(['review', 'flow', 'app', 'a0'].map((lane) => [
|
|
70
|
+
lane,
|
|
71
|
+
baseLane(lane, input.selection[lane]),
|
|
72
|
+
]));
|
|
73
|
+
const reviewSelected = input.selection.review;
|
|
74
|
+
if (reviewSelected) {
|
|
75
|
+
lanes.review = laneStart(lanes.review, startedAt);
|
|
76
|
+
let code = 1;
|
|
77
|
+
let runnerError;
|
|
78
|
+
try {
|
|
79
|
+
code = await input.runners.review();
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
runnerError = error instanceof Error ? error.message : String(error);
|
|
83
|
+
ctxError(runnerError);
|
|
84
|
+
}
|
|
85
|
+
const report = await readJson(join(input.reportDir, 'code-review.json'));
|
|
86
|
+
const inconclusive = report !== undefined &&
|
|
87
|
+
report.skipped !== true &&
|
|
88
|
+
!isHeadBindingConclusive(report.headBinding);
|
|
89
|
+
const status = runnerError !== undefined ? 'failed' : reportStatus(code, report, inconclusive);
|
|
90
|
+
lanes.review = laneEnd({
|
|
91
|
+
...lanes.review,
|
|
92
|
+
status,
|
|
93
|
+
reportPath: report === undefined
|
|
94
|
+
? undefined
|
|
95
|
+
: relativeReport(input.cwd, input.reportDir, 'code-review.json'),
|
|
96
|
+
model: report?.model,
|
|
97
|
+
summary: report?.summary,
|
|
98
|
+
reason: runnerError ??
|
|
99
|
+
(report === undefined
|
|
100
|
+
? 'code-review.json was not produced'
|
|
101
|
+
: status === 'passed'
|
|
102
|
+
? undefined
|
|
103
|
+
: report.summary),
|
|
104
|
+
usage: providerUsageFromCalls(report?.calls),
|
|
105
|
+
budget: budgetFor('review', input, report),
|
|
106
|
+
headBinding: report?.headBinding,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
if (input.selection.flow) {
|
|
110
|
+
lanes.flow = laneStart(lanes.flow, startedAt);
|
|
111
|
+
if (input.runners.flow === undefined) {
|
|
112
|
+
lanes.flow = laneEnd({
|
|
113
|
+
...lanes.flow,
|
|
114
|
+
status: 'unavailable',
|
|
115
|
+
reason: 'flow runner is not available in this build',
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
else if (input.flowUrl === undefined || input.flowUrl === '') {
|
|
119
|
+
lanes.flow = laneEnd({
|
|
120
|
+
...lanes.flow,
|
|
121
|
+
status: 'blocked',
|
|
122
|
+
reason: input.flowUnavailableReason ?? 'no application target configured',
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
let code = 1;
|
|
127
|
+
let runnerError;
|
|
128
|
+
try {
|
|
129
|
+
code = await input.runners.flow(input.flowUrl);
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
runnerError = error instanceof Error ? error.message : String(error);
|
|
133
|
+
ctxError(runnerError);
|
|
134
|
+
}
|
|
135
|
+
const report = await readJson(join(input.reportDir, 'run.json'));
|
|
136
|
+
lanes.flow = laneEnd({
|
|
137
|
+
...lanes.flow,
|
|
138
|
+
status: runnerError !== undefined ? 'failed' : reportStatus(code, report, false),
|
|
139
|
+
reportPath: report === undefined ? undefined : relativeReport(input.cwd, input.reportDir, 'run.json'),
|
|
140
|
+
summary: report === undefined ? undefined : report.ok === true ? 'flow passed' : 'flow failed',
|
|
141
|
+
reason: runnerError ??
|
|
142
|
+
(report === undefined
|
|
143
|
+
? 'run.json was not produced'
|
|
144
|
+
: report.ok === true
|
|
145
|
+
? undefined
|
|
146
|
+
: 'flow failed'),
|
|
147
|
+
usage: flowUsage(report),
|
|
148
|
+
budget: budgetFor('flow', input, report),
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
for (const lane of ['app', 'a0']) {
|
|
153
|
+
if (!input.selection[lane])
|
|
154
|
+
continue;
|
|
155
|
+
const runner = input.runners[lane];
|
|
156
|
+
lanes[lane] = laneStart(lanes[lane], startedAt);
|
|
157
|
+
if (runner === undefined) {
|
|
158
|
+
lanes[lane] = laneEnd({
|
|
159
|
+
...lanes[lane],
|
|
160
|
+
status: 'unavailable',
|
|
161
|
+
reason: `${lane} runner is not available in this build`,
|
|
162
|
+
});
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
let code = 1;
|
|
166
|
+
let runnerError;
|
|
167
|
+
try {
|
|
168
|
+
code = await runner();
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
runnerError = error instanceof Error ? error.message : String(error);
|
|
172
|
+
ctxError(runnerError);
|
|
173
|
+
}
|
|
174
|
+
lanes[lane] = laneEnd({
|
|
175
|
+
...lanes[lane],
|
|
176
|
+
status: code === 0 ? 'passed' : 'failed',
|
|
177
|
+
reason: runnerError ?? (code === 0 ? undefined : `${lane} runner exited ${code}`),
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
const aggregate = aggregateLanes(lanes);
|
|
181
|
+
const manifest = {
|
|
182
|
+
schemaVersion: MANIFEST_SCHEMA_VERSION,
|
|
183
|
+
runId: input.runId,
|
|
184
|
+
startedAt: startedAt.toISOString(),
|
|
185
|
+
finishedAt: new Date().toISOString(),
|
|
186
|
+
identity: input.identity,
|
|
187
|
+
lanes,
|
|
188
|
+
aggregate,
|
|
189
|
+
};
|
|
190
|
+
return { manifest, exitCode: aggregate.ok ? 0 : 1 };
|
|
191
|
+
}
|
|
192
|
+
function ctxError(message) {
|
|
193
|
+
console.error(`verify lane error: ${message}`);
|
|
194
|
+
}
|
|
195
|
+
export function selectedManifestLanes(selection) {
|
|
196
|
+
return selectedLanes(selection);
|
|
197
|
+
}
|
package/dist/probe/queue.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type { RepoIndex } from '../index/scan.js';
|
|
|
6
6
|
import type { VisionClient } from '../engine/loop.js';
|
|
7
7
|
import type { CallCost } from '../vision/cost.js';
|
|
8
8
|
import type { Ledger } from '../vision/ledger.js';
|
|
9
|
+
import type { TriageAreaSignal } from '../review/triage.js';
|
|
9
10
|
import { type ProbeOutcome } from './harness.js';
|
|
10
11
|
/**
|
|
11
12
|
* The B.2 probe lane (U4). Called after `linkFindings` inside `code-review`.
|
|
@@ -69,13 +70,15 @@ export interface ProbeLaneOptions {
|
|
|
69
70
|
/** Configured blocking severities — the queue only admits those. */
|
|
70
71
|
severityGates: string[];
|
|
71
72
|
index: RepoIndex | undefined;
|
|
73
|
+
/** U9 — triage top_risk_area; advisory reorder of probe candidates. */
|
|
74
|
+
triageArea?: TriageAreaSignal | undefined;
|
|
72
75
|
/** Report sink — authored probe CallCosts are pushed here for the report. */
|
|
73
76
|
calls?: CallCost[] | undefined;
|
|
74
77
|
exec?: ExecFn | undefined;
|
|
75
78
|
log?: ((line: string) => void) | undefined;
|
|
76
79
|
}
|
|
77
80
|
/** Pure selection: not_exercised findings at blocking severities, capped. */
|
|
78
|
-
export declare function selectProbeTargets(findings: LinkedFinding[], severityGates: string[], maxProbes: number): LinkedFinding[];
|
|
81
|
+
export declare function selectProbeTargets(findings: LinkedFinding[], severityGates: string[], maxProbes: number, triageArea?: TriageAreaSignal): LinkedFinding[];
|
|
79
82
|
/**
|
|
80
83
|
* Repo-relative path gate for anything model- or index-derived that is read
|
|
81
84
|
* or written on the HOST: no absolute paths, no `..` escapes, no backslashes.
|
package/dist/probe/queue.js
CHANGED
|
@@ -7,14 +7,42 @@ import { isTestFile } from '../evidence/link.js';
|
|
|
7
7
|
import { buildProbeMessages, parseProbe, probeImportsSafe, PROBE_SCHEMA, } from './author.js';
|
|
8
8
|
import { detectHarness } from './harness.js';
|
|
9
9
|
import { checkSandboxPaths, dockerAvailable, resolveSandboxImage, runProbeInSandbox, SANDBOX_OUTPUT_CAP, SCRATCH_DIR_NAME, stripControlChars, sandboxLimits, } from '../executor/sandbox.js';
|
|
10
|
+
/** U9 — below this confidence the triage area signal is ignored. */
|
|
11
|
+
const MIN_AREA_CONFIDENCE = 0.5;
|
|
12
|
+
/**
|
|
13
|
+
* U9 triage-informed ordering: a finding's file path "hits" the flagged
|
|
14
|
+
* risk area when a path segment equals the area token or starts with
|
|
15
|
+
* it at a camelCase/digit boundary — `src/auth/session.ts` and
|
|
16
|
+
* `dataStore.ts` hit auth/data, while `author.ts` and `database.ts`
|
|
17
|
+
* do not. Advisory only.
|
|
18
|
+
*/
|
|
19
|
+
function fileHitsArea(file, area) {
|
|
20
|
+
const token = area.toLowerCase();
|
|
21
|
+
return file.split(/[/._-]+/).some((segment) => {
|
|
22
|
+
const lower = segment.toLowerCase();
|
|
23
|
+
if (lower === token)
|
|
24
|
+
return true;
|
|
25
|
+
if (!lower.startsWith(token))
|
|
26
|
+
return false;
|
|
27
|
+
return /[A-Z0-9]/.test(segment.charAt(token.length));
|
|
28
|
+
});
|
|
29
|
+
}
|
|
10
30
|
/** Pure selection: not_exercised findings at blocking severities, capped. */
|
|
11
|
-
export function selectProbeTargets(findings, severityGates, maxProbes) {
|
|
12
|
-
|
|
13
|
-
.filter((f) => f.evidence.status === 'not_exercised' &&
|
|
31
|
+
export function selectProbeTargets(findings, severityGates, maxProbes, triageArea) {
|
|
32
|
+
const eligible = findings.filter((f) => f.evidence.status === 'not_exercised' &&
|
|
14
33
|
f.file !== undefined &&
|
|
15
34
|
isSafeRepoPath(f.file) &&
|
|
16
|
-
severityGates.includes(f.severity ?? ''))
|
|
17
|
-
|
|
35
|
+
severityGates.includes(f.severity ?? ''));
|
|
36
|
+
// U9 — a confident triage top_risk_area reorders candidates so probes
|
|
37
|
+
// prefer the flagged subsystem. Stable sort keeps the original order
|
|
38
|
+
// within each group; probe count/gates/verdict are unchanged.
|
|
39
|
+
if (triageArea !== undefined && triageArea.confidence >= MIN_AREA_CONFIDENCE) {
|
|
40
|
+
// Hit flags precomputed once — the comparator would re-derive them
|
|
41
|
+
// O(n log n) times otherwise.
|
|
42
|
+
const hit = new Map(eligible.map((f) => [f, fileHitsArea(f.file ?? '', triageArea.area)]));
|
|
43
|
+
eligible.sort((a, b) => Number(hit.get(b)) - Number(hit.get(a)));
|
|
44
|
+
}
|
|
45
|
+
return eligible.slice(0, Math.max(0, maxProbes));
|
|
18
46
|
}
|
|
19
47
|
/**
|
|
20
48
|
* Repo-relative path gate for anything model- or index-derived that is read
|
|
@@ -145,14 +173,24 @@ async function authorProbe(o, target, harness, exemplarPath, indexPaths) {
|
|
|
145
173
|
});
|
|
146
174
|
}
|
|
147
175
|
catch (e) {
|
|
148
|
-
return {
|
|
176
|
+
return {
|
|
177
|
+
probe: undefined,
|
|
178
|
+
reason: `authoring call failed: ${e.message}`,
|
|
179
|
+
costUsd: 0,
|
|
180
|
+
tokens: 0,
|
|
181
|
+
};
|
|
149
182
|
}
|
|
150
183
|
o.ledger.recordCall(response.cost);
|
|
151
184
|
o.calls?.push(response.cost);
|
|
152
185
|
const parsed = parseProbe(response.content);
|
|
153
186
|
if (!parsed.ok) {
|
|
154
187
|
// Failed parses still cost the call — carry the spend on the record.
|
|
155
|
-
return {
|
|
188
|
+
return {
|
|
189
|
+
probe: undefined,
|
|
190
|
+
reason: parsed.reason,
|
|
191
|
+
costUsd: response.cost.costUsd,
|
|
192
|
+
tokens: response.cost.tokens,
|
|
193
|
+
};
|
|
156
194
|
}
|
|
157
195
|
return { probe: parsed.probe, costUsd: response.cost.costUsd, tokens: response.cost.tokens };
|
|
158
196
|
}
|
|
@@ -203,7 +241,7 @@ export async function runProbeLane(findings, o) {
|
|
|
203
241
|
// image/limits/allowForks would self-approve the gate. Forks always run
|
|
204
242
|
// on DEFAULT_SANDBOX and approve only via trusted author or the label.
|
|
205
243
|
const sandbox = o.meta?.isFork ? { ...DEFAULT_SANDBOX, enabled: true } : o.sandbox;
|
|
206
|
-
const targets = selectProbeTargets(findings, o.severityGates, sandbox.maxProbes);
|
|
244
|
+
const targets = selectProbeTargets(findings, o.severityGates, sandbox.maxProbes, o.triageArea);
|
|
207
245
|
if (targets.length === 0)
|
|
208
246
|
return { records: [] };
|
|
209
247
|
if (!mayProbePr(o.meta, sandbox)) {
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { type ExecFn } from '../detect.js';
|
|
2
|
+
import type { CallCost } from '../vision/cost.js';
|
|
3
|
+
/** Whether a report's source can be tied to the intended PR head. */
|
|
4
|
+
export type HeadBindingStatus = 'match' | 'mismatch' | 'unknown' | 'not_applicable';
|
|
5
|
+
export type HeadSource = 'github' | 'fixture' | 'local';
|
|
6
|
+
export declare const LANE_IDS: readonly ["review", "flow", "app", "a0"];
|
|
7
|
+
export type LaneId = (typeof LANE_IDS)[number];
|
|
8
|
+
export declare const LANE_STATUSES: readonly ["passed", "failed", "skipped", "blocked", "unavailable", "inconclusive"];
|
|
9
|
+
export type LaneStatus = (typeof LANE_STATUSES)[number];
|
|
10
|
+
export declare const MANIFEST_SCHEMA_VERSION: 1;
|
|
11
|
+
export interface HeadBinding {
|
|
12
|
+
intendedSha: string | undefined;
|
|
13
|
+
checkoutSha: string | undefined;
|
|
14
|
+
status: HeadBindingStatus;
|
|
15
|
+
source: HeadSource;
|
|
16
|
+
detail: string;
|
|
17
|
+
}
|
|
18
|
+
export interface UsageSummary {
|
|
19
|
+
provider: 'openrouter' | 'a0' | 'unknown';
|
|
20
|
+
model: string | undefined;
|
|
21
|
+
calls: number;
|
|
22
|
+
tokens: number;
|
|
23
|
+
costUsd: number;
|
|
24
|
+
metered: boolean;
|
|
25
|
+
}
|
|
26
|
+
export interface BudgetSummary {
|
|
27
|
+
limitUsd: number | undefined;
|
|
28
|
+
spentUsd: number;
|
|
29
|
+
exceeded: boolean;
|
|
30
|
+
maxDurationMs: number | undefined;
|
|
31
|
+
elapsedMs: number;
|
|
32
|
+
maxTasks: number | undefined;
|
|
33
|
+
tasks: number;
|
|
34
|
+
}
|
|
35
|
+
export interface LaneManifest {
|
|
36
|
+
lane: LaneId;
|
|
37
|
+
selected: boolean;
|
|
38
|
+
status: LaneStatus;
|
|
39
|
+
startedAt: string | undefined;
|
|
40
|
+
finishedAt: string | undefined;
|
|
41
|
+
reportPath: string | undefined;
|
|
42
|
+
model: string | undefined;
|
|
43
|
+
summary: string | undefined;
|
|
44
|
+
reason: string | undefined;
|
|
45
|
+
usage: UsageSummary;
|
|
46
|
+
budget: BudgetSummary;
|
|
47
|
+
headBinding: HeadBinding | undefined;
|
|
48
|
+
}
|
|
49
|
+
export interface RunIdentity {
|
|
50
|
+
repo: string | undefined;
|
|
51
|
+
pr: string | undefined;
|
|
52
|
+
intendedHeadSha: string | undefined;
|
|
53
|
+
checkoutSha: string | undefined;
|
|
54
|
+
baseSha: string | undefined;
|
|
55
|
+
}
|
|
56
|
+
export interface RunManifest {
|
|
57
|
+
schemaVersion: typeof MANIFEST_SCHEMA_VERSION;
|
|
58
|
+
runId: string;
|
|
59
|
+
startedAt: string;
|
|
60
|
+
finishedAt: string;
|
|
61
|
+
identity: RunIdentity;
|
|
62
|
+
lanes: Record<LaneId, LaneManifest>;
|
|
63
|
+
aggregate: {
|
|
64
|
+
status: LaneStatus;
|
|
65
|
+
ok: boolean;
|
|
66
|
+
costUsd: number;
|
|
67
|
+
calls: number;
|
|
68
|
+
tokens: number;
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/** Read the checked-out commit without making git identity a hard dependency. */
|
|
72
|
+
export declare function readCheckoutSha(cwd: string, exec?: ExecFn): Promise<string | undefined>;
|
|
73
|
+
/** Classify the relationship between API/fixture head identity and the checkout. */
|
|
74
|
+
export declare function classifyHeadBinding(intendedSha: string | undefined, checkoutSha: string | undefined, source: HeadSource): HeadBinding;
|
|
75
|
+
/** True when runtime evidence is bound to the intended head (or a fixture). */
|
|
76
|
+
export declare function isHeadBindingConclusive(binding: HeadBinding | undefined): boolean;
|
|
77
|
+
export declare function emptyUsage(provider?: UsageSummary['provider']): UsageSummary;
|
|
78
|
+
export declare function emptyBudget(): BudgetSummary;
|
|
79
|
+
export declare function emptyLane(lane: LaneId, selected: boolean): LaneManifest;
|
|
80
|
+
export declare function aggregateLanes(lanes: Record<LaneId, LaneManifest>): {
|
|
81
|
+
status: LaneStatus;
|
|
82
|
+
ok: boolean;
|
|
83
|
+
costUsd: number;
|
|
84
|
+
calls: number;
|
|
85
|
+
tokens: number;
|
|
86
|
+
};
|
|
87
|
+
export declare function addProviderUsage(usage: UsageSummary, calls: CallCost[] | undefined): UsageSummary;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { defaultExec } from '../detect.js';
|
|
2
|
+
export const LANE_IDS = ['review', 'flow', 'app', 'a0'];
|
|
3
|
+
export const LANE_STATUSES = [
|
|
4
|
+
'passed',
|
|
5
|
+
'failed',
|
|
6
|
+
'skipped',
|
|
7
|
+
'blocked',
|
|
8
|
+
'unavailable',
|
|
9
|
+
'inconclusive',
|
|
10
|
+
];
|
|
11
|
+
export const MANIFEST_SCHEMA_VERSION = 1;
|
|
12
|
+
/** Read the checked-out commit without making git identity a hard dependency. */
|
|
13
|
+
export async function readCheckoutSha(cwd, exec = defaultExec) {
|
|
14
|
+
const result = await exec('git', ['-C', cwd, 'rev-parse', 'HEAD'], 10_000);
|
|
15
|
+
if (result.code !== 0)
|
|
16
|
+
return undefined;
|
|
17
|
+
const sha = result.stdout.trim();
|
|
18
|
+
return sha === '' ? undefined : sha;
|
|
19
|
+
}
|
|
20
|
+
/** Classify the relationship between API/fixture head identity and the checkout. */
|
|
21
|
+
export function classifyHeadBinding(intendedSha, checkoutSha, source) {
|
|
22
|
+
if (source === 'fixture') {
|
|
23
|
+
return {
|
|
24
|
+
intendedSha,
|
|
25
|
+
checkoutSha,
|
|
26
|
+
status: 'not_applicable',
|
|
27
|
+
source,
|
|
28
|
+
detail: 'fixture diff is bound to its local fixture head',
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (intendedSha === undefined || intendedSha === '') {
|
|
32
|
+
return {
|
|
33
|
+
intendedSha,
|
|
34
|
+
checkoutSha,
|
|
35
|
+
status: 'unknown',
|
|
36
|
+
source,
|
|
37
|
+
detail: 'PR head identity was unavailable',
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (checkoutSha === undefined || checkoutSha === '') {
|
|
41
|
+
return {
|
|
42
|
+
intendedSha,
|
|
43
|
+
checkoutSha,
|
|
44
|
+
status: 'unknown',
|
|
45
|
+
source,
|
|
46
|
+
detail: 'checkout identity was unavailable',
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
if (intendedSha === checkoutSha) {
|
|
50
|
+
return {
|
|
51
|
+
intendedSha,
|
|
52
|
+
checkoutSha,
|
|
53
|
+
status: 'match',
|
|
54
|
+
source,
|
|
55
|
+
detail: 'checkout matches the intended PR head',
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
intendedSha,
|
|
60
|
+
checkoutSha,
|
|
61
|
+
status: 'mismatch',
|
|
62
|
+
source,
|
|
63
|
+
detail: `checkout ${checkoutSha} does not match intended PR head ${intendedSha}`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** True when runtime evidence is bound to the intended head (or a fixture). */
|
|
67
|
+
export function isHeadBindingConclusive(binding) {
|
|
68
|
+
return binding?.status === 'match' || binding?.status === 'not_applicable';
|
|
69
|
+
}
|
|
70
|
+
export function emptyUsage(provider = 'unknown') {
|
|
71
|
+
return {
|
|
72
|
+
provider,
|
|
73
|
+
model: undefined,
|
|
74
|
+
calls: 0,
|
|
75
|
+
tokens: 0,
|
|
76
|
+
costUsd: 0,
|
|
77
|
+
metered: provider !== 'a0',
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
export function emptyBudget() {
|
|
81
|
+
return {
|
|
82
|
+
limitUsd: undefined,
|
|
83
|
+
spentUsd: 0,
|
|
84
|
+
exceeded: false,
|
|
85
|
+
maxDurationMs: undefined,
|
|
86
|
+
elapsedMs: 0,
|
|
87
|
+
maxTasks: undefined,
|
|
88
|
+
tasks: 0,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
export function emptyLane(lane, selected) {
|
|
92
|
+
return {
|
|
93
|
+
lane,
|
|
94
|
+
selected,
|
|
95
|
+
status: selected ? 'blocked' : 'skipped',
|
|
96
|
+
startedAt: undefined,
|
|
97
|
+
finishedAt: undefined,
|
|
98
|
+
reportPath: undefined,
|
|
99
|
+
model: undefined,
|
|
100
|
+
summary: undefined,
|
|
101
|
+
reason: undefined,
|
|
102
|
+
usage: emptyUsage(),
|
|
103
|
+
budget: emptyBudget(),
|
|
104
|
+
headBinding: undefined,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
export function aggregateLanes(lanes) {
|
|
108
|
+
const selected = LANE_IDS.map((lane) => lanes[lane]).filter((lane) => lane.selected);
|
|
109
|
+
const hasFailure = selected.some((lane) => ['failed', 'blocked'].includes(lane.status));
|
|
110
|
+
const hasInconclusive = selected.some((lane) => lane.status === 'inconclusive');
|
|
111
|
+
const hasUnavailable = selected.some((lane) => lane.status === 'unavailable');
|
|
112
|
+
const status = hasFailure
|
|
113
|
+
? 'failed'
|
|
114
|
+
: hasInconclusive || hasUnavailable
|
|
115
|
+
? 'inconclusive'
|
|
116
|
+
: selected.length === 0 || selected.every((lane) => lane.status === 'skipped')
|
|
117
|
+
? 'skipped'
|
|
118
|
+
: selected.every((lane) => lane.status === 'passed' || lane.status === 'skipped')
|
|
119
|
+
? 'passed'
|
|
120
|
+
: 'inconclusive';
|
|
121
|
+
return {
|
|
122
|
+
status,
|
|
123
|
+
ok: status === 'passed',
|
|
124
|
+
costUsd: selected.reduce((sum, lane) => sum + lane.usage.costUsd, 0),
|
|
125
|
+
calls: selected.reduce((sum, lane) => sum + lane.usage.calls, 0),
|
|
126
|
+
tokens: selected.reduce((sum, lane) => sum + lane.usage.tokens, 0),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
export function addProviderUsage(usage, calls) {
|
|
130
|
+
if (calls === undefined || calls.length === 0)
|
|
131
|
+
return usage;
|
|
132
|
+
const costUsd = calls.reduce((sum, call) => sum + call.costUsd, 0);
|
|
133
|
+
const tokens = calls.reduce((sum, call) => sum + call.tokens, 0);
|
|
134
|
+
const model = calls.at(-1)?.model;
|
|
135
|
+
return {
|
|
136
|
+
provider: 'openrouter',
|
|
137
|
+
model: usage.model ?? model,
|
|
138
|
+
calls: usage.calls + calls.length,
|
|
139
|
+
tokens: usage.tokens + tokens,
|
|
140
|
+
costUsd: usage.costUsd + costUsd,
|
|
141
|
+
metered: true,
|
|
142
|
+
};
|
|
143
|
+
}
|
package/dist/report/run.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { HealEvent, TdAssertRecord, TdStepRecord } from '../api.js';
|
|
2
|
+
import type { CacheStats } from '../engine/loop.js';
|
|
2
3
|
import type { CallCost } from '../vision/cost.js';
|
|
3
4
|
/**
|
|
4
5
|
* JSON run report consumed by the GitHub Action (R10): per-test verdicts,
|
|
@@ -20,6 +21,8 @@ export interface TestReport {
|
|
|
20
21
|
/** Per-call cost rows for model-level attribution. */
|
|
21
22
|
calls: CallCost[];
|
|
22
23
|
videoPath: string | undefined;
|
|
24
|
+
/** Fingerprint replay/grounding counters for this test. */
|
|
25
|
+
cache?: CacheStats;
|
|
23
26
|
/** Agent Zero's autonomous second opinion on a failure (heal: 'a0'). */
|
|
24
27
|
a0Diagnosis?: string;
|
|
25
28
|
}
|
|
@@ -35,6 +38,12 @@ export interface RunTotals {
|
|
|
35
38
|
costByModel: Record<string, number>;
|
|
36
39
|
/** OpenRouter call count grouped by model id. */
|
|
37
40
|
callsByModel: Record<string, number>;
|
|
41
|
+
cacheHits: number;
|
|
42
|
+
cacheMisses: number;
|
|
43
|
+
cacheHeals: number;
|
|
44
|
+
staleEntries: number;
|
|
45
|
+
assertionHits: number;
|
|
46
|
+
assertionMisses: number;
|
|
38
47
|
}
|
|
39
48
|
export interface RunReport {
|
|
40
49
|
tool: 'argus-reviewer';
|
package/dist/report/run.js
CHANGED
|
@@ -24,6 +24,12 @@ export function buildRunReport(tests, startedAt, durationMs) {
|
|
|
24
24
|
budgetExceeded: tests.some((t) => t.budgetExceeded),
|
|
25
25
|
callsByModel,
|
|
26
26
|
costByModel,
|
|
27
|
+
cacheHits: tests.reduce((sum, t) => sum + (t.cache?.hits ?? 0), 0),
|
|
28
|
+
cacheMisses: tests.reduce((sum, t) => sum + (t.cache?.misses ?? 0), 0),
|
|
29
|
+
cacheHeals: tests.reduce((sum, t) => sum + (t.cache?.heals ?? 0), 0),
|
|
30
|
+
staleEntries: tests.reduce((sum, t) => sum + (t.cache?.staleEntries ?? 0), 0),
|
|
31
|
+
assertionHits: tests.reduce((sum, t) => sum + (t.cache?.assertionHits ?? 0), 0),
|
|
32
|
+
assertionMisses: tests.reduce((sum, t) => sum + (t.cache?.assertionMisses ?? 0), 0),
|
|
27
33
|
},
|
|
28
34
|
tests,
|
|
29
35
|
artifacts: {
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { DecisionClient } from '../vision/decisions.js';
|
|
2
|
+
/**
|
|
3
|
+
* U8 finding adjudication — one batched Jev decide() scores each
|
|
4
|
+
* synthesized finding's true-positive probability before posting. Same
|
|
5
|
+
* posture as the secrets lane: Jev annotates/routes, never gates.
|
|
6
|
+
* `p` lands on the finding record and the sticky confidence column;
|
|
7
|
+
* suppression is scoped to `nit`/`q` severities whose false-positive
|
|
8
|
+
* confidence exceeds `findingThreshold` — `bug`/`risk` are never
|
|
9
|
+
* suppressed, and a Jev outage leaves every finding unadjudicated and
|
|
10
|
+
* unsuppressed (degrade open).
|
|
11
|
+
*
|
|
12
|
+
* `findingThreshold` is the required P(false positive): a nit/q is
|
|
13
|
+
* suppressed when `1 - p > threshold`, i.e. `p < 1 - threshold`.
|
|
14
|
+
* Default 1.0 → nothing can exceed 100% FP → annotate-only.
|
|
15
|
+
*/
|
|
16
|
+
export interface AdjudicableFinding {
|
|
17
|
+
file: string;
|
|
18
|
+
line?: number;
|
|
19
|
+
severity: string;
|
|
20
|
+
category?: string;
|
|
21
|
+
message: string;
|
|
22
|
+
}
|
|
23
|
+
export interface FindingAdjudicationRecord {
|
|
24
|
+
file: string;
|
|
25
|
+
line?: number;
|
|
26
|
+
severity: string;
|
|
27
|
+
category?: string;
|
|
28
|
+
/** Finding text — capped; suppressed findings keep their message in audit. */
|
|
29
|
+
message: string;
|
|
30
|
+
adjudicated: boolean;
|
|
31
|
+
/** Jev P(true positive) for this finding. */
|
|
32
|
+
p?: number;
|
|
33
|
+
/** nit/q below the FP bar — kept for audit, removed from findings. */
|
|
34
|
+
suppressed?: boolean;
|
|
35
|
+
}
|
|
36
|
+
/** The audit half of the result — what report.findingAdjudication stores. */
|
|
37
|
+
export interface FindingAdjudicationAudit {
|
|
38
|
+
records: FindingAdjudicationRecord[];
|
|
39
|
+
/** Findings past MAX_CANDIDATES — unadjudicated, never suppressed. */
|
|
40
|
+
overflow: number;
|
|
41
|
+
/** Whole-call failure — nothing adjudicated, nothing suppressed. */
|
|
42
|
+
unadjudicated?: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface FindingAdjudicationResult<T extends AdjudicableFinding> extends FindingAdjudicationAudit {
|
|
45
|
+
/** Surviving findings — adjudicated ones carry `p`. */
|
|
46
|
+
findings: (T & {
|
|
47
|
+
p?: number;
|
|
48
|
+
})[];
|
|
49
|
+
}
|
|
50
|
+
export declare function adjudicateFindings<T extends AdjudicableFinding>(opts: {
|
|
51
|
+
findings: T[];
|
|
52
|
+
/** PR file patches keyed by filename — Jev state context. */
|
|
53
|
+
patchByFile?: Map<string, string>;
|
|
54
|
+
threshold: number;
|
|
55
|
+
/**
|
|
56
|
+
* User-configured blocking severities — a severity listed here drives
|
|
57
|
+
* the verdict, so it must never be suppressed even if it is nit/q
|
|
58
|
+
* (otherwise Jev suppression could flip the commit-status gate).
|
|
59
|
+
*/
|
|
60
|
+
blockSeverities?: string[];
|
|
61
|
+
client: DecisionClient;
|
|
62
|
+
model?: string;
|
|
63
|
+
}): Promise<FindingAdjudicationResult<T>>;
|