argus-reviewer-e2e 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -71
- package/action/action.yml +129 -11
- package/action/approval-review.mjs +13 -3
- package/action/bootstrap.mjs +2 -0
- package/action/emit-review.mjs +16 -0
- package/action/runtime.mjs +20 -0
- package/action/sticky-comment.cjs +1260 -479
- package/dist/cli.d.ts +103 -7
- package/dist/cli.js +1202 -186
- package/dist/config.d.ts +96 -11
- package/dist/config.js +102 -4
- package/dist/detect.d.ts +29 -2
- package/dist/detect.js +98 -7
- package/dist/driver/browser.d.ts +32 -0
- package/dist/driver/browser.js +56 -1
- package/dist/driver/target.d.ts +4 -1
- package/dist/driver/target.js +27 -6
- package/dist/engine/actions.d.ts +5 -0
- package/dist/engine/actions.js +8 -0
- package/dist/engine/explore.d.ts +78 -0
- package/dist/engine/explore.js +373 -0
- package/dist/engine/loop.d.ts +2 -2
- package/dist/engine/loop.js +8 -8
- package/dist/engine/prompts.d.ts +28 -1
- package/dist/engine/prompts.js +88 -0
- package/dist/evidence/ci.d.ts +13 -1
- package/dist/evidence/ci.js +38 -3
- package/dist/evidence/gate.d.ts +8 -0
- package/dist/evidence/gate.js +1 -1
- package/dist/evidence/link.js +1 -1
- package/dist/executor/a0.d.ts +114 -1
- package/dist/executor/a0.js +216 -4
- package/dist/fsutil.d.ts +3 -2
- package/dist/fsutil.js +7 -4
- package/dist/journal/schema.d.ts +1 -1
- package/dist/log.d.ts +2 -1
- package/dist/log.js +10 -2
- package/dist/mention.d.ts +45 -0
- package/dist/mention.js +107 -0
- package/dist/pipeline/app.d.ts +126 -0
- package/dist/pipeline/app.js +250 -0
- package/dist/pipeline/budget.d.ts +1 -0
- package/dist/pipeline/budget.js +1 -1
- package/dist/pipeline/verify.d.ts +20 -3
- package/dist/pipeline/verify.js +189 -35
- package/dist/probe/persist.d.ts +68 -0
- package/dist/probe/persist.js +184 -0
- package/dist/probe/queue.d.ts +12 -0
- package/dist/probe/queue.js +10 -2
- package/dist/report/brand-assets.generated.d.ts +9 -0
- package/dist/report/brand-assets.generated.js +8 -0
- package/dist/report/comment.d.ts +99 -6
- package/dist/report/comment.js +292 -103
- package/dist/report/html.d.ts +50 -0
- package/dist/report/html.js +879 -0
- package/dist/report/manifest.d.ts +29 -0
- package/dist/report/manifest.js +37 -0
- package/dist/report/run.d.ts +54 -1
- package/dist/report/run.js +34 -9
- package/dist/report/viewmodel.d.ts +91 -0
- package/dist/report/viewmodel.js +241 -0
- package/dist/review/adjudicate.d.ts +6 -6
- package/dist/review/adjudicate.js +2 -2
- package/dist/review/inline.d.ts +44 -0
- package/dist/review/inline.js +95 -0
- package/dist/review/packs.d.ts +21 -0
- package/dist/review/packs.js +47 -0
- package/dist/review/scope.d.ts +16 -0
- package/dist/review/scope.js +74 -0
- package/dist/review/secrets.d.ts +10 -10
- package/dist/review/secrets.js +7 -7
- package/dist/review/testfiles.d.ts +18 -0
- package/dist/review/testfiles.js +26 -0
- package/dist/review/triage.d.ts +1 -1
- package/dist/review/triage.js +10 -10
- package/dist/review/validate.d.ts +41 -0
- package/dist/review/validate.js +76 -0
- package/dist/ui/errors.d.ts +54 -0
- package/dist/ui/errors.js +236 -0
- package/dist/ui/style.d.ts +34 -0
- package/dist/ui/style.js +48 -0
- package/dist/ui/summary.d.ts +38 -0
- package/dist/ui/summary.js +101 -0
- package/dist/vision/cost.d.ts +1 -1
- package/dist/vision/decisions.d.ts +9 -3
- package/dist/vision/decisions.js +31 -21
- package/dist/vision/openrouter.d.ts +4 -0
- package/dist/vision/openrouter.js +30 -4
- package/package.json +11 -2
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import { BrowserDriver as launchBrowserDriver } from '../driver/browser.js';
|
|
2
|
+
import { TargetProcess, waitForReady } from '../driver/target.js';
|
|
3
|
+
import { Actions } from '../engine/actions.js';
|
|
4
|
+
import { runExplore, } from '../engine/explore.js';
|
|
5
|
+
import { Ledger } from '../vision/ledger.js';
|
|
6
|
+
/** Lane detail file the app runner writes and `runVerify` reads back. */
|
|
7
|
+
export const APP_LANE_REPORT = 'app-lane.json';
|
|
8
|
+
/** Wall-clock bound when config.app.timeoutMs is unset. */
|
|
9
|
+
export const APP_LANE_DEFAULT_TIMEOUT_MS = 120_000;
|
|
10
|
+
/** Reachability preflight bound — short enough to fail fast on dead targets. */
|
|
11
|
+
const APP_PREFLIGHT_TIMEOUT_MS = 10_000;
|
|
12
|
+
/**
|
|
13
|
+
* Compile the configured expected state into one AND-ed predicate. An
|
|
14
|
+
* invalid url regex throws here — caught at preflight so a broken marker
|
|
15
|
+
* blocks the lane instead of failing mid-run.
|
|
16
|
+
*/
|
|
17
|
+
export function buildExpectationCheck(expected) {
|
|
18
|
+
const parts = [];
|
|
19
|
+
if (expected.text !== undefined) {
|
|
20
|
+
const needle = expected.text.toLowerCase();
|
|
21
|
+
parts.push({
|
|
22
|
+
describe: `text "${expected.text}"`,
|
|
23
|
+
test: async (ctx) => ctx.observation.a11yYaml.toLowerCase().includes(needle),
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
if (expected.url !== undefined) {
|
|
27
|
+
const re = new RegExp(expected.url);
|
|
28
|
+
parts.push({ describe: `url /${expected.url}/`, test: async (ctx) => re.test(ctx.url) });
|
|
29
|
+
}
|
|
30
|
+
if (expected.selector !== undefined) {
|
|
31
|
+
const selector = expected.selector;
|
|
32
|
+
parts.push({
|
|
33
|
+
describe: `selector "${selector}"`,
|
|
34
|
+
test: async (ctx) => (await ctx.page.locator(selector).count()) > 0,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
describe: parts.map((p) => p.describe).join(' AND '),
|
|
39
|
+
check: async (ctx) => {
|
|
40
|
+
for (const part of parts) {
|
|
41
|
+
if (!(await part.test(ctx)))
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
return parts.length > 0;
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The bounded task loop: ExploreLoop substrate + task + expectation +
|
|
50
|
+
* wall-clock deadline. The lane — not the model's `done` — decides pass:
|
|
51
|
+
* the expected-state predicate is re-verified on a fresh observation after
|
|
52
|
+
* the loop stops, so a page that merely loads can never pass.
|
|
53
|
+
*/
|
|
54
|
+
export async function runAppTask(input) {
|
|
55
|
+
const expectation = buildExpectationCheck(input.expected);
|
|
56
|
+
const result = await runExplore({
|
|
57
|
+
driver: input.driver,
|
|
58
|
+
actions: input.actions,
|
|
59
|
+
client: input.client,
|
|
60
|
+
ledger: input.ledger,
|
|
61
|
+
config: input.config,
|
|
62
|
+
targetUrl: input.targetUrl,
|
|
63
|
+
task: input.task,
|
|
64
|
+
deadlineAt: Date.now() + input.timeoutMs,
|
|
65
|
+
expectation: (ctx) => expectation.check(ctx),
|
|
66
|
+
...(input.maxSteps !== undefined ? { maxSteps: input.maxSteps } : {}),
|
|
67
|
+
...(input.budgetLimitUsd !== undefined ? { budgetUsd: input.budgetLimitUsd } : {}),
|
|
68
|
+
...(input.logger !== undefined ? { logger: input.logger } : {}),
|
|
69
|
+
});
|
|
70
|
+
// Mid-loop 'expectation' already verified on a real observation — no
|
|
71
|
+
// second screenshot. Every other stop re-verifies against fresh state.
|
|
72
|
+
let expectedMet = result.stopReason === 'expectation';
|
|
73
|
+
let verifyError;
|
|
74
|
+
if (!expectedMet) {
|
|
75
|
+
try {
|
|
76
|
+
const observation = await input.driver.observe({ grid: true });
|
|
77
|
+
expectedMet = await expectation.check({
|
|
78
|
+
observation,
|
|
79
|
+
url: input.driver.rawPage.url(),
|
|
80
|
+
page: input.driver.rawPage,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
catch (e) {
|
|
84
|
+
verifyError = e.message;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return { result, expectedMet, verifyError };
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Full `verify --app` lifecycle: contract preflight → trust/reachability →
|
|
91
|
+
* bounded task loop → lane record. Every status path produces a report —
|
|
92
|
+
* the lane can fail loudly, never silently.
|
|
93
|
+
*/
|
|
94
|
+
export async function runAppLane(input) {
|
|
95
|
+
const started = Date.now();
|
|
96
|
+
const config = input.config;
|
|
97
|
+
const deps = input.deps ?? {};
|
|
98
|
+
const task = input.task ?? config.app.task;
|
|
99
|
+
const expected = input.expected ?? config.app.expected;
|
|
100
|
+
const targetUrl = input.url ?? config.target?.url;
|
|
101
|
+
const base = {
|
|
102
|
+
lane: 'app',
|
|
103
|
+
task,
|
|
104
|
+
expected,
|
|
105
|
+
targetUrl,
|
|
106
|
+
finalUrl: undefined,
|
|
107
|
+
expectedMet: false,
|
|
108
|
+
stopReason: undefined,
|
|
109
|
+
steps: [],
|
|
110
|
+
visited: 0,
|
|
111
|
+
anomalies: [],
|
|
112
|
+
notes: [],
|
|
113
|
+
artifacts: { video: undefined },
|
|
114
|
+
calls: [],
|
|
115
|
+
visionCalls: 0,
|
|
116
|
+
visionCostUsd: 0,
|
|
117
|
+
durationMs: 0,
|
|
118
|
+
model: config.model,
|
|
119
|
+
};
|
|
120
|
+
const done = (status, reason, summary, extra) => ({
|
|
121
|
+
...base,
|
|
122
|
+
...extra,
|
|
123
|
+
status,
|
|
124
|
+
reason,
|
|
125
|
+
summary: summary ?? (status === 'passed' ? 'task verified' : reason),
|
|
126
|
+
durationMs: Date.now() - started,
|
|
127
|
+
});
|
|
128
|
+
// Executable lanes are gated on the same trust resolution as config
|
|
129
|
+
// loading (SECURITY.md): an untrusted checkout never reaches the task,
|
|
130
|
+
// the browser, or a PR-controlled expectation — blocked before any
|
|
131
|
+
// preflight, not just before `target.command`.
|
|
132
|
+
if (!input.trusted) {
|
|
133
|
+
return done('blocked', 'verify --app is an executable lane — requires a trusted checkout');
|
|
134
|
+
}
|
|
135
|
+
// Contract preflight — a lane without a task or a verifiable marker
|
|
136
|
+
// blocks before touching the network or the provider (R8).
|
|
137
|
+
if (task === undefined || task.trim() === '') {
|
|
138
|
+
return done('blocked', 'no task configured — set app.task or pass --task');
|
|
139
|
+
}
|
|
140
|
+
if (expected === undefined) {
|
|
141
|
+
return done('blocked', 'app lane needs an expected-state marker (app.expected.text/url/selector or --expect-*) — page load alone cannot pass a task');
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
buildExpectationCheck(expected);
|
|
145
|
+
}
|
|
146
|
+
catch (e) {
|
|
147
|
+
return done('blocked', `invalid expected-state marker: ${e.message}`);
|
|
148
|
+
}
|
|
149
|
+
if (targetUrl === undefined || targetUrl === '') {
|
|
150
|
+
return done('blocked', 'no application target configured; set target.url or pass --url');
|
|
151
|
+
}
|
|
152
|
+
let target;
|
|
153
|
+
let driver;
|
|
154
|
+
try {
|
|
155
|
+
if (config.target?.command !== undefined && config.target.command !== '') {
|
|
156
|
+
const start = deps.startTarget ?? ((spec) => TargetProcess.start(spec));
|
|
157
|
+
try {
|
|
158
|
+
target = await start(config.target);
|
|
159
|
+
}
|
|
160
|
+
catch (e) {
|
|
161
|
+
return done('unavailable', `target command failed to boot: ${e.message}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
// No boot command — the target must already serve. Probe before any
|
|
166
|
+
// model call so an unreachable app costs zero provider spend.
|
|
167
|
+
const ready = deps.waitForReady ?? waitForReady;
|
|
168
|
+
try {
|
|
169
|
+
await ready(targetUrl, config.target?.readyTimeoutMs ?? APP_PREFLIGHT_TIMEOUT_MS);
|
|
170
|
+
}
|
|
171
|
+
catch (e) {
|
|
172
|
+
return done('unavailable', `target unreachable before any model call: ${e.message}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const launch = deps.launchDriver ??
|
|
176
|
+
((cfg) =>
|
|
177
|
+
// The lane's evidence contract needs anomaly taps regardless of the
|
|
178
|
+
// observe-only explore toggle.
|
|
179
|
+
launchBrowserDriver.launch({
|
|
180
|
+
browser: cfg.browser,
|
|
181
|
+
browserTimeoutMs: cfg.browserTimeoutMs,
|
|
182
|
+
captureErrors: true,
|
|
183
|
+
}));
|
|
184
|
+
try {
|
|
185
|
+
driver = await launch(config);
|
|
186
|
+
}
|
|
187
|
+
catch (e) {
|
|
188
|
+
return done('unavailable', `browser failed to launch: ${e.message}`);
|
|
189
|
+
}
|
|
190
|
+
if (deps.applyPageSetup !== undefined)
|
|
191
|
+
await deps.applyPageSetup(driver);
|
|
192
|
+
try {
|
|
193
|
+
await driver.goto(targetUrl);
|
|
194
|
+
}
|
|
195
|
+
catch (e) {
|
|
196
|
+
return done('unavailable', `target navigation failed: ${e.message}`);
|
|
197
|
+
}
|
|
198
|
+
const ledger = new Ledger(input.budgetLimitUsd ?? config.app.budgetUsd ?? config.budgetUsd);
|
|
199
|
+
const client = deps.createClient?.(config) ?? missingClient();
|
|
200
|
+
const { result, expectedMet, verifyError } = await runAppTask({
|
|
201
|
+
driver,
|
|
202
|
+
actions: new Actions(driver),
|
|
203
|
+
client,
|
|
204
|
+
ledger,
|
|
205
|
+
config,
|
|
206
|
+
targetUrl,
|
|
207
|
+
task,
|
|
208
|
+
expected,
|
|
209
|
+
timeoutMs: config.app.timeoutMs ?? APP_LANE_DEFAULT_TIMEOUT_MS,
|
|
210
|
+
maxSteps: config.app.maxSteps,
|
|
211
|
+
budgetLimitUsd: input.budgetLimitUsd ?? config.app.budgetUsd ?? config.budgetUsd,
|
|
212
|
+
logger: deps.logger,
|
|
213
|
+
});
|
|
214
|
+
// Captures + video must be lifted before close() — the taps and the
|
|
215
|
+
// webm path die with the context.
|
|
216
|
+
const anomalies = driver.pageCaptures();
|
|
217
|
+
const video = await driver.close();
|
|
218
|
+
driver = undefined;
|
|
219
|
+
const summary = expectedMet
|
|
220
|
+
? `expected state verified (${result.steps.length} steps, ${result.visited} page(s))`
|
|
221
|
+
: `expected state unmet after ${result.stopReason} (${result.steps.length} steps)`;
|
|
222
|
+
return done(expectedMet ? 'passed' : verifyError !== undefined ? 'inconclusive' : 'failed', expectedMet
|
|
223
|
+
? undefined
|
|
224
|
+
: verifyError !== undefined
|
|
225
|
+
? `expected-state verification failed: ${verifyError}`
|
|
226
|
+
: `task did not reach its expected state — stopped on ${result.stopReason}`, summary, {
|
|
227
|
+
finalUrl: result.finalUrl,
|
|
228
|
+
expectedMet,
|
|
229
|
+
stopReason: result.stopReason,
|
|
230
|
+
steps: result.steps,
|
|
231
|
+
visited: result.visited,
|
|
232
|
+
anomalies,
|
|
233
|
+
notes: result.notes,
|
|
234
|
+
artifacts: { video },
|
|
235
|
+
calls: ledger.calls,
|
|
236
|
+
visionCalls: result.visionCalls,
|
|
237
|
+
visionCostUsd: result.visionCostUsd,
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
finally {
|
|
241
|
+
await driver?.close().catch(() => undefined);
|
|
242
|
+
await target?.stop().catch(() => undefined);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/** Default client when the caller supplies none — fails loudly on first call. */
|
|
246
|
+
function missingClient() {
|
|
247
|
+
return {
|
|
248
|
+
complete: () => Promise.reject(new Error('no vision client wired into the app lane')),
|
|
249
|
+
};
|
|
250
|
+
}
|
|
@@ -6,6 +6,7 @@ export interface BudgetOptions {
|
|
|
6
6
|
maxTasks?: number;
|
|
7
7
|
}
|
|
8
8
|
export declare function createBudget(lane: LaneId, options?: BudgetOptions): BudgetSummary;
|
|
9
|
+
export declare function overLimit(spentUsd: number, limitUsd: number): boolean;
|
|
9
10
|
export declare function addProviderCalls(budget: BudgetSummary, calls: CallCost[] | undefined): BudgetSummary;
|
|
10
11
|
export declare function addA0Task(budget: BudgetSummary, elapsedMs: number, metered: boolean): BudgetSummary;
|
|
11
12
|
export declare function budgetCanSpend(budget: BudgetSummary, nextCostUsd: number): boolean;
|
package/dist/pipeline/budget.js
CHANGED
|
@@ -17,7 +17,7 @@ export function createBudget(lane, options = {}) {
|
|
|
17
17
|
* far below the 1e-6 precision reports render, and far above double noise here.
|
|
18
18
|
*/
|
|
19
19
|
const USD_EPSILON = 1e-9;
|
|
20
|
-
function overLimit(spentUsd, limitUsd) {
|
|
20
|
+
export function overLimit(spentUsd, limitUsd) {
|
|
21
21
|
return spentUsd > limitUsd + USD_EPSILON;
|
|
22
22
|
}
|
|
23
23
|
export function addProviderCalls(budget, calls) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type LaneId, type RunIdentity, type RunManifest } from '../report/manifest.js';
|
|
2
2
|
import { type BudgetOptions } from './budget.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { LaneSelection } from './contracts.js';
|
|
4
4
|
export interface VerifyRunners {
|
|
5
5
|
review: () => Promise<number>;
|
|
6
6
|
flow?: (url: string) => Promise<number>;
|
|
@@ -22,6 +22,23 @@ export interface VerifyResult {
|
|
|
22
22
|
manifest: RunManifest;
|
|
23
23
|
exitCode: number;
|
|
24
24
|
}
|
|
25
|
-
/**
|
|
25
|
+
/**
|
|
26
|
+
* Run selected lanes and return one stable manifest without owning
|
|
27
|
+
* subprocesses. Lanes run sequentially on purpose: flow/app can both boot
|
|
28
|
+
* `target.command` (port collision) and serial execution keeps budget
|
|
29
|
+
* accounting and lane timing honest — each lane's startedAt is its own
|
|
30
|
+
* start, not the run's.
|
|
31
|
+
*/
|
|
26
32
|
export declare function runVerify(input: VerifyInput): Promise<VerifyResult>;
|
|
27
|
-
|
|
33
|
+
/** Package version for report footers; dist/ and src/ both sit two levels deep. */
|
|
34
|
+
export declare function argusVersion(): string;
|
|
35
|
+
/**
|
|
36
|
+
* Write the offline HTML evidence report (U14) beside run-manifest.json.
|
|
37
|
+
* Lane reports are read only when the manifest points at them: a runner that
|
|
38
|
+
* threw leaves `reportPath` unset, and its stale file must not reach the
|
|
39
|
+
* report (same contract as the manifest's own reads).
|
|
40
|
+
*/
|
|
41
|
+
export declare function writeEvidenceReport(reportDir: string, manifest: RunManifest, meta?: {
|
|
42
|
+
version?: string;
|
|
43
|
+
runUrl?: string;
|
|
44
|
+
}): Promise<string>;
|
package/dist/pipeline/verify.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { createRequire } from 'node:module';
|
|
2
3
|
import { join, relative } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { writeAtomicText } from '../fsutil.js';
|
|
5
|
+
import { renderReportHtml, REPORT_HTML } from '../report/html.js';
|
|
6
|
+
import { aggregateLanes, addProviderUsage, emptyLane, emptyUsage, isHeadBindingConclusive, LANE_IDS, LANE_STATUSES, MANIFEST_SCHEMA_VERSION, } from '../report/manifest.js';
|
|
7
|
+
import { addProviderCalls, createBudget, overLimit, } from './budget.js';
|
|
6
8
|
async function readJson(path) {
|
|
7
9
|
try {
|
|
8
10
|
return JSON.parse(await readFile(path, 'utf8'));
|
|
@@ -39,15 +41,42 @@ function flowCalls(report) {
|
|
|
39
41
|
function flowUsage(report) {
|
|
40
42
|
const calls = flowCalls(report);
|
|
41
43
|
const usage = providerUsageFromCalls(calls);
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
44
|
+
const totals = report?.totals;
|
|
45
|
+
if (totals === undefined)
|
|
46
|
+
return usage;
|
|
47
|
+
// totals are the authoritative rollup — they fold in lane-level calls the
|
|
48
|
+
// per-test reports never own (e.g. the explore pass). visionTokens is the
|
|
49
|
+
// same rollup for tokens; older run.json files lack it and fall back to
|
|
50
|
+
// the per-test call records.
|
|
51
|
+
return {
|
|
52
|
+
...usage,
|
|
53
|
+
calls: totals.visionCalls ?? usage.calls,
|
|
54
|
+
costUsd: totals.visionCostUsd ?? usage.costUsd,
|
|
55
|
+
tokens: totals.visionTokens ?? usage.tokens,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function flowCache(report) {
|
|
59
|
+
const totals = report?.totals;
|
|
60
|
+
if (totals === undefined)
|
|
61
|
+
return undefined;
|
|
62
|
+
return {
|
|
63
|
+
hits: totals.cacheHits ?? 0,
|
|
64
|
+
misses: totals.cacheMisses ?? 0,
|
|
65
|
+
heals: totals.cacheHeals ?? 0,
|
|
66
|
+
staleEntries: totals.staleEntries ?? 0,
|
|
67
|
+
assertionHits: totals.assertionHits ?? 0,
|
|
68
|
+
assertionMisses: totals.assertionMisses ?? 0,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/** A green `ok` is only evidence when a test executed or the explore lane
|
|
72
|
+
* actually ran as the evidence source — an enabled-but-skipped/errored
|
|
73
|
+
* explore pass proves nothing, so the lane must still fail closed. */
|
|
74
|
+
function flowEvidenceRan(report) {
|
|
75
|
+
if (report === undefined)
|
|
76
|
+
return false;
|
|
77
|
+
if (report.explore?.enabled === true && report.explore.skipped === undefined)
|
|
78
|
+
return true;
|
|
79
|
+
return (report.totals?.tests ?? report.tests?.length ?? 0) > 0;
|
|
51
80
|
}
|
|
52
81
|
function budgetFor(lane, input, report) {
|
|
53
82
|
const options = input.budgets?.[lane] ?? {};
|
|
@@ -56,23 +85,75 @@ function budgetFor(lane, input, report) {
|
|
|
56
85
|
budget = addProviderCalls(budget, report?.calls);
|
|
57
86
|
}
|
|
58
87
|
else if (lane === 'flow') {
|
|
59
|
-
|
|
88
|
+
const flowReport = report;
|
|
89
|
+
budget = addProviderCalls(budget, flowCalls(flowReport));
|
|
90
|
+
// totals fold in lane-level calls the per-test reports never own (the
|
|
91
|
+
// explore pass) — the budget must count the same spend usage reports.
|
|
92
|
+
const totalUsd = flowReport?.totals?.visionCostUsd;
|
|
93
|
+
if (totalUsd !== undefined && totalUsd > budget.spentUsd) {
|
|
94
|
+
budget = {
|
|
95
|
+
...budget,
|
|
96
|
+
spentUsd: totalUsd,
|
|
97
|
+
exceeded: budget.exceeded ||
|
|
98
|
+
(budget.limitUsd !== undefined && overLimit(totalUsd, budget.limitUsd)),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
60
101
|
}
|
|
61
102
|
return budget;
|
|
62
103
|
}
|
|
63
104
|
function baseLane(lane, selected) {
|
|
64
105
|
return emptyLane(lane, selected);
|
|
65
106
|
}
|
|
66
|
-
/**
|
|
107
|
+
/**
|
|
108
|
+
* Usage from a runner's lane-detail file. Per-call records are authoritative
|
|
109
|
+
* for tokens/model; `visionCalls`/`visionCostUsd` scalars cover runners that
|
|
110
|
+
* only know totals. An `a0` host without usage telemetry reports
|
|
111
|
+
* `metered: false` — never a fabricated dollar amount (R13).
|
|
112
|
+
*/
|
|
113
|
+
function laneDetailUsage(lane, detail) {
|
|
114
|
+
const provider = lane === 'a0' ? 'a0' : 'openrouter';
|
|
115
|
+
const base = addProviderUsage(emptyUsage(provider), detail?.calls);
|
|
116
|
+
return {
|
|
117
|
+
...base,
|
|
118
|
+
provider,
|
|
119
|
+
model: detail?.model ?? base.model,
|
|
120
|
+
calls: detail?.calls === undefined ? (detail?.visionCalls ?? base.calls) : base.calls,
|
|
121
|
+
costUsd: detail?.calls === undefined ? (detail?.visionCostUsd ?? base.costUsd) : base.costUsd,
|
|
122
|
+
metered: lane === 'a0' ? detail?.metered === true : base.metered,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Lane budget from options plus what the runner's detail actually consumed —
|
|
127
|
+
* provider calls, delegated tasks, and elapsed wall-clock each count against
|
|
128
|
+
* their configured bound.
|
|
129
|
+
*/
|
|
130
|
+
function laneDetailBudget(lane, input, detail) {
|
|
131
|
+
let budget = addProviderCalls(createBudget(lane, input.budgets?.[lane] ?? {}), detail?.calls);
|
|
132
|
+
const tasks = detail?.tasks ?? 0;
|
|
133
|
+
const elapsedMs = detail?.durationMs ?? 0;
|
|
134
|
+
const exceeded = budget.exceeded ||
|
|
135
|
+
(budget.maxTasks !== undefined && tasks > budget.maxTasks) ||
|
|
136
|
+
(budget.maxDurationMs !== undefined && elapsedMs > budget.maxDurationMs) ||
|
|
137
|
+
(budget.limitUsd !== undefined &&
|
|
138
|
+
detail?.calls === undefined &&
|
|
139
|
+
detail?.visionCostUsd !== undefined &&
|
|
140
|
+
overLimit(detail.visionCostUsd, budget.limitUsd));
|
|
141
|
+
budget = { ...budget, tasks, elapsedMs, exceeded };
|
|
142
|
+
return budget;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Run selected lanes and return one stable manifest without owning
|
|
146
|
+
* subprocesses. Lanes run sequentially on purpose: flow/app can both boot
|
|
147
|
+
* `target.command` (port collision) and serial execution keeps budget
|
|
148
|
+
* accounting and lane timing honest — each lane's startedAt is its own
|
|
149
|
+
* start, not the run's.
|
|
150
|
+
*/
|
|
67
151
|
export async function runVerify(input) {
|
|
68
152
|
const startedAt = new Date();
|
|
69
|
-
const lanes = Object.fromEntries(
|
|
70
|
-
lane,
|
|
71
|
-
baseLane(lane, input.selection[lane]),
|
|
72
|
-
]));
|
|
153
|
+
const lanes = Object.fromEntries(LANE_IDS.map((lane) => [lane, baseLane(lane, input.selection[lane])]));
|
|
73
154
|
const reviewSelected = input.selection.review;
|
|
74
155
|
if (reviewSelected) {
|
|
75
|
-
lanes.review = laneStart(lanes.review
|
|
156
|
+
lanes.review = laneStart(lanes.review);
|
|
76
157
|
let code = 1;
|
|
77
158
|
let runnerError;
|
|
78
159
|
try {
|
|
@@ -82,7 +163,12 @@ export async function runVerify(input) {
|
|
|
82
163
|
runnerError = error instanceof Error ? error.message : String(error);
|
|
83
164
|
ctxError(runnerError);
|
|
84
165
|
}
|
|
85
|
-
|
|
166
|
+
// A runner that threw before writing leaves a stale report behind —
|
|
167
|
+
// ignore it or the manifest inherits another run's usage, head binding,
|
|
168
|
+
// and evidence links (same contract as the <lane>-lane.json guard).
|
|
169
|
+
const report = runnerError === undefined
|
|
170
|
+
? await readJson(join(input.reportDir, 'code-review.json'))
|
|
171
|
+
: undefined;
|
|
86
172
|
const inconclusive = report !== undefined &&
|
|
87
173
|
report.skipped !== true &&
|
|
88
174
|
!isHeadBindingConclusive(report.headBinding);
|
|
@@ -107,7 +193,7 @@ export async function runVerify(input) {
|
|
|
107
193
|
});
|
|
108
194
|
}
|
|
109
195
|
if (input.selection.flow) {
|
|
110
|
-
lanes.flow = laneStart(lanes.flow
|
|
196
|
+
lanes.flow = laneStart(lanes.flow);
|
|
111
197
|
if (input.runners.flow === undefined) {
|
|
112
198
|
lanes.flow = laneEnd({
|
|
113
199
|
...lanes.flow,
|
|
@@ -132,20 +218,36 @@ export async function runVerify(input) {
|
|
|
132
218
|
runnerError = error instanceof Error ? error.message : String(error);
|
|
133
219
|
ctxError(runnerError);
|
|
134
220
|
}
|
|
135
|
-
const report =
|
|
221
|
+
const report = runnerError === undefined
|
|
222
|
+
? await readJson(join(input.reportDir, 'run.json'))
|
|
223
|
+
: undefined;
|
|
224
|
+
// A run.json claiming ok with zero executed tests carries no evidence —
|
|
225
|
+
// fail closed instead of letting it pass the lane.
|
|
226
|
+
const noEvidence = report !== undefined && report.ok === true && code === 0 && !flowEvidenceRan(report);
|
|
136
227
|
lanes.flow = laneEnd({
|
|
137
228
|
...lanes.flow,
|
|
138
|
-
status: runnerError !== undefined
|
|
229
|
+
status: runnerError !== undefined || noEvidence
|
|
230
|
+
? 'failed'
|
|
231
|
+
: reportStatus(code, report, false),
|
|
139
232
|
reportPath: report === undefined ? undefined : relativeReport(input.cwd, input.reportDir, 'run.json'),
|
|
140
|
-
summary: report === undefined
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
? '
|
|
233
|
+
summary: report === undefined
|
|
234
|
+
? undefined
|
|
235
|
+
: noEvidence
|
|
236
|
+
? 'flow ran zero tests'
|
|
144
237
|
: report.ok === true
|
|
145
|
-
?
|
|
146
|
-
: 'flow failed'
|
|
238
|
+
? 'flow passed'
|
|
239
|
+
: 'flow failed',
|
|
240
|
+
reason: runnerError ??
|
|
241
|
+
(noEvidence
|
|
242
|
+
? 'run.json reports ok but executed zero tests'
|
|
243
|
+
: report === undefined
|
|
244
|
+
? 'run.json was not produced'
|
|
245
|
+
: report.ok === true
|
|
246
|
+
? undefined
|
|
247
|
+
: 'flow failed'),
|
|
147
248
|
usage: flowUsage(report),
|
|
148
249
|
budget: budgetFor('flow', input, report),
|
|
250
|
+
cache: flowCache(report),
|
|
149
251
|
});
|
|
150
252
|
}
|
|
151
253
|
}
|
|
@@ -153,7 +255,7 @@ export async function runVerify(input) {
|
|
|
153
255
|
if (!input.selection[lane])
|
|
154
256
|
continue;
|
|
155
257
|
const runner = input.runners[lane];
|
|
156
|
-
lanes[lane] = laneStart(lanes[lane]
|
|
258
|
+
lanes[lane] = laneStart(lanes[lane]);
|
|
157
259
|
if (runner === undefined) {
|
|
158
260
|
lanes[lane] = laneEnd({
|
|
159
261
|
...lanes[lane],
|
|
@@ -171,10 +273,34 @@ export async function runVerify(input) {
|
|
|
171
273
|
runnerError = error instanceof Error ? error.message : String(error);
|
|
172
274
|
ctxError(runnerError);
|
|
173
275
|
}
|
|
276
|
+
// Runners write their own lane detail (`<lane>-lane.json`); a valid
|
|
277
|
+
// status there wins over the exit-code mapping, so preflight outcomes
|
|
278
|
+
// (blocked/unavailable/inconclusive) reach the manifest faithfully.
|
|
279
|
+
// A runner that threw before writing leaves a stale detail file behind
|
|
280
|
+
// — ignore it or the manifest inherits another run's usage and summary.
|
|
281
|
+
const detail = runnerError === undefined
|
|
282
|
+
? await readJson(join(input.reportDir, `${lane}-lane.json`))
|
|
283
|
+
: undefined;
|
|
284
|
+
const detailStatus = detail?.status !== undefined && LANE_STATUSES.includes(detail.status)
|
|
285
|
+
? detail.status
|
|
286
|
+
: undefined;
|
|
287
|
+
const status = runnerError !== undefined
|
|
288
|
+
? 'failed'
|
|
289
|
+
: (detailStatus ?? (code === 0 ? 'passed' : 'failed'));
|
|
290
|
+
const usage = laneDetailUsage(lane, detail);
|
|
174
291
|
lanes[lane] = laneEnd({
|
|
175
292
|
...lanes[lane],
|
|
176
|
-
status
|
|
177
|
-
|
|
293
|
+
status,
|
|
294
|
+
reportPath: detail === undefined
|
|
295
|
+
? undefined
|
|
296
|
+
: relativeReport(input.cwd, input.reportDir, `${lane}-lane.json`),
|
|
297
|
+
model: detail?.model,
|
|
298
|
+
summary: detail?.summary,
|
|
299
|
+
reason: runnerError ??
|
|
300
|
+
detail?.reason ??
|
|
301
|
+
(code === 0 ? undefined : `${lane} runner exited ${code}`),
|
|
302
|
+
usage,
|
|
303
|
+
budget: laneDetailBudget(lane, input, detail),
|
|
178
304
|
});
|
|
179
305
|
}
|
|
180
306
|
const aggregate = aggregateLanes(lanes);
|
|
@@ -189,9 +315,37 @@ export async function runVerify(input) {
|
|
|
189
315
|
};
|
|
190
316
|
return { manifest, exitCode: aggregate.ok ? 0 : 1 };
|
|
191
317
|
}
|
|
318
|
+
/** Package version for report footers; dist/ and src/ both sit two levels deep. */
|
|
319
|
+
export function argusVersion() {
|
|
320
|
+
try {
|
|
321
|
+
return createRequire(import.meta.url)('../../package.json').version;
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
return 'unknown';
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Write the offline HTML evidence report (U14) beside run-manifest.json.
|
|
329
|
+
* Lane reports are read only when the manifest points at them: a runner that
|
|
330
|
+
* threw leaves `reportPath` unset, and its stale file must not reach the
|
|
331
|
+
* report (same contract as the manifest's own reads).
|
|
332
|
+
*/
|
|
333
|
+
export async function writeEvidenceReport(reportDir, manifest, meta = {}) {
|
|
334
|
+
const [codeReview, run] = await Promise.all([
|
|
335
|
+
manifest.lanes.review.reportPath !== undefined ? readJson(join(reportDir, 'code-review.json')) : undefined,
|
|
336
|
+
manifest.lanes.flow.reportPath !== undefined ? readJson(join(reportDir, 'run.json')) : undefined,
|
|
337
|
+
]);
|
|
338
|
+
const html = renderReportHtml({
|
|
339
|
+
manifestText: JSON.stringify(manifest),
|
|
340
|
+
codeReview,
|
|
341
|
+
run,
|
|
342
|
+
version: meta.version ?? argusVersion(),
|
|
343
|
+
...(meta.runUrl !== undefined ? { runUrl: meta.runUrl } : {}),
|
|
344
|
+
});
|
|
345
|
+
const path = join(reportDir, REPORT_HTML);
|
|
346
|
+
await writeAtomicText(path, html);
|
|
347
|
+
return path;
|
|
348
|
+
}
|
|
192
349
|
function ctxError(message) {
|
|
193
350
|
console.error(`verify lane error: ${message}`);
|
|
194
351
|
}
|
|
195
|
-
export function selectedManifestLanes(selection) {
|
|
196
|
-
return selectedLanes(selection);
|
|
197
|
-
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { type ProbeRecord } from './queue.js';
|
|
2
|
+
interface Ctx {
|
|
3
|
+
err: (line: string) => void;
|
|
4
|
+
}
|
|
5
|
+
/**
|
|
6
|
+
* `@argus persist` — reproduced probes become regression tests (roadmap
|
|
7
|
+
* E1.U3). Probe files are written transiently and deleted during review,
|
|
8
|
+
* so `reproduced` records carry their source + suggested path in
|
|
9
|
+
* code-review.json. The sticky comment embeds them as a base64 HTML-comment
|
|
10
|
+
* payload (`PERSIST_MARKER`) — a later `issue_comment` run on a base-only
|
|
11
|
+
* checkout decodes it and commits the probes to a new branch via the
|
|
12
|
+
* contents API, then opens one regression-test PR per source PR.
|
|
13
|
+
*
|
|
14
|
+
* Trust: payload content is model-authored. It passed the probe lane's
|
|
15
|
+
* validation once (forced `argus-probe-` filename, safe repo path, bounded
|
|
16
|
+
* imports), but the sticky comment is editable — decode() re-validates
|
|
17
|
+
* every path and bound before any write, and writes go to a dedicated
|
|
18
|
+
* branch, never the base ref.
|
|
19
|
+
*/
|
|
20
|
+
export interface PersistableProbe {
|
|
21
|
+
/** Repo-relative write path — `argus-probe-*` basename, validated. */
|
|
22
|
+
path: string;
|
|
23
|
+
/** Probe source (bounded). */
|
|
24
|
+
content: string;
|
|
25
|
+
/** Probe filename for messaging. */
|
|
26
|
+
file: string;
|
|
27
|
+
/** Finding file the probe covers, when known. */
|
|
28
|
+
findingFile?: string | undefined;
|
|
29
|
+
}
|
|
30
|
+
export declare const PERSIST_MARKER = "<!-- argus-probe-persist ";
|
|
31
|
+
/** Reproduced probes carrying serialized content + a safe suggested path. */
|
|
32
|
+
export declare function selectPersistable(records: ProbeRecord[] | undefined): PersistableProbe[];
|
|
33
|
+
/**
|
|
34
|
+
* Serialize the payload for the sticky comment — the full marker string,
|
|
35
|
+
* or undefined when nothing persistable exists or the payload would bust
|
|
36
|
+
* the cap (the copy-paste tier still renders probes that don't fit).
|
|
37
|
+
*/
|
|
38
|
+
export declare function encodeProbePayload(records: ProbeRecord[] | undefined, headSha: string | undefined): string | undefined;
|
|
39
|
+
export interface DecodedPayload {
|
|
40
|
+
head?: string | undefined;
|
|
41
|
+
probes: PersistableProbe[];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Parse a sticky comment body back into probes. Strict: exactly one
|
|
45
|
+
* marker, alphabet-checked base64, shaped JSON, and every entry through
|
|
46
|
+
* the same selectPersistable validation — a hand-edited comment fails
|
|
47
|
+
* closed to undefined rather than persisting attacker-controlled paths.
|
|
48
|
+
*/
|
|
49
|
+
export declare function decodeProbePayload(body: string): DecodedPayload | undefined;
|
|
50
|
+
export interface PersistResult {
|
|
51
|
+
/** Opened (or reused) regression-test PR URL. */
|
|
52
|
+
prUrl?: string;
|
|
53
|
+
/** Probe paths committed to the branch. */
|
|
54
|
+
written: string[];
|
|
55
|
+
/** Probe paths skipped because they already exist on the branch. */
|
|
56
|
+
skipped: string[];
|
|
57
|
+
/** Failure message when the lane couldn't complete. */
|
|
58
|
+
error?: string;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Commit persistable probes to `argus/probe-regression-pr-<pr>` and open a
|
|
62
|
+
* regression-test PR against `baseRef`. Idempotent: an existing branch is
|
|
63
|
+
* reused, an existing file is skipped (never overwritten — contents API
|
|
64
|
+
* exclusive create), and an already-open PR on the branch is returned
|
|
65
|
+
* rather than duplicated.
|
|
66
|
+
*/
|
|
67
|
+
export declare function persistProbes(repo: string, pr: string, baseRef: string, probes: PersistableProbe[], token: string, ctx: Ctx): Promise<PersistResult>;
|
|
68
|
+
export {};
|