argus-reviewer-e2e 0.2.0 → 0.3.1

package/dist/config.js

@@ -47,6 +47,7 @@ const defaults = {
         triage: 'annotate',
         lowRiskModel: undefined,
         findingThreshold: 1.0,
+        requestChanges: true,
     },
 };
 export function defineConfig(input) {
@@ -123,6 +124,9 @@ export function resolveConfig(input = {}) {
         review.lowRiskModel = undefined;
     }
     review.findingThreshold = prob01(review.findingThreshold, defaults.review.findingThreshold);
+    // Advisory-only escape hatch — only literal `false` opts out; anything
+    // else (mis-typed values included) keeps the default-true posture.
+    review.requestChanges = review.requestChanges !== false;
     const resolved = { ...defaults, ...input, provider, sandbox, review };
     resolved.recordStepCap = posInt(resolved.recordStepCap, DEFAULT_RECORD_STEP_CAP);
     if (resolved.heal !== 'a0')
@@ -153,6 +157,30 @@ export async function loadConfig(cwd, opts) {
     const fs = await import('node:fs/promises');
     const path = await import('node:path');
     const untrusted = opts.trust === 'untrusted';
+    // cacheDir is the documented CLI default '.argus-reviewer-cache' — normalize
+    // it to an absolute path here so engine record/replay persistence (gated on
+    // config.cacheDir) writes where every other consumer already falls back to.
+    const finish = async (input = {}) => {
+        const config = resolveConfig(input);
+        const cacheDir = path.resolve(cwd, config.cacheDir ?? '.argus-reviewer-cache');
+        if (untrusted) {
+            // A hostile PR can commit the default path as a symlink and redirect
+            // cache writes outside the checkout — fail closed before writers run.
+            let isSymlink = false;
+            try {
+                isSymlink = (await fs.lstat(cacheDir)).isSymbolicLink();
+            }
+            catch (e) {
+                if (e.code !== 'ENOENT')
+                    throw e;
+            }
+            if (isSymlink) {
+                throw new Error(`untrusted cache directory must not be a symlink: ${cacheDir}`);
+            }
+        }
+        config.cacheDir = cacheDir;
+        return config;
+    };
     const names = ['argus-reviewer.config', 'vision-e2e.config'];
     for (const name of names) {
         if (untrusted) {
@@ -181,9 +209,9 @@ export async function loadConfig(cwd, opts) {
                     const parsed = JSON.parse(raw);
                     if (untrusted) {
                         opts.note?.(`config: ${name}.json loaded untrusted — honoring ${[...UNTRUSTED_CONFIG_KEYS].join(', ')} only`);
-                        return resolveConfig(filterUntrustedConfig(parsed));
+                        return finish(filterUntrustedConfig(parsed));
                     }
-                    return resolveConfig(parsed);
+                    return finish(parsed);
                 }
                 // Always transpile .ts to a temp .mjs rather than importing natively:
                 // Node's built-in type stripping resolves the module type from the
@@ -214,7 +242,7 @@ export async function loadConfig(cwd, opts) {
                     await rm(out, { force: true }).catch(() => undefined);
                 }
                 const exported = mod.default ?? mod;
-                return resolveConfig(exported);
+                return finish(exported);
             }
             catch (e) {
                 const code = e.code;
@@ -224,7 +252,7 @@ export async function loadConfig(cwd, opts) {
             }
         }
     }
-    return resolveConfig();
+    return finish();
 }
 /**
  * Provider slugs the harness recognizes for `provider.only/ignore/order`

package/dist/engine/loop.d.ts

@@ -62,6 +62,15 @@ export interface RunResult {
     reason?: string;
     steps: StepResult[];
     visionCalls: number;
+    cache: CacheStats;
+}
+export interface CacheStats {
+    hits: number;
+    misses: number;
+    heals: number;
+    staleEntries: number;
+    assertionHits: number;
+    assertionMisses: number;
 }
 export interface AssertResult extends AssertionResult {
     cached: boolean;
@@ -81,6 +90,7 @@ export declare class Engine {
     private _fingerprints;
     private _assertCache;
     private _errors;
+    private _cacheStats;
     /** Structured, non-fatal anomalies — journaled as evidence, never thrown. */
     get errorRecords(): ErrorRecord[];
     private _note;
@@ -88,6 +98,7 @@ export declare class Engine {
     /** Assertion verdicts collected/known this run — persist into the flow cache. */
     get assertEntries(): CachedAssert[];
     get visionCalls(): number;
+    get cacheStats(): CacheStats;
     record(instruction: string, tdApi?: TestDriverApi, options?: RecordOptions): Promise<RunResult>;
     replay(flow: FlowCache, options?: ReplayOptions): Promise<RunResult>;
     /**
@@ -114,5 +125,6 @@ export declare class Engine {
     private _buildFingerprint;
     private _regionScreenshot;
     private _result;
+    private _resetCacheStats;
 }
 export declare function instructionMatchesNode(instruction: string, nodeSnippet: string): boolean;

package/dist/engine/loop.js

@@ -9,6 +9,14 @@ export class Engine {
     _fingerprints = [];
     _assertCache = new Map();
     _errors = [];
+    _cacheStats = {
+        hits: 0,
+        misses: 0,
+        heals: 0,
+        staleEntries: 0,
+        assertionHits: 0,
+        assertionMisses: 0,
+    };
     /** Structured, non-fatal anomalies — journaled as evidence, never thrown. */
     get errorRecords() {
         return this._errors;
@@ -31,10 +39,14 @@ export class Engine {
     get visionCalls() {
         return this._visionCalls;
     }
+    get cacheStats() {
+        return { ...this._cacheStats };
+    }
     async record(instruction, tdApi = this._opts.actions, options = {}) {
         this._visionCalls = 0;
         this._steps = [];
         this._fingerprints = [];
+        this._resetCacheStats();
         const cap = options.stepCap ?? this._opts.config.recordStepCap ?? DEFAULT_RECORD_STEP_CAP;
         let observation = await this._opts.driver.observe({ grid: true });
         for (let i = 0; i < cap; i++) {
@@ -82,16 +94,18 @@ export class Engine {
     async replay(flow, options = {}) {
         this._visionCalls = 0;
         this._steps = [];
+        this._resetCacheStats();
         for (let i = 0; i < flow.steps.length; i++) {
             const step = flow.steps[i];
             if (!step)
                 continue;
-            let observation = await this._opts.driver.observe({ grid: true });
+            const observation = await this._opts.driver.observe({ grid: true });
             // Diff-invalidated entries skip hash verification entirely and go
             // straight to the heal path — the diff already told us they're stale.
             let resolve;
             if (step.stale !== undefined) {
                 this._note('heal', 'cache entry invalidated by diff', step.stale);
+                this._cacheStats.staleEntries++;
                 resolve = { matched: false, currentHash: '', regionMatched: false, a11yMatched: false };
             }
             else {
@@ -99,10 +113,12 @@ export class Engine {
                 resolve = new Fingerprint(step).resolve(regionBuffer, observation.a11yYaml);
             }
             if (resolve.matched) {
+                this._cacheStats.hits++;
                 await this._executeAction(this._opts.actions, step.action);
                 this._steps.push({ instruction: step.instruction, action: step.action.action, ok: true });
                 continue;
             }
+            this._cacheStats.misses++;
             if (this._opts.ledger.replayOnly || !this._opts.ledger.canSpend(0.001)) {
                 this._steps.push({
                     instruction: step.instruction,
@@ -144,9 +160,10 @@ export class Engine {
                 return this._result(false);
             }
             const resolved = await this._resolveAction(action);
-            const nextObservation = await this._executeAction(this._opts.actions, action);
+            await this._executeAction(this._opts.actions, action);
             const newFingerprint = await this._buildFingerprint(step.instruction, action, resolved, response.model);
             flow.steps[i] = newFingerprint;
+            this._cacheStats.heals++;
             this._note('heal', 'fingerprint mismatch healed by model', step.instruction);
             this._steps.push({
                 instruction: step.instruction,
@@ -155,7 +172,6 @@ export class Engine {
                 healed: true,
                 model: response.model,
             });
-            observation = nextObservation;
         }
         if (options.flowName && this._opts.config.cacheDir) {
             await saveFlow(this._opts.config.cacheDir, options.flowName, flow.steps);
@@ -177,6 +193,7 @@ export class Engine {
             const regionBuffer = await this._regionScreenshot(cached.bbox);
             const resolve = new Fingerprint(cached).resolve(regionBuffer, observation.a11yYaml);
             if (resolve.matched) {
+                this._cacheStats.hits++;
                 return {
                     ok: true,
                     reason: undefined,
@@ -186,6 +203,7 @@ export class Engine {
                     model: undefined,
                 };
             }
+            this._cacheStats.misses++;
             if (this._opts.ledger.replayOnly || !this._opts.ledger.canSpend(0.001)) {
                 return {
                     ok: false,
@@ -197,6 +215,13 @@ export class Engine {
                 };
             }
         }
+        else if (cached?.stale !== undefined) {
+            this._cacheStats.misses++;
+            this._cacheStats.staleEntries++;
+        }
+        else {
+            this._cacheStats.misses++;
+        }
         // A diff-invalidated (stale) entry is a fresh ground, not a heal — heal
         // implies the fingerprint *checked out as wrong*, stale means we never
         // verified it. Keeping the kind split honest keeps the heal-rate signal
@@ -205,8 +230,11 @@ export class Engine {
         const isStale = cached !== undefined && cached.stale !== undefined;
         const useHeal = cached !== undefined && !isStale;
         const primary = await this._locateWithModel(instruction, observation, useHeal);
-        if (primary.ok)
+        if (primary.ok) {
+            if (useHeal)
+                this._cacheStats.heals++;
             return primary;
+        }
         // Semantic escalation fallback (issue #14): the model answered but could
         // not ground — provider-level OpenRouter fallback only covers unavailable
         // models, not bad answers. Retry once with escalation_model as primary on
@@ -221,7 +249,10 @@ export class Engine {
         }
         this._note('locate', 'escalating to fallback model', `failed=${failedModel} esc=${esc} reason=${(primary.reason ?? '').slice(0, 80)}`);
         const fresh = await this._opts.driver.observe({ grid: true });
-        return this._locateWithModel(instruction, fresh, useHeal, esc);
+        const escalated = await this._locateWithModel(instruction, fresh, useHeal, esc);
+        if (escalated.ok && useHeal)
+            this._cacheStats.heals++;
+        return escalated;
     }
     /**
      * One locate attempt against a specific model: initial call plus the
@@ -388,8 +419,10 @@ export class Engine {
         const key = `${question}${a11yHash}`;
         const cached = this._assertCache.get(key);
         if (cached) {
+            this._cacheStats.assertionHits++;
             return { verdict: cached.verdict, reasoning: cached.reasoning, cached: true };
         }
+        this._cacheStats.assertionMisses++;
         if (this._opts.ledger.replayOnly || !this._opts.ledger.canSpend(0.001)) {
             return { verdict: 'fail', reasoning: 'budget exceeded or replay-only', cached: false };
         }
@@ -621,7 +654,23 @@ export class Engine {
         return Buffer.from(raw);
     }
     _result(ok, reason) {
-        return { ok, steps: this._steps, visionCalls: this._visionCalls, ...(reason ? { reason } : {}) };
+        return {
+            ok,
+            steps: this._steps,
+            visionCalls: this._visionCalls,
+            cache: this.cacheStats,
+            ...(reason ? { reason } : {}),
+        };
+    }
+    _resetCacheStats() {
+        this._cacheStats = {
+            hits: 0,
+            misses: 0,
+            heals: 0,
+            staleEntries: 0,
+            assertionHits: 0,
+            assertionMisses: 0,
+        };
     }
 }
 /**

package/dist/pipeline/budget.d.ts

@@ -0,0 +1,11 @@
+import type { CallCost } from '../vision/cost.js';
+import type { BudgetSummary, LaneId } from '../report/manifest.js';
+export interface BudgetOptions {
+    limitUsd?: number;
+    maxDurationMs?: number;
+    maxTasks?: number;
+}
+export declare function createBudget(lane: LaneId, options?: BudgetOptions): BudgetSummary;
+export declare function addProviderCalls(budget: BudgetSummary, calls: CallCost[] | undefined): BudgetSummary;
+export declare function addA0Task(budget: BudgetSummary, elapsedMs: number, metered: boolean): BudgetSummary;
+export declare function budgetCanSpend(budget: BudgetSummary, nextCostUsd: number): boolean;

package/dist/pipeline/budget.js

@@ -0,0 +1,48 @@
+export function createBudget(lane, options = {}) {
+    void lane;
+    return {
+        limitUsd: options.limitUsd,
+        spentUsd: 0,
+        exceeded: false,
+        maxDurationMs: options.maxDurationMs,
+        elapsedMs: 0,
+        maxTasks: options.maxTasks,
+        tasks: 0,
+    };
+}
+/**
+ * USD spend is summed as IEEE-754 doubles, so a lane that lands exactly on its
+ * cap (0.1 + 0.2 === 0.30000000000000004, not 0.3) would otherwise read as over
+ * budget and abort a run that never exceeded the cap. Compare with a tolerance
+ * far below the 1e-6 precision reports render, and far above double noise here.
+ */
+const USD_EPSILON = 1e-9;
+function overLimit(spentUsd, limitUsd) {
+    return spentUsd > limitUsd + USD_EPSILON;
+}
+export function addProviderCalls(budget, calls) {
+    if (calls === undefined || calls.length === 0)
+        return budget;
+    const spentUsd = calls.reduce((sum, call) => sum + call.costUsd, 0);
+    const exceeded = budget.exceeded ||
+        (budget.limitUsd !== undefined && overLimit(budget.spentUsd + spentUsd, budget.limitUsd));
+    return {
+        ...budget,
+        spentUsd: budget.spentUsd + spentUsd,
+        exceeded,
+    };
+}
+export function addA0Task(budget, elapsedMs, metered) {
+    const tasks = budget.tasks + 1;
+    const elapsed = budget.elapsedMs + elapsedMs;
+    const exceeded = budget.exceeded ||
+        (budget.maxTasks !== undefined && tasks > budget.maxTasks) ||
+        (budget.maxDurationMs !== undefined && elapsed > budget.maxDurationMs) ||
+        (budget.limitUsd !== undefined && metered && overLimit(budget.spentUsd, budget.limitUsd));
+    return { ...budget, tasks, elapsedMs: elapsed, exceeded };
+}
+export function budgetCanSpend(budget, nextCostUsd) {
+    return (!budget.exceeded &&
+        (budget.limitUsd === undefined ||
+            budget.spentUsd + nextCostUsd <= budget.limitUsd + USD_EPSILON));
+}

package/dist/pipeline/contracts.d.ts

@@ -0,0 +1,12 @@
+import { LANE_IDS, type LaneId } from '../report/manifest.js';
+export { LANE_IDS };
+export type { LaneId };
+export interface LaneSelection {
+    review: boolean;
+    flow: boolean;
+    app: boolean;
+    a0: boolean;
+}
+export declare function defaultLaneSelection(): LaneSelection;
+export declare function selectionFromFlags(flags: Partial<Record<LaneId, boolean>>): LaneSelection;
+export declare function selectedLanes(selection: LaneSelection): LaneId[];

package/dist/pipeline/contracts.js

@@ -0,0 +1,16 @@
+import { LANE_IDS } from '../report/manifest.js';
+export { LANE_IDS };
+export function defaultLaneSelection() {
+    return { review: true, flow: false, app: false, a0: false };
+}
+export function selectionFromFlags(flags) {
+    const selection = defaultLaneSelection();
+    for (const lane of LANE_IDS) {
+        if (flags[lane] !== undefined)
+            selection[lane] = flags[lane];
+    }
+    return selection;
+}
+export function selectedLanes(selection) {
+    return LANE_IDS.filter((lane) => selection[lane]);
+}

package/dist/pipeline/verify.d.ts

@@ -0,0 +1,27 @@
+import { type LaneId, type RunIdentity, type RunManifest } from '../report/manifest.js';
+import { type BudgetOptions } from './budget.js';
+import { type LaneSelection } from './contracts.js';
+export interface VerifyRunners {
+    review: () => Promise<number>;
+    flow?: (url: string) => Promise<number>;
+    app?: () => Promise<number>;
+    a0?: () => Promise<number>;
+}
+export interface VerifyInput {
+    cwd: string;
+    runId: string;
+    reportDir: string;
+    identity: RunIdentity;
+    selection: LaneSelection;
+    runners: VerifyRunners;
+    budgets?: Partial<Record<LaneId, BudgetOptions>>;
+    flowUrl?: string;
+    flowUnavailableReason?: string;
+}
+export interface VerifyResult {
+    manifest: RunManifest;
+    exitCode: number;
+}
+/** Run selected lanes and return one stable manifest without owning subprocesses. */
+export declare function runVerify(input: VerifyInput): Promise<VerifyResult>;
+export declare function selectedManifestLanes(selection: LaneSelection): LaneId[];

package/dist/pipeline/verify.js

@@ -0,0 +1,197 @@
+import { readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { aggregateLanes, addProviderUsage, emptyLane, emptyUsage, isHeadBindingConclusive, MANIFEST_SCHEMA_VERSION, } from '../report/manifest.js';
+import { addProviderCalls, createBudget } from './budget.js';
+import { selectedLanes } from './contracts.js';
+async function readJson(path) {
+    try {
+        return JSON.parse(await readFile(path, 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+}
+function relativeReport(cwd, reportDir, name) {
+    return relative(cwd, join(reportDir, name)) || name;
+}
+function laneStart(lane, at = new Date()) {
+    return { ...lane, startedAt: at.toISOString() };
+}
+function laneEnd(lane, at = new Date()) {
+    return { ...lane, finishedAt: at.toISOString() };
+}
+function reportStatus(code, report, inconclusive) {
+    if (report?.skipped === true)
+        return 'skipped';
+    if (inconclusive)
+        return 'inconclusive';
+    if (report?.ok === true && code === 0)
+        return 'passed';
+    return 'failed';
+}
+function providerUsageFromCalls(calls) {
+    const usage = emptyUsage('openrouter');
+    return calls === undefined ? usage : addProviderUsage(usage, calls);
+}
+function flowCalls(report) {
+    return report?.tests?.flatMap((test) => test.calls ?? []) ?? [];
+}
+function flowUsage(report) {
+    const calls = flowCalls(report);
+    const usage = providerUsageFromCalls(calls);
+    if (report?.totals !== undefined && calls.length === 0) {
+        return {
+            ...usage,
+            calls: report.totals.visionCalls ?? 0,
+            tokens: 0,
+            costUsd: report.totals.visionCostUsd ?? 0,
+        };
+    }
+    return usage;
+}
+function budgetFor(lane, input, report) {
+    const options = input.budgets?.[lane] ?? {};
+    let budget = createBudget(lane, options);
+    if (lane === 'review') {
+        budget = addProviderCalls(budget, report?.calls);
+    }
+    else if (lane === 'flow') {
+        budget = addProviderCalls(budget, flowCalls(report));
+    }
+    return budget;
+}
+function baseLane(lane, selected) {
+    return emptyLane(lane, selected);
+}
+/** Run selected lanes and return one stable manifest without owning subprocesses. */
+export async function runVerify(input) {
+    const startedAt = new Date();
+    const lanes = Object.fromEntries(['review', 'flow', 'app', 'a0'].map((lane) => [
+        lane,
+        baseLane(lane, input.selection[lane]),
+    ]));
+    const reviewSelected = input.selection.review;
+    if (reviewSelected) {
+        lanes.review = laneStart(lanes.review, startedAt);
+        let code = 1;
+        let runnerError;
+        try {
+            code = await input.runners.review();
+        }
+        catch (error) {
+            runnerError = error instanceof Error ? error.message : String(error);
+            ctxError(runnerError);
+        }
+        const report = await readJson(join(input.reportDir, 'code-review.json'));
+        const inconclusive = report !== undefined &&
+            report.skipped !== true &&
+            !isHeadBindingConclusive(report.headBinding);
+        const status = runnerError !== undefined ? 'failed' : reportStatus(code, report, inconclusive);
+        lanes.review = laneEnd({
+            ...lanes.review,
+            status,
+            reportPath: report === undefined
+                ? undefined
+                : relativeReport(input.cwd, input.reportDir, 'code-review.json'),
+            model: report?.model,
+            summary: report?.summary,
+            reason: runnerError ??
+                (report === undefined
+                    ? 'code-review.json was not produced'
+                    : status === 'passed'
+                        ? undefined
+                        : report.summary),
+            usage: providerUsageFromCalls(report?.calls),
+            budget: budgetFor('review', input, report),
+            headBinding: report?.headBinding,
+        });
+    }
+    if (input.selection.flow) {
+        lanes.flow = laneStart(lanes.flow, startedAt);
+        if (input.runners.flow === undefined) {
+            lanes.flow = laneEnd({
+                ...lanes.flow,
+                status: 'unavailable',
+                reason: 'flow runner is not available in this build',
+            });
+        }
+        else if (input.flowUrl === undefined || input.flowUrl === '') {
+            lanes.flow = laneEnd({
+                ...lanes.flow,
+                status: 'blocked',
+                reason: input.flowUnavailableReason ?? 'no application target configured',
+            });
+        }
+        else {
+            let code = 1;
+            let runnerError;
+            try {
+                code = await input.runners.flow(input.flowUrl);
+            }
+            catch (error) {
+                runnerError = error instanceof Error ? error.message : String(error);
+                ctxError(runnerError);
+            }
+            const report = await readJson(join(input.reportDir, 'run.json'));
+            lanes.flow = laneEnd({
+                ...lanes.flow,
+                status: runnerError !== undefined ? 'failed' : reportStatus(code, report, false),
+                reportPath: report === undefined ? undefined : relativeReport(input.cwd, input.reportDir, 'run.json'),
+                summary: report === undefined ? undefined : report.ok === true ? 'flow passed' : 'flow failed',
+                reason: runnerError ??
+                    (report === undefined
+                        ? 'run.json was not produced'
+                        : report.ok === true
+                            ? undefined
+                            : 'flow failed'),
+                usage: flowUsage(report),
+                budget: budgetFor('flow', input, report),
+            });
+        }
+    }
+    for (const lane of ['app', 'a0']) {
+        if (!input.selection[lane])
+            continue;
+        const runner = input.runners[lane];
+        lanes[lane] = laneStart(lanes[lane], startedAt);
+        if (runner === undefined) {
+            lanes[lane] = laneEnd({
+                ...lanes[lane],
+                status: 'unavailable',
+                reason: `${lane} runner is not available in this build`,
+            });
+            continue;
+        }
+        let code = 1;
+        let runnerError;
+        try {
+            code = await runner();
+        }
+        catch (error) {
+            runnerError = error instanceof Error ? error.message : String(error);
+            ctxError(runnerError);
+        }
+        lanes[lane] = laneEnd({
+            ...lanes[lane],
+            status: code === 0 ? 'passed' : 'failed',
+            reason: runnerError ?? (code === 0 ? undefined : `${lane} runner exited ${code}`),
+        });
+    }
+    const aggregate = aggregateLanes(lanes);
+    const manifest = {
+        schemaVersion: MANIFEST_SCHEMA_VERSION,
+        runId: input.runId,
+        startedAt: startedAt.toISOString(),
+        finishedAt: new Date().toISOString(),
+        identity: input.identity,
+        lanes,
+        aggregate,
+    };
+    return { manifest, exitCode: aggregate.ok ? 0 : 1 };
+}
+function ctxError(message) {
+    console.error(`verify lane error: ${message}`);
+}
+export function selectedManifestLanes(selection) {
+    return selectedLanes(selection);
+}