argus-reviewer-e2e 0.1.2

package/dist/report/junit.js

@@ -0,0 +1,46 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+function escapeXml(value) {
+    return value
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+/** Render a JUnit XML report: one <testcase> per test with pass/fail + duration. */
+export function renderJunitXml(suiteName, cases) {
+    const failures = cases.filter((c) => !c.ok).length;
+    const totalSeconds = cases.reduce((sum, c) => sum + c.durationMs, 0) / 1000;
+    const body = cases
+        .map((c) => {
+        const attrs = [
+            `name="${escapeXml(c.name)}"`,
+            `classname="${escapeXml(c.className ?? suiteName)}"`,
+            `time="${(c.durationMs / 1000).toFixed(3)}"`,
+        ].join(' ');
+        if (c.ok) {
+            return `    <testcase ${attrs}/>`;
+        }
+        const message = escapeXml(c.failureMessage ?? 'test failed');
+        return [
+            `    <testcase ${attrs}>`,
+            `      <failure message="${message}"/>`,
+            `    </testcase>`,
+        ].join('\n');
+    })
+        .join('\n');
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites tests="${cases.length}" failures="${failures}" time="${totalSeconds.toFixed(3)}">`,
+        `  <testsuite name="${escapeXml(suiteName)}" tests="${cases.length}" failures="${failures}" time="${totalSeconds.toFixed(3)}">`,
+        body,
+        '  </testsuite>',
+        '</testsuites>',
+        '',
+    ].join('\n');
+}
+export async function writeJunitXml(path, suiteName, cases) {
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, renderJunitXml(suiteName, cases), 'utf8');
+}

package/dist/report/run.d.ts

@@ -0,0 +1,51 @@
+import type { HealEvent, TdAssertRecord, TdStepRecord } from '../api.js';
+import type { CallCost } from '../vision/cost.js';
+/**
+ * JSON run report consumed by the GitHub Action (R10): per-test verdicts,
+ * heal events, vision-call counts, ledger totals, and artifact paths.
+ */
+export interface TestReport {
+    name: string;
+    file: string;
+    ok: boolean;
+    durationMs: number;
+    failureMessage: string | undefined;
+    steps: TdStepRecord[];
+    asserts: TdAssertRecord[];
+    healEvents: HealEvent[];
+    visionCalls: number;
+    visionCostUsd: number;
+    sandboxSeconds: number;
+    budgetExceeded: boolean;
+    /** Per-call cost rows for model-level attribution. */
+    calls: CallCost[];
+    videoPath: string | undefined;
+    /** Agent Zero's autonomous second opinion on a failure (heal: 'a0'). */
+    a0Diagnosis?: string;
+}
+export interface RunTotals {
+    tests: number;
+    passed: number;
+    failed: number;
+    visionCalls: number;
+    visionCostUsd: number;
+    sandboxSeconds: number;
+    budgetExceeded: boolean;
+    /** OpenRouter spend grouped by model id. */
+    costByModel: Record<string, number>;
+    /** OpenRouter call count grouped by model id. */
+    callsByModel: Record<string, number>;
+}
+export interface RunReport {
+    tool: 'argus-reviewer';
+    startedAt: string;
+    durationMs: number;
+    ok: boolean;
+    totals: RunTotals;
+    tests: TestReport[];
+    artifacts: {
+        videos: string[];
+    };
+}
+export declare function buildRunReport(tests: TestReport[], startedAt: Date, durationMs: number): RunReport;
+export declare function writeRunReport(path: string, report: RunReport): Promise<void>;

package/dist/report/run.js

@@ -0,0 +1,36 @@
+import { writeAtomicJson } from '../fsutil.js';
+export function buildRunReport(tests, startedAt, durationMs) {
+    const failed = tests.filter((t) => !t.ok).length;
+    const callsByModel = {};
+    const costByModel = {};
+    for (const t of tests) {
+        for (const c of t.calls) {
+            callsByModel[c.model] = (callsByModel[c.model] ?? 0) + 1;
+            costByModel[c.model] = (costByModel[c.model] ?? 0) + c.costUsd;
+        }
+    }
+    return {
+        tool: 'argus-reviewer',
+        startedAt: startedAt.toISOString(),
+        durationMs,
+        ok: failed === 0,
+        totals: {
+            tests: tests.length,
+            passed: tests.length - failed,
+            failed,
+            visionCalls: tests.reduce((sum, t) => sum + t.visionCalls, 0),
+            visionCostUsd: tests.reduce((sum, t) => sum + t.visionCostUsd, 0),
+            sandboxSeconds: tests.reduce((sum, t) => sum + t.sandboxSeconds, 0),
+            budgetExceeded: tests.some((t) => t.budgetExceeded),
+            callsByModel,
+            costByModel,
+        },
+        tests,
+        artifacts: {
+            videos: tests.map((t) => t.videoPath).filter((p) => p !== undefined),
+        },
+    };
+}
+export async function writeRunReport(path, report) {
+    await writeAtomicJson(path, report);
+}

package/dist/vision/cost.d.ts

@@ -0,0 +1,36 @@
+export type CallKind = 'ground' | 'heal' | 'assert' | 'code';
+export interface CallCost {
+    model: string;
+    provider: string;
+    tokens: number;
+    costUsd: number;
+    kind: CallKind;
+}
+export interface ChoiceMessage {
+    content?: unknown;
+}
+export interface Choice {
+    message?: ChoiceMessage;
+}
+export interface Usage {
+    total_tokens: number;
+    cost: number;
+    cost_details: {
+        upstream_inference_cost: number;
+    };
+}
+export type ProviderValue = string | {
+    name?: string;
+} | undefined;
+export interface OpenRouterResponse {
+    id: string;
+    model: string;
+    provider?: ProviderValue;
+    choices: Choice[];
+    usage: Usage;
+}
+export declare function makeCallCost(response: OpenRouterResponse, kind: CallKind): CallCost;
+export declare function extractUsageCost(response: OpenRouterResponse): {
+    costUsd: number;
+    upstreamCostUsd: number;
+};

package/dist/vision/cost.js

@@ -0,0 +1,16 @@
+export function makeCallCost(response, kind) {
+    const providerName = typeof response.provider === 'string' ? response.provider : response.provider?.name ?? 'unknown';
+    return {
+        model: response.model,
+        provider: providerName,
+        tokens: response.usage.total_tokens,
+        costUsd: response.usage.cost,
+        kind,
+    };
+}
+export function extractUsageCost(response) {
+    return {
+        costUsd: response.usage.cost,
+        upstreamCostUsd: response.usage.cost_details.upstream_inference_cost,
+    };
+}

package/dist/vision/ledger.d.ts

@@ -0,0 +1,29 @@
+import { CallCost } from './cost.js';
+export interface LedgerState {
+    visionCostUsd: number;
+    sandboxSeconds: number;
+    budgetExceeded: boolean;
+    replayOnly: boolean;
+    calls: CallCost[];
+}
+export declare class Ledger {
+    private _budgetUsd;
+    private _visionCostUsd;
+    private _sandboxStartedAt;
+    private _sandboxSeconds;
+    private _calls;
+    private _budgetExceeded;
+    private _replayOnly;
+    constructor(_budgetUsd: number | undefined);
+    get state(): LedgerState;
+    get budgetExceeded(): boolean;
+    get replayOnly(): boolean;
+    get visionCostUsd(): number;
+    get sandboxSeconds(): number;
+    get calls(): CallCost[];
+    startSandbox(): void;
+    stopSandbox(): void;
+    recordCall(cost: CallCost): void;
+    canSpend(estimatedUsd: number): boolean;
+    flagBudgetExceeded(): void;
+}

package/dist/vision/ledger.js

@@ -0,0 +1,65 @@
+export class Ledger {
+    _budgetUsd;
+    _visionCostUsd = 0;
+    _sandboxStartedAt;
+    _sandboxSeconds = 0;
+    _calls = [];
+    _budgetExceeded = false;
+    _replayOnly = false;
+    constructor(_budgetUsd) {
+        this._budgetUsd = _budgetUsd;
+    }
+    get state() {
+        return {
+            visionCostUsd: this._visionCostUsd,
+            sandboxSeconds: this._sandboxSeconds,
+            budgetExceeded: this._budgetExceeded,
+            replayOnly: this._replayOnly,
+            calls: [...this._calls],
+        };
+    }
+    get budgetExceeded() {
+        return this._budgetExceeded;
+    }
+    get replayOnly() {
+        return this._replayOnly;
+    }
+    get visionCostUsd() {
+        return this._visionCostUsd;
+    }
+    get sandboxSeconds() {
+        return this._sandboxSeconds;
+    }
+    get calls() {
+        return [...this._calls];
+    }
+    startSandbox() {
+        this._sandboxStartedAt = Date.now();
+    }
+    stopSandbox() {
+        if (this._sandboxStartedAt === undefined)
+            return;
+        this._sandboxSeconds += (Date.now() - this._sandboxStartedAt) / 1000;
+        this._sandboxStartedAt = undefined;
+    }
+    recordCall(cost) {
+        this._calls.push(cost);
+        this._visionCostUsd += cost.costUsd;
+    }
+    canSpend(estimatedUsd) {
+        if (this._budgetUsd === undefined)
+            return true;
+        if (this._budgetExceeded)
+            return false;
+        const projected = this._visionCostUsd + estimatedUsd;
+        if (projected <= this._budgetUsd)
+            return true;
+        this._budgetExceeded = true;
+        this._replayOnly = true;
+        return false;
+    }
+    flagBudgetExceeded() {
+        this._budgetExceeded = true;
+        this._replayOnly = true;
+    }
+}

package/dist/vision/openrouter.d.ts

@@ -0,0 +1,70 @@
+import { ProviderRules } from '../config.js';
+import { CallCost, CallKind } from './cost.js';
+export interface TextContentPart {
+    type: 'text';
+    text: string;
+}
+export interface ImageContentPart {
+    type: 'image';
+    source: string;
+}
+export type ContentPart = TextContentPart | ImageContentPart;
+export interface Message {
+    role: 'user' | 'system' | 'assistant';
+    content: ContentPart[];
+}
+export interface JsonSchema {
+    name: string;
+    schema: Record<string, unknown>;
+    strict?: boolean;
+}
+export interface OpenRouterClientOptions {
+    apiKey: string;
+    fetch?: typeof fetch;
+    /**
+     * Extra metadata sent on every request. `trace` is merged into the request
+     * body `trace` field for cost attribution. `headers` are merged into the
+     * request headers (e.g. HTTP-Referer, X-Title).
+     */
+    trace?: Record<string, string>;
+    headers?: Record<string, string>;
+    /**
+     * Called once for every successful OpenRouter request with the resolved
+     * model, cost, and token usage. Useful for per-call logging.
+     */
+    onCall?: (call: {
+        id: string;
+        model: string;
+        kind: CallKind;
+        costUsd: number;
+        tokens: number;
+        trace?: Record<string, string>;
+    }) => void;
+}
+export declare class OpenRouterClient {
+    private _apiKey;
+    private _fetch;
+    private _trace;
+    private _headers;
+    private _onCall;
+    constructor(opts: OpenRouterClientOptions);
+    complete(opts: {
+        model: string;
+        messages: Message[];
+        schema?: JsonSchema;
+        escalationModels?: string[];
+        provider?: ProviderRules;
+        kind?: CallKind;
+    }): Promise<{
+        id: string;
+        content: string;
+        cost: CallCost;
+        model: string;
+    }>;
+    reconcile(id: string): Promise<{
+        costUsd: number;
+    }>;
+    private _tryComplete;
+    private _toApiMessages;
+    private _extractContent;
+}

package/dist/vision/openrouter.js

@@ -0,0 +1,134 @@
+import { debug } from '../debug.js';
+import { makeCallCost } from './cost.js';
+export class OpenRouterClient {
+    _apiKey;
+    _fetch;
+    _trace;
+    _headers;
+    _onCall;
+    constructor(opts) {
+        if (!opts.apiKey) {
+            throw new Error('OPENROUTER_API_KEY is required: pass a non-empty apiKey at construction');
+        }
+        this._apiKey = opts.apiKey;
+        this._fetch = opts.fetch ?? globalThis.fetch;
+        this._trace = opts.trace;
+        this._headers = opts.headers;
+        this._onCall = opts.onCall;
+    }
+    async complete(opts) {
+        const candidates = [opts.model, ...(opts.escalationModels ?? [])];
+        const errors = [];
+        const kind = opts.kind ?? 'ground';
+        debug('openrouter', `kind=${kind} candidates=[${candidates.join(', ')}] messages=${opts.messages.length}`);
+        for (let i = 0; i < candidates.length; i++) {
+            const model = candidates[i];
+            if (model === undefined)
+                continue;
+            const remaining = candidates.slice(i + 1);
+            try {
+                const response = await this._tryComplete({
+                    model,
+                    messages: opts.messages,
+                    ...(opts.schema !== undefined ? { schema: opts.schema } : {}),
+                    ...(opts.provider !== undefined ? { provider: opts.provider } : {}),
+                    models: remaining,
+                    kind,
+                });
+                const cost = makeCallCost(response, kind);
+                debug('openrouter', `model=${response.model} tokens=${cost.tokens} costUsd=${cost.costUsd.toFixed(6)} content_len=${this._extractContent(response).length}`);
+                this._onCall?.({
+                    id: response.id,
+                    model: response.model,
+                    kind,
+                    costUsd: cost.costUsd,
+                    tokens: cost.tokens,
+                    ...(this._trace ? { trace: this._trace } : {}),
+                });
+                return { id: response.id, content: this._extractContent(response), cost, model: response.model };
+            }
+            catch (e) {
+                const err = e;
+                debug('openrouter', `candidate ${model} failed: ${err.message}`);
+                errors.push(err);
+            }
+        }
+        throw new Error(`OpenRouter completion failed for all candidate models: ${errors.map((e) => e.message).join('; ')}`);
+    }
+    async reconcile(id) {
+        const res = await this._fetch(`https://openrouter.ai/api/v1/generation?id=${encodeURIComponent(id)}`, {
+            method: 'GET',
+            headers: { Authorization: `Bearer ${this._apiKey}` },
+        });
+        if (!res.ok) {
+            throw new Error(`OpenRouter generation reconcile failed: ${res.status} ${res.statusText}`);
+        }
+        const data = (await res.json());
+        const total = data.data?.total_cost ?? data.total_cost ?? 0;
+        return { costUsd: total };
+    }
+    async _tryComplete(req) {
+        const body = {
+            model: req.model,
+            messages: this._toApiMessages(req.messages),
+        };
+        if (req.provider) {
+            body.provider = req.provider;
+        }
+        if (req.models && req.models.length > 0) {
+            body.models = req.models;
+        }
+        if (req.schema) {
+            body.response_format = {
+                type: 'json_schema',
+                json_schema: {
+                    name: req.schema.name,
+                    schema: req.schema.schema,
+                    strict: req.schema.strict ?? true,
+                },
+            };
+        }
+        if (this._trace) {
+            body.trace = this._trace;
+        }
+        const headers = {
+            Authorization: `Bearer ${this._apiKey}`,
+            'Content-Type': 'application/json',
+            'X-Title': 'argus-reviewer',
+            'X-OpenRouter-Metadata': 'enabled',
+            ...(this._headers ?? {}),
+        };
+        const res = await this._fetch('https://openrouter.ai/api/v1/chat/completions', {
+            method: 'POST',
+            headers,
+            body: JSON.stringify(body),
+        });
+        if (!res.ok) {
+            throw new Error(`OpenRouter request failed: ${res.status} ${res.statusText}`);
+        }
+        return (await res.json());
+    }
+    _toApiMessages(messages) {
+        return messages.map((m) => ({
+            role: m.role,
+            content: m.content
+                .map((part) => {
+                if (part.type === 'text') {
+                    return { type: 'text', text: part.text };
+                }
+                return {
+                    type: 'image_url',
+                    image_url: { url: `data:image/jpeg;base64,${part.source}` },
+                };
+            })
+                .sort((a) => (a.type === 'text' ? -1 : 1)),
+        }));
+    }
+    _extractContent(response) {
+        const first = response.choices[0];
+        const content = first?.message?.content;
+        if (typeof content === 'string')
+            return content;
+        return '';
+    }
+}

package/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "argus-reviewer-e2e",
+  "version": "0.1.2",
+  "description": "argus-reviewer — open-source vision-model E2E testing harness. Record, replay, heal, and assert with any OpenRouter model under a hard budget cap.",
+  "type": "module",
+  "license": "MIT",
+  "author": "Luke Kimball",
+  "homepage": "https://github.com/duketopceo/Argus#readme",
+  "bugs": "https://github.com/duketopceo/Argus/issues",
+  "keywords": [
+    "e2e-testing",
+    "vision-model",
+    "playwright",
+    "code-review",
+    "openrouter",
+    "github-actions",
+    "ai-testing",
+    "byok"
+  ],
+  "bin": {
+    "argus-reviewer": "dist/cli.js"
+  },
+  "exports": {
+    ".": "./dist/api.js",
+    "./api": "./dist/api.js",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "action"
+  ],
+  "engines": {
+    "node": ">=20.19.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "typecheck": "tsc --noEmit",
+    "prepare": "npm run build && chmod +x dist/cli.js",
+    "eval": "node evals/run.mjs",
+    "watch": "node scripts/watch.mjs",
+    "app": "electron electron"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.0.0",
+    "@types/node": "^20.0.0",
+    "electron": "^44.2.0",
+    "eslint": "^9.0.0",
+    "globals": "^15.0.0",
+    "prettier": "^3.0.0",
+    "typescript-eslint": "^8.0.0",
+    "vitest": "^2.0.0"
+  },
+  "dependencies": {
+    "playwright": "1.63.0",
+    "typescript": "^5.5.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/duketopceo/Argus.git"
+  }
+}