argus-reviewer-e2e 0.1.3 → 0.2.0

package/dist/trust.js

@@ -0,0 +1,103 @@
+import { readFile } from 'node:fs/promises';
+async function readEventPayload(env, readEventFile) {
+    const eventPath = env.GITHUB_EVENT_PATH;
+    if (eventPath === undefined || eventPath === '')
+        return undefined;
+    try {
+        const parsed = JSON.parse(await readEventFile(eventPath));
+        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            return undefined;
+        }
+        return parsed;
+    }
+    catch {
+        return undefined;
+    }
+}
+function tracePr(env) {
+    const raw = env.ARGUS_REVIEWER_TRACE;
+    if (raw === undefined)
+        return undefined;
+    try {
+        const parsed = JSON.parse(raw);
+        return typeof parsed.pr === 'string' && parsed.pr !== '' ? parsed.pr : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function fetchMetaFor(opts, prOverride) {
+    const { env, fetchMeta } = opts;
+    if (fetchMeta === undefined)
+        return undefined;
+    const repo = env.GITHUB_REPOSITORY;
+    const token = env.GITHUB_TOKEN ?? env.GH_TOKEN;
+    const pr = prOverride ?? tracePr(env);
+    if (repo === undefined || token === undefined || pr === undefined)
+        return undefined;
+    return fetchMeta(repo, pr, token);
+}
+/**
+ * Resolves whether the checked-out tree may execute config code.
+ *
+ * - `pull_request*` events: fork status from the event payload — no token
+ *   needed (`head.repo.fork`). Payload absent/unreadable → `fetchMeta`
+ *   fallback → still unknown → untrusted.
+ * - `issue_comment`: `fetchMeta` on `issue.number`; unavailable → untrusted.
+ * - Any other present `GITHUB_EVENT_NAME` (`workflow_run`, `push`,
+ *   `workflow_dispatch`, …): untrusted — unlisted CI events fail closed
+ *   because privileged-CI-over-fork-checkout patterns (workflow_run over a
+ *   fork SHA) land exactly there. Maintainers opt out explicitly with
+ *   `ARGUS_TRUSTED=1`.
+ * - No event env at all (local run): trusted unless `ARGUS_UNTRUSTED=1`.
+ * - `ARGUS_UNTRUSTED=1` always wins; `ARGUS_TRUSTED=1` overrides event
+ *   resolution but never `ARGUS_UNTRUSTED`.
+ */
+export async function resolveTrust(opts) {
+    const { env, note } = opts;
+    const readEventFile = opts.readEventFile ?? ((p) => readFile(p, 'utf8'));
+    const done = (trust, reason, pr) => {
+        note?.(`trust: ${trust} — ${reason}`);
+        return { trust, reason, pr };
+    };
+    if (env.ARGUS_UNTRUSTED === '1') {
+        return done('untrusted', 'ARGUS_UNTRUSTED=1');
+    }
+    if (env.ARGUS_TRUSTED === '1') {
+        return done('trusted', 'ARGUS_TRUSTED=1 override');
+    }
+    const eventName = env.GITHUB_EVENT_NAME;
+    if (eventName === undefined || eventName === '') {
+        return done('trusted', 'local run — no CI event context');
+    }
+    const payload = await readEventPayload(env, readEventFile);
+    if (eventName === 'pull_request' || eventName === 'pull_request_target') {
+        const fork = payload?.pull_request?.head?.repo?.fork;
+        if (fork === true)
+            return done('untrusted', 'fork PR head checkout');
+        if (fork === false)
+            return done('trusted', 'same-repo PR checkout');
+        // Payload absent/unreadable (or head.repo null — a deleted source fork
+        // fails closed the same way evidence/ci.ts's isFork does): try the API.
+        const meta = await fetchMetaFor(opts);
+        if (meta !== undefined) {
+            return meta.isFork
+                ? done('untrusted', 'fork PR (metadata)')
+                : done('trusted', 'same-repo PR (metadata)');
+        }
+        return done('untrusted', 'PR fork status unavailable — failing closed');
+    }
+    if (eventName === 'issue_comment') {
+        const pr = payload?.issue?.pull_request !== undefined && typeof payload.issue?.number === 'number'
+            ? String(payload.issue.number)
+            : undefined;
+        const meta = await fetchMetaFor(opts, pr);
+        if (meta !== undefined) {
+            return meta.isFork
+                ? done('untrusted', 'issue_comment on fork PR', pr)
+                : done('trusted', 'issue_comment on same-repo PR', pr);
+        }
+        return done('untrusted', 'issue_comment PR metadata unavailable — failing closed', pr);
+    }
+    return done('untrusted', `unlisted CI event "${eventName}" — failing closed`);
+}

package/dist/vision/cost.d.ts

@@ -1,4 +1,4 @@
-export type CallKind = 'ground' | 'heal' | 'assert' | 'code';
+export type CallKind = 'ground' | 'heal' | 'assert' | 'code' | 'decide';
 export interface CallCost {
     model: string;
     provider: string;
@@ -30,6 +30,19 @@ export interface OpenRouterResponse {
     usage: Usage;
 }
 export declare function makeCallCost(response: OpenRouterResponse, kind: CallKind): CallCost;
+/** Decisions API (`/api/alpha/decisions`) usage shape — no `cost_details`. */
+export interface DecisionsResponse {
+    id?: string;
+    model?: string;
+    provider?: ProviderValue;
+    answers?: Record<string, unknown>;
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+        cost?: number;
+    };
+}
+export declare function makeDecisionsCallCost(response: DecisionsResponse, kind: CallKind): CallCost;
 export declare function extractUsageCost(response: OpenRouterResponse): {
     costUsd: number;
     upstreamCostUsd: number;

package/dist/vision/cost.js

@@ -8,6 +8,19 @@ export function makeCallCost(response, kind) {
         kind,
     };
 }
+export function makeDecisionsCallCost(response, kind) {
+    const providerName = typeof response.provider === 'string' ? response.provider : response.provider?.name ?? 'unknown';
+    const usage = response.usage ?? {};
+    // Coerce — a string cost would throw downstream at toFixed and discard
+    // a valid decision; a string token count would concatenate.
+    return {
+        model: response.model ?? 'unknown',
+        provider: providerName,
+        tokens: (Number(usage.input_tokens) || 0) + (Number(usage.output_tokens) || 0),
+        costUsd: Number(usage.cost) || 0,
+        kind,
+    };
+}
 export function extractUsageCost(response) {
     return {
         costUsd: response.usage.cost,

package/dist/vision/decisions.d.ts

@@ -0,0 +1,95 @@
+import type { CallCost, ProviderValue } from './cost.js';
+/**
+ * Pinned Jev slug — the alias `~typesafe/jev-latest` drifts silently and
+ * adjudication thresholds are calibrated to a version. The alias stays
+ * usable via `config.decisionModel` for experimentation.
+ */
+export declare const JEV_DEFAULT_MODEL = "typesafe/jev-1.13-20260917";
+export type DecisionErrorKind = 'auth' | 'validation' | 'rate_limited' | 'overloaded' | 'server_error' | 'timeout' | 'unexpected';
+export declare class DecisionError extends Error {
+    readonly kind: DecisionErrorKind;
+    readonly retryable: boolean;
+    constructor(kind: DecisionErrorKind, message: string, retryable: boolean);
+}
+export interface NoulQuestion {
+    type: 'noul';
+    instructions: string;
+    /** Optional yes/no clarifications, sent verbatim. */
+    true?: string;
+    false?: string;
+}
+export interface ChoiceQuestion {
+    type: 'choice';
+    instructions: string;
+    /** Option ID -> description. Up to 255 options. */
+    criteria: Record<string, string>;
+}
+export interface ScoreQuestion {
+    type: 'score';
+    instructions: string;
+    /** 2-10 ordered rubric levels, low to high. */
+    criteria: string[];
+}
+export type DecisionQuestion = NoulQuestion | ChoiceQuestion | ScoreQuestion;
+export interface NoulAnswer {
+    noul: number;
+}
+export interface ChoiceAnswer {
+    choice: string;
+    probabilities?: Record<string, number>;
+    confidence?: number;
+}
+export interface ScoreAnswer {
+    score: number;
+    legend?: string[];
+    probabilities?: number[];
+    confidence?: number;
+}
+export type DecisionAnswer = NoulAnswer | ChoiceAnswer | ScoreAnswer;
+/** Type guards over the answer union — one `in` check per lane otherwise. */
+export declare const isNoulAnswer: (a: DecisionAnswer) => a is NoulAnswer;
+export declare const isChoiceAnswer: (a: DecisionAnswer) => a is ChoiceAnswer;
+export declare const isScoreAnswer: (a: DecisionAnswer) => a is ScoreAnswer;
+/** Short error label for lane debug lines: DecisionError kind, else message. */
+export declare function describeDecisionError(e: unknown): string;
+/** Shared per-call batch cap for the Jev lanes (secrets, findings, triage). */
+export declare const MAX_CANDIDATES = 50;
+export interface DecisionClientOptions {
+    apiKey: string;
+    fetch?: typeof fetch;
+    trace?: Record<string, string>;
+    /** Request timeout in ms. Default 15_000. */
+    timeoutMs?: number;
+    onCall?: (call: {
+        id: string;
+        model: string;
+        provider: string;
+        kind: 'decide';
+        costUsd: number;
+        tokens: number;
+        trace?: Record<string, string>;
+    }) => void;
+}
+export declare class DecisionClient {
+    private _apiKey;
+    private _fetch;
+    private _trace;
+    private _timeoutMs;
+    private _onCall;
+    constructor(opts: DecisionClientOptions);
+    /**
+     * Ask typed questions about `state`. Throws a typed `DecisionError` on
+     * any failure — callers catch and degrade to unadjudicated; a decision
+     * failure must never suppress findings (KTD6).
+     */
+    decide(opts: {
+        model?: string;
+        state: unknown;
+        questions: Record<string, DecisionQuestion>;
+    }): Promise<{
+        answers: Record<string, DecisionAnswer>;
+        cost: CallCost;
+        model: string;
+    }>;
+}
+export type { ProviderValue };

package/dist/vision/decisions.js

@@ -0,0 +1,232 @@
+import { debug } from '../debug.js';
+import { makeDecisionsCallCost } from './cost.js';
+/**
+ * Pinned Jev slug — the alias `~typesafe/jev-latest` drifts silently and
+ * adjudication thresholds are calibrated to a version. The alias stays
+ * usable via `config.decisionModel` for experimentation.
+ */
+export const JEV_DEFAULT_MODEL = 'typesafe/jev-1.13-20260917';
+const DECISIONS_URL = 'https://openrouter.ai/api/alpha/decisions';
+const TIMEOUT_MS = 15_000;
+const MAX_CHOICE_OPTIONS = 255;
+const MIN_SCORE_LEVELS = 2;
+const MAX_SCORE_LEVELS = 10;
+export class DecisionError extends Error {
+    kind;
+    retryable;
+    constructor(kind, message, retryable) {
+        super(message);
+        this.kind = kind;
+        this.retryable = retryable;
+        this.name = 'DecisionError';
+    }
+}
+/** Type guards over the answer union — one `in` check per lane otherwise. */
+export const isNoulAnswer = (a) => 'noul' in a;
+export const isChoiceAnswer = (a) => 'choice' in a;
+export const isScoreAnswer = (a) => 'score' in a;
+/** Short error label for lane debug lines: DecisionError kind, else message. */
+export function describeDecisionError(e) {
+    return e instanceof DecisionError ? e.kind : e.message;
+}
+/** Shared per-call batch cap for the Jev lanes (secrets, findings, triage). */
+export const MAX_CANDIDATES = 50;
+function isRecord(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+function validateQuestions(questions) {
+    const entries = Object.entries(questions);
+    if (entries.length === 0) {
+        throw new DecisionError('validation', 'decide: at least one question is required', false);
+    }
+    for (const [id, q] of entries) {
+        if (id === '') {
+            throw new DecisionError('validation', 'decide: question IDs must be non-empty', false);
+        }
+        if (q.type === 'choice') {
+            const n = Object.keys(q.criteria).length;
+            if (n === 0 || n > MAX_CHOICE_OPTIONS) {
+                throw new DecisionError('validation', `decide: choice question "${id}" needs 1-${MAX_CHOICE_OPTIONS} options, got ${n}`, false);
+            }
+        }
+        else if (q.type === 'score') {
+            const n = q.criteria.length;
+            if (n < MIN_SCORE_LEVELS || n > MAX_SCORE_LEVELS) {
+                throw new DecisionError('validation', `decide: score question "${id}" needs ${MIN_SCORE_LEVELS}-${MAX_SCORE_LEVELS} levels, got ${n}`, false);
+            }
+        }
+        else if (q.type !== 'noul') {
+            throw new DecisionError('validation', `decide: question "${id}" has unknown type`, false);
+        }
+    }
+}
+function validateAnswers(questions, answers) {
+    // Per-question salvage: a missing or malformed answer is skipped, not
+    // fatal — every caller degrades open per item (undefined answer →
+    // unadjudicated), so one bad f_i must not void the whole batch.
+    const out = {};
+    for (const [id, q] of Object.entries(questions)) {
+        const a = answers[id];
+        if (!isRecord(a))
+            continue;
+        if (q.type === 'noul') {
+            if (typeof a.noul === 'number' && Number.isFinite(a.noul) && a.noul >= 0 && a.noul <= 1) {
+                out[id] = { noul: a.noul };
+            }
+        }
+        else if (q.type === 'choice') {
+            const confOk = a.confidence === undefined ||
+                (typeof a.confidence === 'number' &&
+                    Number.isFinite(a.confidence) &&
+                    a.confidence >= 0 &&
+                    a.confidence <= 1);
+            if (typeof a.choice === 'string' && Object.hasOwn(q.criteria, a.choice) && confOk) {
+                out[id] = a;
+            }
+        }
+        else {
+            // Score answers must land inside the rubric — an out-of-range
+            // score could otherwise satisfy a low-risk routing predicate.
+            const maxScore = Array.isArray(q.criteria) ? q.criteria.length : 0;
+            if (typeof a.score === 'number' &&
+                Number.isFinite(a.score) &&
+                a.score >= 1 &&
+                a.score <= maxScore) {
+                out[id] = a;
+            }
+        }
+    }
+    return out;
+}
+function classifyStatus(status) {
+    if (status === 401 || status === 403) {
+        return new DecisionError('auth', `decide: HTTP ${status} — check OPENROUTER_API_KEY`, false);
+    }
+    if (status === 400 || status === 404 || status === 422) {
+        return new DecisionError('validation', `decide: HTTP ${status} — request rejected`, false);
+    }
+    if (status === 429) {
+        return new DecisionError('rate_limited', 'decide: HTTP 429 rate limited', true);
+    }
+    if (status === 503) {
+        return new DecisionError('overloaded', 'decide: HTTP 503 provider overloaded', true);
+    }
+    if (status >= 500) {
+        return new DecisionError('server_error', `decide: HTTP ${status}`, true);
+    }
+    return new DecisionError('unexpected', `decide: HTTP ${status}`, false);
+}
+function retryDelayMs(res) {
+    const retryAfterMs = res.headers.get('retry-after-ms');
+    if (retryAfterMs !== null) {
+        const parsed = Number(retryAfterMs);
+        if (Number.isFinite(parsed) && parsed >= 0)
+            return Math.min(parsed, 30_000);
+    }
+    const retryAfter = res.headers.get('retry-after');
+    if (retryAfter !== null) {
+        const parsed = Number(retryAfter);
+        if (Number.isFinite(parsed) && parsed >= 0)
+            return Math.min(parsed * 1000, 30_000);
+    }
+    return 1_000;
+}
+export class DecisionClient {
+    _apiKey;
+    _fetch;
+    _trace;
+    _timeoutMs;
+    _onCall;
+    constructor(opts) {
+        if (!opts.apiKey) {
+            throw new Error('OPENROUTER_API_KEY is required: pass a non-empty apiKey at construction');
+        }
+        this._apiKey = opts.apiKey;
+        this._fetch = opts.fetch ?? globalThis.fetch;
+        this._trace = opts.trace;
+        this._timeoutMs = opts.timeoutMs ?? TIMEOUT_MS;
+        this._onCall = opts.onCall;
+    }
+    /**
+     * Ask typed questions about `state`. Throws a typed `DecisionError` on
+     * any failure — callers catch and degrade to unadjudicated; a decision
+     * failure must never suppress findings (KTD6).
+     */
+    async decide(opts) {
+        const model = opts.model ?? JEV_DEFAULT_MODEL;
+        validateQuestions(opts.questions);
+        const body = {
+            model,
+            state: opts.state,
+            questions: opts.questions,
+        };
+        if (this._trace)
+            body.trace = this._trace;
+        let lastErr;
+        for (let attempt = 0; attempt < 2; attempt++) {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), this._timeoutMs);
+            try {
+                const res = await this._fetch(DECISIONS_URL, {
+                    method: 'POST',
+                    signal: controller.signal,
+                    headers: {
+                        Authorization: `Bearer ${this._apiKey}`,
+                        'Content-Type': 'application/json',
+                        'X-Title': 'argus-reviewer',
+                    },
+                    body: JSON.stringify(body),
+                });
+                if (!res.ok) {
+                    const err = classifyStatus(res.status);
+                    if (err.retryable && attempt === 0) {
+                        const delay = retryDelayMs(res);
+                        debug('decisions', `${err.kind} — retrying in ${delay}ms`);
+                        await new Promise((r) => setTimeout(r, delay));
+                        continue;
+                    }
+                    throw err;
+                }
+                const data = (await res.json());
+                if (!isRecord(data.answers)) {
+                    throw new DecisionError('unexpected', 'decide: response has no answers object', false);
+                }
+                const answers = validateAnswers(opts.questions, data.answers);
+                const cost = makeDecisionsCallCost(data, 'decide');
+                debug('decisions', `model=${cost.model} provider=${cost.provider} tokens=${cost.tokens} costUsd=${cost.costUsd.toFixed(6)}`);
+                this._onCall?.({
+                    id: data.id ?? '',
+                    model: cost.model,
+                    provider: cost.provider,
+                    kind: 'decide',
+                    costUsd: cost.costUsd,
+                    tokens: cost.tokens,
+                    ...(this._trace ? { trace: this._trace } : {}),
+                });
+                return { answers, cost, model: cost.model };
+            }
+            catch (e) {
+                if (e instanceof DecisionError) {
+                    lastErr = e;
+                    // Validation/unexpected errors never retry — they retry identically.
+                    break;
+                }
+                const err = e;
+                if (err.name === 'AbortError') {
+                    lastErr = new DecisionError('timeout', `decide: timed out after ${this._timeoutMs}ms`, true);
+                    if (attempt === 0) {
+                        debug('decisions', 'timeout — retrying once');
+                        continue;
+                    }
+                    break;
+                }
+                lastErr = new DecisionError('unexpected', `decide: ${err.message}`, false);
+                break;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        }
+        throw lastErr ?? new DecisionError('unexpected', 'decide: failed', false);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "argus-reviewer-e2e",
-  "version": "0.1.3",
+  "version": "0.2.0",
   "description": "argus-reviewer — open-source vision-model E2E testing harness. Record, replay, heal, and assert with any OpenRouter model under a hard budget cap.",
   "type": "module",
   "license": "MIT",
@@ -42,7 +42,8 @@
     "prepare": "npm run build && chmod +x dist/cli.js",
     "eval": "node evals/run.mjs",
     "watch": "node scripts/watch.mjs",
-    "app": "electron electron"
+    "app": "electron electron",
+    "demo": "node scripts/demo.mjs"
   },
   "devDependencies": {
     "@eslint/js": "^9.0.0",