argus-reviewer-e2e 0.1.2 → 0.1.3

package/dist/probe/author.js

@@ -0,0 +1,149 @@
+import { posix } from 'node:path';
+import ts from 'typescript';
+/**
+ * Probe authoring (KTD7): one bounded model call per `not_exercised`
+ * finding produces a single test file asserting the *correct* behavior —
+ * so the defect's presence fails the test on head while the base checkout
+ * passes. The file is written on the HOST (inside the ro workspace mount),
+ * so `filename` and `content` are validated hard: the model's input includes
+ * PR-controlled file contents, and a prompt-injected traversal or secret
+ * read would escape the sandbox's whole purpose.
+ */
+export const PROBE_SCHEMA = {
+    name: 'argus-probe',
+    strict: true,
+    schema: {
+        type: 'object',
+        properties: {
+            filename: {
+                type: 'string',
+                description: 'Bare test filename matching the repo convention, e.g. probe-xyz.test.ts',
+            },
+            content: { type: 'string', description: 'Complete test file contents' },
+            reasoning: { type: 'string' },
+        },
+        required: ['filename', 'content', 'reasoning'],
+        additionalProperties: false,
+    },
+};
+/**
+ * Basename-only, forced test extension — no separators, no `..`. The write
+ * happens on the host, so this is the traversal boundary.
+ */
+export const PROBE_FILENAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]*\.test\.[jt]sx?$/;
+/** Probe files are bounded — a runaway generation is rejected, not truncated. */
+export const PROBE_CONTENT_CAP = 32 * 1024;
+/** Reads of secret-looking env vars are forbidden — defense in depth on the stripped container env. */
+const SECRET_ENV_RE = /process\.env\s*(?:\.|\[)\s*['"]?\w*(KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)/i;
+/**
+ * Import specifiers via the TypeScript scanner (same seam as the index) —
+ * a regex misses bare side-effect imports like `import '/abs/x'`. Returns
+ * undefined when the scanner itself fails — fail closed, never silently
+ * skip the import check.
+ */
+function importSpecifiers(content) {
+    try {
+        const info = ts.preProcessFile(content, true, true);
+        return [...info.importedFiles, ...info.referencedFiles].map((f) => f.fileName);
+    }
+    catch {
+        return undefined;
+    }
+}
+export function parseProbe(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return { ok: false, reason: 'unparseable model response' };
+    }
+    if (typeof parsed.filename !== 'string' || !PROBE_FILENAME_RE.test(parsed.filename)) {
+        return { ok: false, reason: `unsafe filename: ${String(parsed.filename)}` };
+    }
+    if (typeof parsed.content !== 'string' || parsed.content.trim() === '') {
+        return { ok: false, reason: 'missing or empty content' };
+    }
+    if (parsed.content.length > PROBE_CONTENT_CAP) {
+        return { ok: false, reason: `content exceeds ${PROBE_CONTENT_CAP} bytes` };
+    }
+    if (SECRET_ENV_RE.test(parsed.content)) {
+        return { ok: false, reason: 'probe reads secret-looking env vars' };
+    }
+    const imports = importSpecifiers(parsed.content);
+    if (imports === undefined) {
+        return { ok: false, reason: 'probe content failed import scanning' };
+    }
+    for (const spec of imports) {
+        // Reject anything that isn't a bare specifier or plain relative path:
+        // absolute paths, drive letters, UNC, and scheme-prefixed specifiers
+        // (`file:///etc/passwd` is a host read, `data:` smuggles a module).
+        // `node:` builtins stay allowed — node-test probes need `node:test`.
+        if (spec.startsWith('/') ||
+            /^[a-zA-Z]:/.test(spec) ||
+            spec.startsWith('\\\\') ||
+            (/^[a-z][a-z0-9+.-]*:/i.test(spec) && !spec.startsWith('node:'))) {
+            return { ok: false, reason: `unsafe import: ${spec}` };
+        }
+    }
+    return {
+        ok: true,
+        probe: {
+            filename: parsed.filename,
+            content: parsed.content,
+            reasoning: typeof parsed.reasoning === 'string' ? parsed.reasoning : '',
+            imports,
+        },
+    };
+}
+/**
+ * Verify every relative import in a probe resolves inside the repo, given
+ * the probe's repo-relative write path. `../../etc/passwd` from a shallow
+ * dir escapes the checkout — reject.
+ */
+export function probeImportsSafe(probe, relProbePath) {
+    // Repo-relative paths are always `/`-separated — compute in posix so a
+    // Windows host can't turn `../x` into `..\x` and slip the `../` check.
+    const dir = posix.dirname(relProbePath);
+    return probe.imports.every((spec) => {
+        if (!spec.startsWith('.'))
+            return true; // bare package specifier — resolved from node_modules
+        const resolved = posix.normalize(posix.join(dir, spec));
+        return resolved !== '..' && !resolved.startsWith('../') && !posix.isAbsolute(resolved);
+    });
+}
+export function buildProbeMessages(target, fileContents, exemplarTest, harness) {
+    const exemplar = exemplarTest === undefined
+        ? '(no existing test file was available as an exemplar — follow standard idiom)'
+        : `Exemplar test file (${exemplarTest.path}) — match its imports, naming, and assertion style:\n\n${exemplarTest.content}`;
+    const implicated = fileContents === undefined
+        ? `(file contents unavailable; the finding references ${target.file ?? 'unknown'})`
+        : `Contents of ${target.file}:\n\n${fileContents}`;
+    return [
+        {
+            role: 'system',
+            content: [
+                {
+                    type: 'text',
+                    text: `You author a single failing test — a "probe" — that reproduces a suspected defect ` +
+                        `found by code review. The probe runs under ${harness.kind} in an offline sandbox. ` +
+                        `Write a self-contained ${harness.kind} test that asserts the CORRECT behavior: if the ` +
+                        `defect is real, the test fails; if the code is fine, it passes. Import only repo ` +
+                        `modules via relative paths and packages the repo already uses. No network, no ` +
+                        `filesystem outside the repo, no timers, no process.env reads. filename must be a ` +
+                        `bare filename ending in .test.ts/.test.js matching the exemplar's convention.`,
+                },
+            ],
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `Finding (${target.severity}) in ${target.file ?? 'unknown'}${target.line ? `:${target.line}` : ''}:\n` +
+                        `${target.message}\n\n${implicated}\n\n${exemplar}`,
+                },
+            ],
+        },
+    ];
+}

package/dist/probe/harness.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Test-harness detection + result classification for the B.2 probe lane
+ * (KTD3). v1 is deliberately Node-only: vitest, jest, and `node --test`.
+ *
+ * Two implementation details carry the weight:
+ *  - Positional file args are *filters* intersected with the consumer's
+ *    configured include/testMatch/roots, so probes must live where the
+ *    runner already looks (the queue writes them beside an exemplar test).
+ *  - Invocations bypass `npx` — as `nobody` on a read-only root there is no
+ *    writable HOME, so the installed binary is invoked directly and each
+ *    runner's cache is redirected into the /tmp tmpfs.
+ */
+export type ProbeOutcome = 'failed-test' | 'load-error' | 'not-collected' | 'clean';
+export interface Harness {
+    kind: 'vitest' | 'jest' | 'node-test';
+    /** In-container argv running exactly one probe file (path relative to /work). */
+    runCmd(probeRelPath: string): string[];
+    /** Map captured output to the report vocabulary. */
+    classify(res: {
+        exitCode: number;
+        stdout: string;
+        stderr: string;
+    }): ProbeOutcome;
+}
+/**
+ * Detect the consumer's test harness from package.json. Returns undefined
+ * when no supported harness exists — the probe lane degrades to a detail
+ * note, not a failure.
+ */
+export declare function detectHarness(cwd: string): Promise<Harness | undefined>;

package/dist/probe/harness.js

@@ -0,0 +1,99 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+/**
+ * Load-failure signatures are checked BEFORE test-failure summaries — a
+ * suite that can't import (vitest counts it as "Test Files 1 failed", TAP
+ * emits "not ok") must never classify as failed-test, and the exit code
+ * binds the result: probe output is attacker-printable, so a forged
+ * "N failed" line on a clean exit still classifies clean.
+ */
+const LOAD_ERROR_RE = /Cannot find module|ERR_UNKNOWN_FILE_EXTENSION|ERR_MODULE_NOT_FOUND|Failed to load|SyntaxError/;
+const vitestHarness = {
+    kind: 'vitest',
+    runCmd: (file) => ['node', 'node_modules/vitest/vitest.mjs', 'run', file],
+    classify: ({ exitCode, stdout, stderr }) => {
+        const out = `${stdout}\n${stderr}`;
+        if (exitCode === 0)
+            return 'clean';
+        if (LOAD_ERROR_RE.test(out))
+            return 'load-error';
+        if (/no test files? found/i.test(out))
+            return 'not-collected';
+        if (/test files?\s+\d+ failed|tests\s+\d+ failed/i.test(out))
+            return 'failed-test';
+        return 'load-error';
+    },
+};
+const jestHarness = {
+    kind: 'jest',
+    runCmd: (file) => [
+        'node',
+        'node_modules/jest-cli/bin/jest.js',
+        '--runTestsByPath',
+        '--cacheDirectory=/tmp/jest',
+        file,
+    ],
+    classify: ({ exitCode, stdout, stderr }) => {
+        const out = `${stdout}\n${stderr}`;
+        if (exitCode === 0)
+            return 'clean';
+        if (LOAD_ERROR_RE.test(out))
+            return 'load-error';
+        if (/no tests found/i.test(out))
+            return 'not-collected';
+        if (/tests:\s+\d+ failed/i.test(out))
+            return 'failed-test';
+        return 'load-error';
+    },
+};
+/**
+ * `node --test` in a TS repo only works because `scripts.test` carries the
+ * loader flags (`--import tsx`, `--experimental-strip-types`, `--loader`) —
+ * capture them verbatim or the authored .ts probe dies with
+ * ERR_UNKNOWN_FILE_EXTENSION. Both `--import tsx` and `--import=tsx` forms.
+ */
+const NODE_TEST_FLAG_RE = /(?:--import|--loader)(?:[\s=])\S+|--experimental-(?:strip|transform)-types/g;
+function nodeTestHarness(flags) {
+    return {
+        kind: 'node-test',
+        // --test-reporter=tap pins the format `classify` parses — Node ≥22
+        // defaults to the spec reporter (`✖`, not TAP `not ok`) when stdout
+        // is a TTY, and real failures would fall through to load-error.
+        runCmd: (file) => ['node', ...flags, '--test', '--test-reporter=tap', file],
+        classify: ({ exitCode, stdout, stderr }) => {
+            const out = `${stdout}\n${stderr}`;
+            if (exitCode === 0)
+                return 'clean';
+            if (LOAD_ERROR_RE.test(out))
+                return 'load-error';
+            if (/^not ok/im.test(out))
+                return 'failed-test';
+            return 'load-error';
+        },
+    };
+}
+/**
+ * Detect the consumer's test harness from package.json. Returns undefined
+ * when no supported harness exists — the probe lane degrades to a detail
+ * note, not a failure.
+ */
+export async function detectHarness(cwd) {
+    let pkg;
+    try {
+        pkg = JSON.parse(await readFile(join(cwd, 'package.json'), 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+    const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+    const testScript = pkg.scripts?.test ?? '';
+    if (deps['vitest'] !== undefined || /\bvitest\b/.test(testScript))
+        return vitestHarness;
+    if (deps['jest'] !== undefined || /\bjest\b/.test(testScript))
+        return jestHarness;
+    if (/\bnode\b.*--test|node:test/.test(testScript)) {
+        const flags = testScript.match(NODE_TEST_FLAG_RE)?.flatMap((f) => f.split(/[\s=]+/)) ?? [];
+        return nodeTestHarness(flags);
+    }
+    return undefined;
+}

package/dist/probe/queue.d.ts

@@ -0,0 +1,102 @@
+import { type ProviderRules, type Sandbox } from '../config.js';
+import { type ExecFn } from '../detect.js';
+import type { PrMeta } from '../evidence/ci.js';
+import { type Evidence } from '../evidence/link.js';
+import type { RepoIndex } from '../index/scan.js';
+import type { VisionClient } from '../engine/loop.js';
+import type { CallCost } from '../vision/cost.js';
+import type { Ledger } from '../vision/ledger.js';
+import { type ProbeOutcome } from './harness.js';
+/**
+ * The B.2 probe lane (U4). Called after `linkFindings` inside `code-review`.
+ * Selects `not_exercised` blocking-severity findings, authors one probe each,
+ * runs it in the sandbox against head AND a merge-base worktree, and upgrades
+ * evidence to `reproduced` only on fail-head ∧ clean-base (KTD6). Everything
+ * else leaves the finding untouched — additive evidence only, never a new
+ * failure surface (KTD8): every error path degrades to a probe record or a
+ * debug note, and the queue never changes verdict, ok, or exit code.
+ *
+ * Trust note: for `pull_request` events the config file ships in the PR's
+ * own tree, so for FORK PRs the lane ignores config-supplied
+ * image/limits/allowForks and runs on `DEFAULT_SANDBOX` — forks approve via
+ * the argus-probe label or a trusted author association, never via config.
+ */
+/** Finding shape shared by the review report, probe targets, and this lane. */
+export interface LinkedFinding {
+    file?: string | undefined;
+    line?: number | undefined;
+    severity?: string | undefined;
+    message?: string | undefined;
+    evidence: Evidence;
+}
+export type ProbeReportOutcome = 'reproduced' | 'clean' | 'load-error' | 'not-collected' | 'error';
+export interface ProbeRecord {
+    /** Probe filename (basename — it is written beside the exemplar test). */
+    file: string;
+    findingFile: string | undefined;
+    findingLine: number | undefined;
+    outcome: ProbeReportOutcome;
+    /** Raw harness outcomes per checkout, when the run happened ('error' = infra/timeout). */
+    headOutcome?: ProbeOutcome | 'error' | undefined;
+    baseOutcome?: ProbeOutcome | 'error' | undefined;
+    durationMs: number;
+    costUsd: number;
+    tokens: number;
+    detail: string;
+    /** Capped, control-char-stripped stdout+stderr for audit. */
+    output?: string | undefined;
+}
+export interface ProbeLaneResult {
+    records: ProbeRecord[];
+    /** Present when the lane was enabled but skipped before running probes. */
+    skipReason?: string | undefined;
+}
+export interface ProbeLaneOptions {
+    /** PR checkout root (the head tree). */
+    cwd: string;
+    reportDir: string;
+    /** Caller resolves `sandbox.enabled || ARGUS_SANDBOX=1` into this object. */
+    sandbox: Sandbox;
+    meta: PrMeta | undefined;
+    token: string | undefined;
+    client: VisionClient;
+    model: string;
+    provider: ProviderRules | undefined;
+    /** Shared codeReviewBudgetUsd ledger — authoring calls record on it. */
+    ledger: Ledger;
+    /** codeReviewBudgetUsd — authoring stops once spend reaches it. */
+    budgetUsd: number | undefined;
+    /** Configured blocking severities — the queue only admits those. */
+    severityGates: string[];
+    index: RepoIndex | undefined;
+    /** Report sink — authored probe CallCosts are pushed here for the report. */
+    calls?: CallCost[] | undefined;
+    exec?: ExecFn | undefined;
+    log?: ((line: string) => void) | undefined;
+}
+/** Pure selection: not_exercised findings at blocking severities, capped. */
+export declare function selectProbeTargets(findings: LinkedFinding[], severityGates: string[], maxProbes: number): LinkedFinding[];
+/**
+ * Repo-relative path gate for anything model- or index-derived that is read
+ * or written on the HOST: no absolute paths, no `..` escapes, no backslashes.
+ * The write side is already basename-bound (PROBE_FILENAME_RE); this is the
+ * read-side and exemplar-dir boundary — a prompt-injected `../../.env`
+ * finding.file or a crafted index entry must never reach fs calls.
+ */
+export declare function isSafeRepoPath(p: string): boolean;
+/**
+ * Nearest existing test file to the finding's file — same directory first,
+ * then same top-level segment, then any test file. The exemplar sets the
+ * probe's write location so the consumer's own include/roots cover it.
+ * Index paths are filtered through isSafeRepoPath — a committed/crafted
+ * index could otherwise aim the host write outside the checkout.
+ */
+export declare function findExemplarTest(index: RepoIndex | undefined, findingFile: string): string | undefined;
+/**
+ * Run the probe lane. Mutates `findings` evidence in place for reproduced
+ * results and returns the per-probe audit records for code-review.json.
+ * Returns undefined only when the lane is disabled entirely; otherwise a
+ * result carrying records and (when it bowed out early) a skipReason the
+ * report surface can render.
+ */
+export declare function runProbeLane(findings: LinkedFinding[], o: ProbeLaneOptions): Promise<ProbeLaneResult | undefined>;