@kylecheng3146/agent-ops 0.1.15 → 0.1.17

package/dist/runtime/src/review/execute.js

@@ -1,20 +1,26 @@
-import { mkdtemp, realpath, rm } from "node:fs/promises";
+import { chmod, copyFile, lstat, mkdir, mkdtemp, realpath, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { dirname, join } from "node:path";
 import { runVerificationCommand } from "../verify/spawn.js";
+import { redactSecrets } from "../security/redact.js";
 import { extractReviewObject } from "./extract.js";
 import { buildTargetInvocation } from "./invocation.js";
 import { reviewReportResults, reviewReportStatus, validateReviewReport } from "./report.js";
 import { detectHostTarget, orderChain } from "./roles.js";
-import { buildReviewPrompt } from "./runner.js";
+import { buildAdversarialPrompt, buildReviewPrompt } from "./runner.js";
 /**
- * Deliberately below the five-minute `spawn.ts` default: a timeout advances the
- * chain, so the worst case is targets x timeout.
+ * Full repository reviews need far more headroom than the lightweight auth
+ * probe. Five minutes was not enough: reviewing a real working tree, codex and
+ * claude both exceeded it on this repository, and a timeout costs the whole
+ * review while looking like an unavailable target.
  */
-export const DEFAULT_REVIEW_TIMEOUT_MS = 120_000;
+export const DEFAULT_REVIEW_TIMEOUT_MS = 900_000;
+// USER is load-bearing, not cosmetic: a credential store keyed by account name
+// — the macOS keychain claude reads — cannot be opened without it, and its
+// absence surfaces as "Not logged in" on an install that is logged in.
 const EXECUTION_ENV = [
     "PATH", "PATHEXT", "SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC",
-    "LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP"
+    "LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP", "USER"
 ];
 const AUTH_ENV = {
     claude: ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"],
@@ -29,23 +35,45 @@ export function isolatedReviewEnvironment(target, directory, source) {
             env[key] = value;
         }
     }
-    env.HOME = directory;
-    env.USERPROFILE = directory;
-    env.XDG_CONFIG_HOME = join(directory, "config");
-    env.XDG_CACHE_HOME = join(directory, "cache");
+    // claude and agy read their credentials out of the invoking user's home, so
+    // replacing it does not isolate them — it only makes them report "not logged
+    // in" on an install that is logged in. claude is isolated by its own flags
+    // instead: `--safe-mode` disables CLAUDE.md, skills, plugins, hooks, MCP
+    // servers and custom agents while explicitly keeping auth. codex keeps a
+    // replaced home because CODEX_HOME carries its credentials separately, so
+    // the isolation costs it nothing. agy uses its native sandbox and plan mode,
+    // but keeps its home because that is where its OAuth session lives.
+    if (target === "agy" || target === "claude") {
+        env.HOME = source.HOME ?? directory;
+        env.USERPROFILE = source.USERPROFILE ?? env.HOME;
+        env.XDG_CONFIG_HOME = source.XDG_CONFIG_HOME ?? join(env.HOME, ".config");
+        env.XDG_CACHE_HOME = source.XDG_CACHE_HOME ?? join(env.HOME, ".cache");
+    }
+    else {
+        env.HOME = directory;
+        env.USERPROFILE = directory;
+        env.XDG_CONFIG_HOME = join(directory, "config");
+        env.XDG_CACHE_HOME = join(directory, "cache");
+    }
+    if (target === "codex") {
+        const codexHome = source.CODEX_HOME ??
+            (source.HOME === undefined ? undefined : join(source.HOME, ".codex"));
+        if (codexHome !== undefined) {
+            env.CODEX_HOME = codexHome;
+        }
+    }
     return env;
 }
-/** Codex and agy currently lack documented instruction/customization isolation. */
-export function hasRequiredReviewIsolation(target) {
-    return target === "claude";
-}
+/** Only eligible targets appear: an ineligible one never reaches this gate. */
 const REQUIRED_HELP_FLAGS = {
     claude: [
         "--add-dir", "--permission-mode", "--no-session-persistence",
         "--safe-mode", "--disable-slash-commands", "--json-schema"
     ],
-    codex: [],
-    agy: []
+    agy: ["--add-dir", "--sandbox", "--mode", "--json-schema", "--log-file"],
+    codex: [
+        "--cd", "--ephemeral", "--ignore-user-config", "--ignore-rules"
+    ]
 };
 /**
  * Failure classes that mean no review happened, so trying the next target is
@@ -56,128 +84,327 @@ const ADVANCING = new Set([
     "spawn-failed",
     "timeout"
 ]);
+const DIAGNOSTIC_MAX_CHARS = 200;
+/**
+ * The target's first line of complaint, redacted and clipped. It travels on the
+ * attempt record rather than only in a progress line, because progress is
+ * suppressed under `--json`: a machine consumer would otherwise be left with
+ * the bare authentication guess this exists to qualify. Never evidence.
+ */
+function firstComplaint(...streams) {
+    for (const stream of streams) {
+        const line = redactSecrets(stream)
+            .split(/\r?\n/u)
+            .map((value) => value.trim())
+            .find((value) => value.length > 0);
+        if (line !== undefined) {
+            return line.slice(0, DIAGNOSTIC_MAX_CHARS);
+        }
+    }
+    return undefined;
+}
+async function snapshotRepository(request, destination, options) {
+    const cloned = await runVerificationCommand({
+        id: `review-snapshot-${request.label}`,
+        command: "git",
+        args: ["clone", "--no-hardlinks", "--quiet", "--", request.repositoryRoot, destination],
+        cwd: dirname(destination),
+        required: true,
+        evidence: { kind: "exit-code" },
+        timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 60_000)
+    }, {
+        cwd: dirname(destination),
+        ...(options.runner === undefined ? {} : { runner: options.runner })
+    });
+    if (cloned.status !== "PASS") {
+        return firstComplaint(cloned.stderr, cloned.stdout) ??
+            `git clone failed (${cloned.failureClass})`;
+    }
+    for (const path of request.changedFiles ?? []) {
+        const source = join(request.repositoryRoot, path);
+        const target = join(destination, path);
+        try {
+            const stat = await lstat(source);
+            if (!stat.isFile()) {
+                return `changed path is not a regular file: ${path}`;
+            }
+            await mkdir(dirname(target), { recursive: true });
+            await copyFile(source, target);
+            await chmod(target, stat.mode & 0o777);
+        }
+        catch (error) {
+            if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+                await rm(target, { recursive: true, force: true });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return undefined;
+}
+/**
+ * One target's attempt at one prompt, in a throwaway home directory. Returns a
+ * validated report or the reason this target produced no usable verdict; the
+ * caller decides whether that reason is worth advancing past.
+ */
+async function attemptTarget(request, options) {
+    const { target } = request;
+    const skip = (reason, diagnostic, verb = "trying next target") => ({
+        kind: "skip",
+        reason,
+        diagnostic,
+        message: `${target}: ${reason} → ${verb} (${diagnostic})`
+    });
+    const attemptDirectory = await mkdtemp(join(tmpdir(), "agent-ops-review-"));
+    try {
+        const invocationRequest = {
+            target,
+            prompt: request.prompt,
+            repositoryRoot: request.repositoryRoot,
+            ...(target === "agy"
+                ? { logFile: join(attemptDirectory, "agy.log") }
+                : {}),
+            ...(options.model === undefined ? {} : { model: options.model }),
+            ...(options.effort === undefined ? {} : { effort: options.effort })
+        };
+        let invocation = buildTargetInvocation(invocationRequest);
+        let executionDirectory = attemptDirectory;
+        if (invocation === undefined) {
+            return skip("capability-unavailable", "no read-only mode is available for this target", "skipping");
+        }
+        const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
+        const capability = await runVerificationCommand({
+            id: `review-capability-${request.label}`,
+            command: invocation.command,
+            args: target === "codex" ? ["exec", "--help"] : ["--help"],
+            cwd: attemptDirectory,
+            required: true,
+            evidence: { kind: "exit-code" },
+            timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
+        }, {
+            cwd: attemptDirectory,
+            ...(options.runner === undefined ? {} : { runner: options.runner }),
+            env: environment,
+            replaceEnv: true
+        });
+        const help = `${capability.stdout}\n${capability.stderr}`;
+        const missingFlags = (REQUIRED_HELP_FLAGS[target] ?? []).filter((flag) => !help.includes(flag));
+        if (capability.status !== "PASS" ||
+            capability.stdoutTruncated ||
+            capability.stderrTruncated ||
+            missingFlags.length > 0) {
+            // This gate is the one a renamed upstream flag trips, so it names the
+            // flags it could not find. Without them the skip is indistinguishable
+            // from an uninstalled CLI, in the human line and in the attempt record.
+            return skip("capability-unavailable", missingFlags.length > 0
+                ? `help output is missing ${missingFlags.join(", ")}`
+                : capability.stdoutTruncated || capability.stderrTruncated
+                    ? "help output exceeded the capture limit"
+                    : firstComplaint(capability.stderr, capability.stdout) ??
+                        `help probe failed (${capability.failureClass})`, "skipping");
+        }
+        if (target === "agy") {
+            const snapshotRoot = join(attemptDirectory, "repository");
+            const snapshotError = await snapshotRepository(request, snapshotRoot, options);
+            if (snapshotError !== undefined) {
+                return skip("capability-unavailable", snapshotError, "skipping");
+            }
+            invocation = buildTargetInvocation({
+                ...invocationRequest,
+                prompt: [
+                    `Repository root: ${snapshotRoot}`,
+                    "Run every repository-relative inspection in that directory.",
+                    "For terminal commands, use only git status, git diff, git log, or git show; " +
+                        "read specific files with file-reading tools instead of ls, find, cat, or rg.",
+                    request.prompt
+                ].join("\n"),
+                repositoryRoot: snapshotRoot
+            });
+            executionDirectory = snapshotRoot;
+        }
+        if (invocation === undefined) {
+            return skip("capability-unavailable", "review invocation disappeared", "skipping");
+        }
+        const spawned = await runVerificationCommand({
+            id: `review-${request.label}`,
+            command: invocation.command,
+            args: [...invocation.args],
+            cwd: executionDirectory,
+            required: true,
+            evidence: { kind: "exit-code" },
+            timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
+        }, {
+            cwd: executionDirectory,
+            ...(options.runner === undefined ? {} : { runner: options.runner }),
+            ...(options.outputLimitBytes === undefined
+                ? {}
+                : { outputLimitBytes: options.outputLimitBytes }),
+            stdin: invocation.stdin,
+            env: environment,
+            replaceEnv: true
+        });
+        if (ADVANCING.has(spawned.failureClass)) {
+            return {
+                ...skip("missing-cli", `the process did not complete: ${spawned.failureClass}`),
+                ...(spawned.failureClass === undefined
+                    ? {}
+                    : { attemptReason: spawned.failureClass })
+            };
+        }
+        if (spawned.stdoutTruncated ||
+            (spawned.stderrTruncated && target !== "codex")) {
+            return skip("output-too-large", `${spawned.stdoutTruncated ? "stdout" : "stderr"} exceeded the capture limit`);
+        }
+        if (spawned.failureClass === "nonzero-exit") {
+            // A rejected call is usually missing authentication, but a stale flag
+            // shape exits non-zero too, and reporting only "login-required" sends
+            // the reader to `doctor --check-auth` for a problem it cannot see. The
+            // target's own first line of complaint distinguishes the two.
+            return skip("login-required", firstComplaint(spawned.stderr, spawned.stdout) ??
+                `the call was rejected with exit ${spawned.exitCode ?? "unknown"} and no output`);
+        }
+        const payload = extractReviewObject(target, spawned.stdout);
+        const parsed = payload === undefined
+            ? undefined
+            : validateReviewReport(payload, request.expectedCriterionIds, request.changedFiles);
+        if (parsed === undefined || !parsed.ok) {
+            const reason = parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE") ? "incomplete-scope" : "unparseable-output";
+            // Which contract the answer broke, or what the target said instead of
+            // answering. Without this the skip names only the classification, and a
+            // target that runs but never returns a usable report is undebuggable.
+            const errors = parsed === undefined
+                ? ""
+                : parsed.errors
+                    .slice(0, 3)
+                    .map((error) => `${error.path}: ${error.code}`)
+                    .join("; ");
+            const fields = parsed?.errors.some((error) => error.code === "INVALID_FIELDS") &&
+                payload !== undefined
+                ? ` (fields: ${Object.keys(payload).sort().join(", ")})`
+                : "";
+            return skip(reason, errors.length > 0
+                ? `${errors}${fields}`
+                : firstComplaint(spawned.stdout, spawned.stderr) ??
+                    "the answer carried no review report");
+        }
+        return { kind: "verdict", report: parsed.value };
+    }
+    finally {
+        await rm(attemptDirectory, { recursive: true, force: true });
+    }
+}
 /**
  * Builds the `execute` callback `runIndependentReview` expects: walk the
- * configured targets in order and return the first real verdict.
+ * configured targets in order and return the first real verdict. A PASS is then
+ * handed to a different target to refute, so a single agreeable reviewer cannot
+ * wave a change through on its own.
  */
 export function createReviewExecutor(options) {
     const report = options.onProgress ?? (() => { });
     const host = detectHostTarget(options.env ?? process.env);
     const chain = orderChain(options.targets, host);
     return async (request) => {
-        const expected = request.invocation.packet.criteria.map((criterion) => criterion.id);
-        const prompt = buildReviewPrompt(request.invocation);
+        const expectedCriterionIds = request.invocation.packet.criteria.map((criterion) => criterion.id);
         const repositoryRoot = await realpath(options.cwd);
-        let unavailable = false;
-        for (const [index, target] of chain.entries()) {
-            if (!hasRequiredReviewIsolation(target)) {
-                unavailable = true;
-                report(`${target}: required context-isolation controls unavailable → skipping`);
-                continue;
-            }
-            const attemptDirectory = await mkdtemp(join(tmpdir(), "agent-ops-review-"));
-            try {
-                const invocation = buildTargetInvocation({
+        const shared = {
+            repositoryRoot,
+            expectedCriterionIds,
+            ...(request.invocation.scope?.changedFiles === undefined
+                ? {}
+                : { changedFiles: request.invocation.scope.changedFiles })
+        };
+        const attempts = [];
+        let lastReason = "missing-cli";
+        /**
+         * Ask a target other than the one that passed — and other than the host —
+         * to refute the verdict. Returns undefined when no such target produced a
+         * report, which leaves the primary PASS standing unchallenged. Targets the
+         * primary pass already walked past are excluded: a CLI that could not
+         * answer the review prompt will not answer this one either.
+         */
+        const refute = async (primary, primaryTarget) => {
+            const walked = new Set(attempts.map((attempt) => attempt.target));
+            const candidates = chain.filter((target) => target !== primaryTarget && target !== host && !walked.has(target));
+            for (const [index, target] of candidates.entries()) {
+                const outcome = await attemptTarget({
+                    ...shared,
                     target,
-                    prompt,
-                    repositoryRoot,
-                    ...(options.model === undefined ? {} : { model: options.model }),
-                    ...(options.effort === undefined ? {} : { effort: options.effort })
-                });
-                if (invocation === undefined) {
-                    unavailable = true;
-                    report(`${target}: no read-only mode available → skipping`);
-                    continue;
-                }
-                if (target === host) {
-                    report(`${target}: reviewer == host; no independent target configured`);
-                }
-                const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
-                const capability = await runVerificationCommand({
-                    id: `review-capability-${target}-${index}`,
-                    command: invocation.command,
-                    args: ["--help"],
-                    cwd: attemptDirectory,
-                    required: true,
-                    evidence: { kind: "exit-code" },
-                    timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
-                }, {
-                    cwd: attemptDirectory,
-                    ...(options.runner === undefined ? {} : { runner: options.runner }),
-                    env: environment,
-                    replaceEnv: true
-                });
-                if (capability.status !== "PASS" ||
-                    capability.stdoutTruncated ||
-                    capability.stderrTruncated ||
-                    REQUIRED_HELP_FLAGS[target].some((flag) => !capability.stdout.includes(flag))) {
-                    unavailable = true;
-                    report(`${target}: required CLI capabilities unavailable → skipping`);
+                    label: `adversarial-${target}-${index}`,
+                    prompt: buildAdversarialPrompt({ ...request.invocation, harness: target }, primary)
+                }, options);
+                if (outcome.kind === "skip") {
+                    // Recorded, not just reported: progress is suppressed under --json,
+                    // and without this a PASS with no `adversarial` field cannot be told
+                    // apart from a PASS that had no second target to challenge it.
+                    attempts.push({
+                        target,
+                        status: "NOT_RUN",
+                        reason: outcome.attemptReason ?? outcome.reason,
+                        diagnostic: outcome.diagnostic
+                    });
+                    report(`${target}: adversarial re-check unavailable (${outcome.reason})`);
                     continue;
                 }
-                const spawned = await runVerificationCommand({
-                    id: `review-${target}-${index}`,
-                    command: invocation.command,
-                    args: [...invocation.args],
-                    cwd: attemptDirectory,
-                    required: true,
-                    evidence: { kind: "exit-code" },
-                    timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
-                }, {
-                    cwd: attemptDirectory,
-                    ...(options.runner === undefined ? {} : { runner: options.runner }),
-                    ...(options.outputLimitBytes === undefined
-                        ? {}
-                        : { outputLimitBytes: options.outputLimitBytes }),
-                    stdin: invocation.stdin,
-                    env: environment,
-                    replaceEnv: true
+                const refuted = reviewReportStatus(outcome.report) === "FAIL";
+                report(`${target}: adversarial re-check ${refuted ? "refuted the PASS" : "upheld the PASS"}`);
+                return { target, refuted, report: outcome.report };
+            }
+            if (candidates.length > 0) {
+                report("no independent target completed an adversarial re-check");
+            }
+            return undefined;
+        };
+        for (const [index, target] of chain.entries()) {
+            if (target === host) {
+                report(`${target}: reviewer == host; no independent target configured`);
+            }
+            const outcome = await attemptTarget({
+                ...shared,
+                target,
+                label: `${target}-${index}`,
+                prompt: buildReviewPrompt({ ...request.invocation, harness: target })
+            }, options);
+            if (outcome.kind === "skip") {
+                lastReason = outcome.reason;
+                attempts.push({
+                    target,
+                    status: "NOT_RUN",
+                    reason: outcome.attemptReason ?? outcome.reason,
+                    diagnostic: outcome.diagnostic
                 });
-                if (ADVANCING.has(spawned.failureClass)) {
-                    report(`${target}: ${spawned.failureClass} → trying next target`);
-                    continue;
-                }
-                if (spawned.stdoutTruncated || spawned.stderrTruncated) {
-                    return { status: "NOT_RUN", reason: "output-too-large", harness: target };
-                }
-                if (spawned.failureClass === "nonzero-exit") {
-                    return { status: "NOT_RUN", reason: "login-required", harness: target };
-                }
-                const payload = extractReviewObject(target, spawned.stdout);
-                const parsed = payload === undefined
-                    ? undefined
-                    : validateReviewReport(payload, expected, request.invocation.scope?.changedFiles);
-                if (parsed === undefined || !parsed.ok) {
-                    return {
-                        status: "NOT_RUN",
-                        reason: parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE")
-                            ? "incomplete-scope"
-                            : "unparseable-output",
-                        harness: target,
-                        ...(parsed === undefined ? {} : { validationErrors: parsed.errors })
-                    };
-                }
-                const reportValue = parsed.value;
-                const results = reviewReportResults(reportValue);
-                return {
-                    status: reviewReportStatus(reportValue),
-                    results,
-                    report: reportValue,
-                    harness: target,
-                    independence: host === undefined
-                        ? "unknown"
-                        : host === target
-                            ? "same-target"
-                            : "different-target"
-                };
+                report(outcome.message);
+                continue;
             }
-            finally {
-                await rm(attemptDirectory, { recursive: true, force: true });
+            const reportValue = outcome.report;
+            const status = reviewReportStatus(reportValue);
+            attempts.push({ target, status });
+            const independence = host === undefined
+                ? "unknown"
+                : host === target
+                    ? "same-target"
+                    : "different-target";
+            const verdict = {
+                results: reviewReportResults(reportValue),
+                report: reportValue,
+                harness: target,
+                attempts,
+                independence
+            };
+            if (status === "FAIL") {
+                return { status, ...verdict };
             }
+            const adversarial = await refute(reportValue, target);
+            return {
+                status: adversarial?.refuted === true ? "FAIL" : "PASS",
+                ...verdict,
+                ...(adversarial === undefined ? {} : { adversarial })
+            };
         }
         return {
             status: "NOT_RUN",
-            reason: unavailable ? "capability-unavailable" : "missing-cli"
+            reason: lastReason,
+            ...(attempts.length === 0 ? {} : { harness: attempts.at(-1)?.target }),
+            attempts
         };
     };
 }

package/dist/runtime/src/review/extract.js

@@ -21,6 +21,11 @@ function parseObject(text) {
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
+/** Agy appends plan UI metadata even when native JSON Schema is enabled. */
+function withoutAgyPlanMetadata(value) {
+    const { toolAction: _toolAction, toolSummary: _toolSummary, ...report } = value;
+    return report;
+}
 /**
  * The model's answer as text, before any JSON contract is applied. Returns
  * undefined rather than throwing so the caller can report
@@ -63,7 +68,10 @@ export function extractReviewObject(target, stdout) {
     const key = target === "claude" ? "structured_output" : "response";
     const value = envelope?.[key];
     if (isRecord(value)) {
-        return value;
+        return target === "agy" ? withoutAgyPlanMetadata(value) : value;
     }
-    return typeof value === "string" ? extractJsonObject(value) : undefined;
+    const parsed = typeof value === "string" ? extractJsonObject(value) : undefined;
+    return parsed === undefined || target !== "agy"
+        ? parsed
+        : withoutAgyPlanMetadata(parsed);
 }