@kylecheng3146/agent-ops 0.1.7 → 0.1.8

package/dist/runtime/src/review/execute.js

@@ -1,6 +1,10 @@
+import { mkdtemp, realpath, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { runVerificationCommand } from "../verify/spawn.js";
-import { extractFinalMessage, extractJsonObject } from "./extract.js";
+import { extractReviewObject } from "./extract.js";
 import { buildTargetInvocation } from "./invocation.js";
+import { reviewReportResults, reviewReportStatus, validateReviewReport } from "./report.js";
 import { detectHostTarget, orderChain } from "./roles.js";
 import { buildReviewPrompt } from "./runner.js";
 /**
@@ -8,6 +12,41 @@ import { buildReviewPrompt } from "./runner.js";
  * chain, so the worst case is targets x timeout.
  */
 export const DEFAULT_REVIEW_TIMEOUT_MS = 120_000;
+const EXECUTION_ENV = [
+    "PATH", "PATHEXT", "SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC",
+    "LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP"
+];
+const AUTH_ENV = {
+    claude: ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"],
+    codex: ["OPENAI_API_KEY"],
+    agy: ["AGY_API_KEY"]
+};
+export function isolatedReviewEnvironment(target, directory, source) {
+    const env = {};
+    for (const key of [...EXECUTION_ENV, ...AUTH_ENV[target]]) {
+        const value = source[key];
+        if (value !== undefined) {
+            env[key] = value;
+        }
+    }
+    env.HOME = directory;
+    env.USERPROFILE = directory;
+    env.XDG_CONFIG_HOME = join(directory, "config");
+    env.XDG_CACHE_HOME = join(directory, "cache");
+    return env;
+}
+/** Codex and agy currently lack documented instruction/customization isolation. */
+export function hasRequiredReviewIsolation(target) {
+    return target === "claude";
+}
+const REQUIRED_HELP_FLAGS = {
+    claude: [
+        "--add-dir", "--permission-mode", "--no-session-persistence",
+        "--safe-mode", "--disable-slash-commands", "--json-schema"
+    ],
+    codex: [],
+    agy: []
+};
 /**
  * Failure classes that mean no review happened, so trying the next target is
  * not review shopping. Everything else — including FAIL — is terminal.
@@ -17,47 +56,6 @@ const ADVANCING = new Set([
     "spawn-failed",
     "timeout"
 ]);
-function statusOf(value) {
-    return value === "PASS" || value === "FAIL" ? value : undefined;
-}
-/**
- * The response must name every requested criterion exactly once, with at least
- * one non-blank evidence reference. A response that breaks the contract is
- * unparseable output, never a FAIL verdict: FAIL has to keep meaning "the
- * reviewer looked and judged it inadequate".
- */
-function parseResults(payload, expected) {
-    const raw = payload.results;
-    if (!Array.isArray(raw) || raw.length !== expected.length) {
-        return undefined;
-    }
-    const results = [];
-    const seen = new Set();
-    for (const entry of raw) {
-        if (typeof entry !== "object" || entry === null) {
-            return undefined;
-        }
-        const item = entry;
-        const criterionId = item.criterionId;
-        const status = statusOf(item.status);
-        if (typeof criterionId !== "string" ||
-            status === undefined ||
-            !expected.includes(criterionId) ||
-            seen.has(criterionId) ||
-            !Array.isArray(item.evidence) ||
-            item.evidence.length === 0 ||
-            !item.evidence.every((reference) => typeof reference === "string" && reference.trim().length > 0)) {
-            return undefined;
-        }
-        seen.add(criterionId);
-        results.push({
-            criterionId,
-            status,
-            evidence: item.evidence.map((reference) => String(reference))
-        });
-    }
-    return results;
-}
 /**
  * Builds the `execute` callback `runIndependentReview` expects: walk the
  * configured targets in order and return the first real verdict.
@@ -69,52 +67,114 @@ export function createReviewExecutor(options) {
     return async (request) => {
         const expected = request.invocation.packet.criteria.map((criterion) => criterion.id);
         const prompt = buildReviewPrompt(request.invocation);
+        const repositoryRoot = await realpath(options.cwd);
+        let unavailable = false;
         for (const [index, target] of chain.entries()) {
-            const invocation = buildTargetInvocation({
-                target,
-                prompt,
-                ...(options.model === undefined ? {} : { model: options.model }),
-                ...(options.effort === undefined ? {} : { effort: options.effort })
-            });
-            if (invocation === undefined) {
-                report(`${target}: no read-only mode available → skipping`);
-                continue;
-            }
-            if (target === host) {
-                report(`${target}: reviewer == host; no independent target configured`);
-            }
-            const spawned = await runVerificationCommand({
-                id: `review-${target}-${index}`,
-                command: invocation.command,
-                args: [...invocation.args],
-                cwd: options.cwd,
-                required: true,
-                evidence: { kind: "exit-code" },
-                timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
-            }, {
-                cwd: options.cwd,
-                ...(options.runner === undefined ? {} : { runner: options.runner }),
-                ...(options.outputLimitBytes === undefined
-                    ? {}
-                    : { outputLimitBytes: options.outputLimitBytes })
-            });
-            if (ADVANCING.has(spawned.failureClass)) {
-                report(`${target}: ${spawned.failureClass} → trying next target`);
+            if (!hasRequiredReviewIsolation(target)) {
+                unavailable = true;
+                report(`${target}: required context-isolation controls unavailable → skipping`);
                 continue;
             }
-            if (spawned.stdoutTruncated) {
-                return { status: "NOT_RUN", reason: "unparseable-output" };
+            const attemptDirectory = await mkdtemp(join(tmpdir(), "agent-ops-review-"));
+            try {
+                const invocation = buildTargetInvocation({
+                    target,
+                    prompt,
+                    repositoryRoot,
+                    ...(options.model === undefined ? {} : { model: options.model }),
+                    ...(options.effort === undefined ? {} : { effort: options.effort })
+                });
+                if (invocation === undefined) {
+                    unavailable = true;
+                    report(`${target}: no read-only mode available → skipping`);
+                    continue;
+                }
+                if (target === host) {
+                    report(`${target}: reviewer == host; no independent target configured`);
+                }
+                const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
+                const capability = await runVerificationCommand({
+                    id: `review-capability-${target}-${index}`,
+                    command: invocation.command,
+                    args: ["--help"],
+                    cwd: attemptDirectory,
+                    required: true,
+                    evidence: { kind: "exit-code" },
+                    timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
+                }, {
+                    cwd: attemptDirectory,
+                    ...(options.runner === undefined ? {} : { runner: options.runner }),
+                    env: environment,
+                    replaceEnv: true
+                });
+                if (capability.status !== "PASS" ||
+                    capability.stdoutTruncated ||
+                    capability.stderrTruncated ||
+                    REQUIRED_HELP_FLAGS[target].some((flag) => !capability.stdout.includes(flag))) {
+                    unavailable = true;
+                    report(`${target}: required CLI capabilities unavailable → skipping`);
+                    continue;
+                }
+                const spawned = await runVerificationCommand({
+                    id: `review-${target}-${index}`,
+                    command: invocation.command,
+                    args: [...invocation.args],
+                    cwd: attemptDirectory,
+                    required: true,
+                    evidence: { kind: "exit-code" },
+                    timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
+                }, {
+                    cwd: attemptDirectory,
+                    ...(options.runner === undefined ? {} : { runner: options.runner }),
+                    ...(options.outputLimitBytes === undefined
+                        ? {}
+                        : { outputLimitBytes: options.outputLimitBytes }),
+                    stdin: invocation.stdin,
+                    env: environment,
+                    replaceEnv: true
+                });
+                if (ADVANCING.has(spawned.failureClass)) {
+                    report(`${target}: ${spawned.failureClass} → trying next target`);
+                    continue;
+                }
+                if (spawned.stdoutTruncated || spawned.stderrTruncated) {
+                    return { status: "NOT_RUN", reason: "output-too-large", harness: target };
+                }
+                const payload = extractReviewObject(target, spawned.stdout);
+                const parsed = payload === undefined
+                    ? undefined
+                    : validateReviewReport(payload, expected, request.invocation.scope?.changedFiles);
+                if (parsed === undefined || !parsed.ok) {
+                    return {
+                        status: "NOT_RUN",
+                        reason: parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE")
+                            ? "incomplete-scope"
+                            : "unparseable-output",
+                        harness: target,
+                        ...(parsed === undefined ? {} : { validationErrors: parsed.errors })
+                    };
+                }
+                const reportValue = parsed.value;
+                const results = reviewReportResults(reportValue);
+                return {
+                    status: reviewReportStatus(reportValue),
+                    results,
+                    report: reportValue,
+                    harness: target,
+                    independence: host === undefined
+                        ? "unknown"
+                        : host === target
+                            ? "same-target"
+                            : "different-target"
+                };
             }
-            const message = extractFinalMessage(target, spawned.stdout);
-            const payload = message === undefined ? undefined : extractJsonObject(message);
-            const results = payload === undefined ? undefined : parseResults(payload, expected);
-            if (results === undefined) {
-                return { status: "NOT_RUN", reason: "unparseable-output" };
+            finally {
+                await rm(attemptDirectory, { recursive: true, force: true });
             }
-            return results.every((result) => result.status === "PASS")
-                ? { status: "PASS", results }
-                : { status: "FAIL", results };
         }
-        return { status: "NOT_RUN", reason: "missing-cli" };
+        return {
+            status: "NOT_RUN",
+            reason: unavailable ? "capability-unavailable" : "missing-cli"
+        };
     };
 }

package/dist/runtime/src/review/extract.js

@@ -18,6 +18,9 @@ function parseObject(text) {
         ? parsed
         : undefined;
 }
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
 /**
  * The model's answer as text, before any JSON contract is applied. Returns
  * undefined rather than throwing so the caller can report
@@ -45,27 +48,22 @@ export function extractFinalMessage(target, stdout) {
  * raw stdout, so it cannot capture a transport envelope.
  */
 export function extractJsonObject(text) {
-    for (let end = text.lastIndexOf("}"); end !== -1; end = text.lastIndexOf("}", end - 1)) {
-        let depth = 0;
-        for (let start = end; start >= 0; start -= 1) {
-            const character = text[start];
-            if (character === "}") {
-                depth += 1;
-                continue;
-            }
-            if (character !== "{") {
-                continue;
-            }
-            depth -= 1;
-            if (depth !== 0) {
-                continue;
-            }
-            const candidate = parseObject(text.slice(start, end + 1));
-            if (candidate !== undefined) {
-                return candidate;
-            }
-            break;
-        }
+    return parseObject(text.trim());
+}
+/**
+ * Review results use a strict native structured-output transport. Unlike the
+ * legacy probe parser above, this path neither recovers fenced JSON nor accepts
+ * a provider's generic text result field.
+ */
+export function extractReviewObject(target, stdout) {
+    if (target === "codex") {
+        return extractJsonObject(stdout);
+    }
+    const envelope = parseObject(stdout);
+    const key = target === "claude" ? "structured_output" : "response";
+    const value = envelope?.[key];
+    if (isRecord(value)) {
+        return value;
     }
-    return undefined;
+    return typeof value === "string" ? extractJsonObject(value) : undefined;
 }

package/dist/runtime/src/review/invocation.js

@@ -1,3 +1,19 @@
+import { existsSync, readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+function reviewSchemaPath() {
+    const packaged = fileURLToPath(new URL("../../../../schemas/review-report.schema.json", import.meta.url));
+    if (existsSync(packaged)) {
+        return packaged;
+    }
+    const source = fileURLToPath(new URL("../../../schemas/review-report.schema.json", import.meta.url));
+    return existsSync(source)
+        ? source
+        : resolve(process.cwd(), "schemas", "review-report.schema.json");
+}
+function reviewSchemaText() {
+    return readFileSync(reviewSchemaPath(), "utf8");
+}
 /**
  * Read-only enforcement per target, verified against each CLI's own help
  * output. A target absent from this table is ineligible: review never runs an
@@ -42,11 +58,59 @@ export function buildTargetInvocation(request) {
         // directory, which a caller would otherwise read as "not authenticated".
         return {
             command: "codex",
-            args: ["exec", request.prompt, "--skip-git-repo-check", ...shared]
+            args: [
+                "exec",
+                "-",
+                "--skip-git-repo-check",
+                "--output-schema",
+                reviewSchemaPath(),
+                ...(request.repositoryRoot === undefined
+                    ? []
+                    : ["--add-dir", request.repositoryRoot, "--ephemeral", "--ignore-rules"]),
+                ...shared
+            ],
+            stdin: request.prompt
+        };
+    }
+    const isolation = request.target === "claude"
+        ? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
+        : [];
+    return {
+        command: request.target,
+        args: [
+            "-p",
+            "--output-format",
+            "json",
+            "--json-schema",
+            reviewSchemaText(),
+            ...(request.repositoryRoot === undefined
+                ? []
+                : ["--add-dir", request.repositoryRoot]),
+            ...isolation,
+            ...shared
+        ],
+        stdin: request.prompt
+    };
+}
+/** A deep doctor probe uses stdin but keeps its simple text response contract. */
+export function buildProbeInvocation(request) {
+    const readOnly = READ_ONLY_ARGS[request.target];
+    if (readOnly === undefined || readOnly.length === 0) {
+        return undefined;
+    }
+    if (request.target === "codex") {
+        return {
+            command: "codex",
+            args: ["exec", "-", "--skip-git-repo-check", ...readOnly],
+            stdin: request.prompt
         };
     }
+    const isolation = request.target === "claude"
+        ? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
+        : [];
     return {
         command: request.target,
-        args: ["-p", request.prompt, "--output-format", "json", ...shared]
+        args: ["-p", "--output-format", "json", ...isolation, ...readOnly],
+        stdin: request.prompt
     };
 }

package/dist/runtime/src/review/packet.js

@@ -1,10 +1,47 @@
+const MAX_PACKET_BYTES = 64 * 1024;
+function safe(value) {
+    return safeTaskText(redactSecrets(value));
+}
+function checkSensitive(value) {
+    const decision = evaluateGuardrail({
+        kind: "content",
+        content: value,
+        scope: "review-packet"
+    });
+    if (decision.action === "block") {
+        throw new AgentOpsError("REVIEW_SENSITIVE_INPUT", "Review input contains credential-shaped content.");
+    }
+}
 export function buildReviewPacket(input) {
-    return {
-        request: input.request,
-        criteria: input.criteria.map((criterion) => ({ ...criterion })),
-        artifactRefs: [...input.artifactRefs],
+    for (const value of [
+        input.request,
+        ...input.criteria.flatMap((criterion) => [criterion.id, criterion.description, ...(criterion.verifierIds ?? [])]),
+        ...input.artifactRefs,
+        ...input.evidenceRequirements.flatMap((requirement) => [requirement.criterionId, requirement.requirement])
+    ]) {
+        checkSensitive(value);
+    }
+    const packet = {
+        request: safe(input.request),
+        criteria: input.criteria.map((criterion) => ({
+            id: safe(criterion.id),
+            description: safe(criterion.description),
+            ...(criterion.verifierIds === undefined
+                ? {}
+                : { verifierIds: criterion.verifierIds.map(safe) })
+        })),
+        artifactRefs: input.artifactRefs.map(safe),
         evidenceRequirements: input.evidenceRequirements.map((requirement) => ({
-            ...requirement
+            criterionId: safe(requirement.criterionId),
+            requirement: safe(requirement.requirement)
         }))
     };
+    if (Buffer.byteLength(JSON.stringify(packet), "utf8") > MAX_PACKET_BYTES) {
+        throw new AgentOpsError("REVIEW_SCOPE_TOO_LARGE", "Review packet exceeds the 64 KiB limit.");
+    }
+    return packet;
 }
+import { evaluateGuardrail } from "../guardrails/evaluate.js";
+import { AgentOpsError } from "../fs/paths.js";
+import { redactSecrets } from "../security/redact.js";
+import { safeTaskText } from "../task/render.js";

package/dist/runtime/src/review/probe.js

@@ -1,6 +1,10 @@
 import { runVerificationCommand } from "../verify/spawn.js";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { extractFinalMessage } from "./extract.js";
-import { buildTargetInvocation } from "./invocation.js";
+import { hasRequiredReviewIsolation, isolatedReviewEnvironment } from "./execute.js";
+import { buildProbeInvocation } from "./invocation.js";
 const PROBE_PROMPT = "Reply with the single word OK and nothing else.";
 /**
  * Matches the review timeout rather than being "quick": codex at high
@@ -15,34 +19,54 @@ const PROBE_TIMEOUT_MS = 120_000;
  * is not evidence at all.
  */
 export async function probeReviewTarget(target, options) {
-    const invocation = buildTargetInvocation({ target, prompt: PROBE_PROMPT });
-    if (invocation === undefined) {
+    const deep = options.deep === true;
+    if (deep && !hasRequiredReviewIsolation(target)) {
         return "ineligible";
     }
-    const deep = options.deep === true;
-    const spawned = await runVerificationCommand({
-        id: `review-probe-${target}`,
-        command: invocation.command,
-        args: deep ? [...invocation.args] : ["--version"],
-        cwd: options.cwd,
-        required: true,
-        evidence: { kind: "exit-code" },
-        timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS
-    }, {
-        cwd: options.cwd,
-        ...(options.runner === undefined ? {} : { runner: options.runner })
-    });
-    if (spawned.failureClass === "missing-executable") {
-        return "missing-executable";
+    const invocation = buildProbeInvocation({ target, prompt: PROBE_PROMPT });
+    if (invocation === undefined) {
+        return "ineligible";
     }
-    if (spawned.timedOut) {
-        return "timeout";
+    const directory = deep
+        ? await mkdtemp(join(tmpdir(), "agent-ops-review-probe-"))
+        : options.cwd;
+    try {
+        const spawned = await runVerificationCommand({
+            id: `review-probe-${target}`,
+            command: invocation.command,
+            args: deep ? [...invocation.args] : ["--version"],
+            cwd: directory,
+            required: true,
+            evidence: { kind: "exit-code" },
+            timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS
+        }, {
+            cwd: directory,
+            ...(options.runner === undefined ? {} : { runner: options.runner }),
+            ...(deep
+                ? {
+                    stdin: invocation.stdin,
+                    env: isolatedReviewEnvironment(target, directory, process.env),
+                    replaceEnv: true
+                }
+                : {})
+        });
+        if (spawned.failureClass === "missing-executable") {
+            return "missing-executable";
+        }
+        if (spawned.timedOut) {
+            return "timeout";
+        }
+        if (!deep) {
+            return spawned.status === "PASS" ? "ok" : "unauthenticated";
+        }
+        return spawned.status === "PASS" &&
+            extractFinalMessage(target, spawned.stdout) !== undefined
+            ? "ok"
+            : "unauthenticated";
     }
-    if (!deep) {
-        return spawned.status === "PASS" ? "ok" : "unauthenticated";
+    finally {
+        if (deep) {
+            await rm(directory, { recursive: true, force: true });
+        }
     }
-    return spawned.status === "PASS" &&
-        extractFinalMessage(target, spawned.stdout) !== undefined
-        ? "ok"
-        : "unauthenticated";
 }

package/dist/runtime/src/review/render.js

@@ -0,0 +1,62 @@
+import { redactSecrets } from "../security/redact.js";
+import { safeTaskText } from "../task/render.js";
+function safe(value) {
+    return safeTaskText(redactSecrets(value));
+}
+function lineList(values) {
+    return values.length === 0 ? ["- none"] : values.map((value) => `- ${safe(value)}`);
+}
+export function renderReviewResult(result) {
+    const lines = [
+        `Independent review: ${result.status}`,
+        `Reviewer: ${result.harness}; model: ${safe(result.model)}; effort: ${safe(result.effort)}.`
+    ];
+    if (result.scope !== undefined) {
+        lines.push(result.scope.mode === "base"
+            ? `Scope: ${result.scope.mode} ${safe(result.scope.baseRef)} (${safe(result.scope.resolvedBase)}).`
+            : "Scope: worktree.");
+    }
+    if (result.independence !== undefined) {
+        lines.push(`Independence: ${result.independence}.`);
+    }
+    if (result.verification !== undefined) {
+        lines.push("Machine verification:");
+        for (const command of result.verification.commands) {
+            lines.push(`- ${safe(command.criterionId)}/${safe(command.commandId)}: ${command.status}` +
+                `${command.required ? " (required)" : " (optional)"}` +
+                `${command.evidenceReference === undefined ? "" : ` — ${safe(command.evidenceReference)}`}`);
+        }
+    }
+    if (result.report === undefined) {
+        lines.push(`Reason: ${result.reason ?? "unknown"}.`);
+        for (const error of result.validationErrors ?? []) {
+            lines.push(`- ${safe(error.path)}: ${safe(error.code)} — ${safe(error.message)}`);
+        }
+        lines.push("Run: agent-ops doctor --check-auth to verify target authentication.");
+        return `${lines.join("\n")}\n`;
+    }
+    const report = result.report;
+    const nonBlocking = report.findings.filter((finding) => !finding.blocking).length;
+    lines.push(`Non-blocking findings: ${nonBlocking}.`, "", "Summary:", safe(report.summary), "", "Criteria:");
+    for (const item of report.results) {
+        lines.push(`- ${safe(item.criterionId)}: ${item.status} — ${safe(item.summary)}`);
+        lines.push(...item.evidence.map((evidence) => `  - ${safe(evidence)}`));
+    }
+    lines.push("", "Findings:");
+    if (report.findings.length === 0) {
+        lines.push("- none");
+    }
+    else {
+        for (const finding of report.findings) {
+            lines.push(`- [${finding.severity}] ${finding.blocking ? "blocking" : "non-blocking"}: ${safe(finding.title)}`);
+            lines.push(`  ${safe(finding.details)}`);
+            lines.push(`  Recommendation: ${safe(finding.recommendation)}`);
+            lines.push(...finding.locations.map((location) => `  Location: ${safe(location.path)}${location.line === undefined ? "" : `:${location.line}`}`));
+            lines.push(...finding.evidence.map((evidence) => `  Evidence: ${safe(evidence)}`));
+        }
+    }
+    lines.push("", "Residual risks:", ...lineList(report.residualRisks));
+    lines.push("", "Changed files inspected:", ...lineList(report.changedFilesInspected));
+    lines.push("", "Supporting files inspected:", ...lineList(report.supportingFilesInspected));
+    return `${lines.join("\n")}\n`;
+}