@kylecheng3146/agent-ops 0.1.5 → 0.1.7

package/dist/runtime/src/review/execute.js

@@ -0,0 +1,120 @@
+import { runVerificationCommand } from "../verify/spawn.js";
+import { extractFinalMessage, extractJsonObject } from "./extract.js";
+import { buildTargetInvocation } from "./invocation.js";
+import { detectHostTarget, orderChain } from "./roles.js";
+import { buildReviewPrompt } from "./runner.js";
+/**
+ * Deliberately below the five-minute `spawn.ts` default: a timeout advances the
+ * chain, so the worst case is targets x timeout.
+ */
+export const DEFAULT_REVIEW_TIMEOUT_MS = 120_000;
+/**
+ * Failure classes that mean no review happened, so trying the next target is
+ * not review shopping. Everything else — including FAIL — is terminal.
+ */
+const ADVANCING = new Set([
+    "missing-executable",
+    "spawn-failed",
+    "timeout"
+]);
+function statusOf(value) {
+    return value === "PASS" || value === "FAIL" ? value : undefined;
+}
+/**
+ * The response must name every requested criterion exactly once, with at least
+ * one non-blank evidence reference. A response that breaks the contract is
+ * unparseable output, never a FAIL verdict: FAIL has to keep meaning "the
+ * reviewer looked and judged it inadequate".
+ */
+function parseResults(payload, expected) {
+    const raw = payload.results;
+    if (!Array.isArray(raw) || raw.length !== expected.length) {
+        return undefined;
+    }
+    const results = [];
+    const seen = new Set();
+    for (const entry of raw) {
+        if (typeof entry !== "object" || entry === null) {
+            return undefined;
+        }
+        const item = entry;
+        const criterionId = item.criterionId;
+        const status = statusOf(item.status);
+        if (typeof criterionId !== "string" ||
+            status === undefined ||
+            !expected.includes(criterionId) ||
+            seen.has(criterionId) ||
+            !Array.isArray(item.evidence) ||
+            item.evidence.length === 0 ||
+            !item.evidence.every((reference) => typeof reference === "string" && reference.trim().length > 0)) {
+            return undefined;
+        }
+        seen.add(criterionId);
+        results.push({
+            criterionId,
+            status,
+            evidence: item.evidence.map((reference) => String(reference))
+        });
+    }
+    return results;
+}
+/**
+ * Builds the `execute` callback `runIndependentReview` expects: walk the
+ * configured targets in order and return the first real verdict.
+ */
+export function createReviewExecutor(options) {
+    const report = options.onProgress ?? (() => { });
+    const host = detectHostTarget(options.env ?? process.env);
+    const chain = orderChain(options.targets, host);
+    return async (request) => {
+        const expected = request.invocation.packet.criteria.map((criterion) => criterion.id);
+        const prompt = buildReviewPrompt(request.invocation);
+        for (const [index, target] of chain.entries()) {
+            const invocation = buildTargetInvocation({
+                target,
+                prompt,
+                ...(options.model === undefined ? {} : { model: options.model }),
+                ...(options.effort === undefined ? {} : { effort: options.effort })
+            });
+            if (invocation === undefined) {
+                report(`${target}: no read-only mode available → skipping`);
+                continue;
+            }
+            if (target === host) {
+                report(`${target}: reviewer == host; no independent target configured`);
+            }
+            const spawned = await runVerificationCommand({
+                id: `review-${target}-${index}`,
+                command: invocation.command,
+                args: [...invocation.args],
+                cwd: options.cwd,
+                required: true,
+                evidence: { kind: "exit-code" },
+                timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
+            }, {
+                cwd: options.cwd,
+                ...(options.runner === undefined ? {} : { runner: options.runner }),
+                ...(options.outputLimitBytes === undefined
+                    ? {}
+                    : { outputLimitBytes: options.outputLimitBytes })
+            });
+            if (ADVANCING.has(spawned.failureClass)) {
+                report(`${target}: ${spawned.failureClass} → trying next target`);
+                continue;
+            }
+            if (spawned.stdoutTruncated) {
+                return { status: "NOT_RUN", reason: "unparseable-output" };
+            }
+            const message = extractFinalMessage(target, spawned.stdout);
+            const payload = message === undefined ? undefined : extractJsonObject(message);
+            const results = payload === undefined ? undefined : parseResults(payload, expected);
+            if (results === undefined) {
+                return { status: "NOT_RUN", reason: "unparseable-output" };
+            }
+            return results.every((result) => result.status === "PASS")
+                ? { status: "PASS", results }
+                : { status: "FAIL", results };
+        }
+        return { status: "NOT_RUN", reason: "missing-cli" };
+    };
+}

package/dist/runtime/src/review/extract.js

@@ -0,0 +1,71 @@
+/**
+ * The key holding the model's answer differs in every envelope, so every
+ * target gets its own branch. Verified against tests/fixtures/review/.
+ */
+const ENVELOPE_KEYS = {
+    agy: "response",
+    claude: "result"
+};
+function parseObject(text) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+    return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
+        ? parsed
+        : undefined;
+}
+/**
+ * The model's answer as text, before any JSON contract is applied. Returns
+ * undefined rather than throwing so the caller can report
+ * `unparseable-output` for every transport failure through one path.
+ */
+export function extractFinalMessage(target, stdout) {
+    const key = ENVELOPE_KEYS[target];
+    if (key === undefined) {
+        // codex: stdout is the final message itself.
+        const trimmed = stdout.trim();
+        return trimmed.length === 0 ? undefined : trimmed;
+    }
+    const envelope = parseObject(stdout);
+    const value = envelope?.[key];
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed.length === 0 ? undefined : trimmed;
+}
+/**
+ * The last balanced JSON object in a block of model text. Scanning backwards
+ * matters: models often restate the schema before answering, and the answer is
+ * what comes last. This only ever runs on the extracted final message, never on
+ * raw stdout, so it cannot capture a transport envelope.
+ */
+export function extractJsonObject(text) {
+    for (let end = text.lastIndexOf("}"); end !== -1; end = text.lastIndexOf("}", end - 1)) {
+        let depth = 0;
+        for (let start = end; start >= 0; start -= 1) {
+            const character = text[start];
+            if (character === "}") {
+                depth += 1;
+                continue;
+            }
+            if (character !== "{") {
+                continue;
+            }
+            depth -= 1;
+            if (depth !== 0) {
+                continue;
+            }
+            const candidate = parseObject(text.slice(start, end + 1));
+            if (candidate !== undefined) {
+                return candidate;
+            }
+            break;
+        }
+    }
+    return undefined;
+}

package/dist/runtime/src/review/invocation.js

@@ -0,0 +1,52 @@
+/**
+ * Read-only enforcement per target, verified against each CLI's own help
+ * output. A target absent from this table is ineligible: review never runs an
+ * agent that can edit the code it is reviewing. This is what excludes
+ * opencode, whose `--agent plan` is rejected as a subagent and silently falls
+ * back to a writable agent.
+ */
+export const READ_ONLY_ARGS = {
+    agy: ["--sandbox", "--mode", "plan"],
+    claude: ["--permission-mode", "plan"],
+    codex: ["-s", "read-only"]
+};
+function modelArgs(target, model) {
+    if (model === undefined) {
+        return [];
+    }
+    return target === "codex" ? ["-m", model] : ["--model", model];
+}
+function effortArgs(target, effort) {
+    if (effort === undefined) {
+        return [];
+    }
+    // codex has no --effort flag; reasoning effort is a config override.
+    return target === "codex"
+        ? ["-c", `model_reasoning_effort=${effort}`]
+        : ["--effort", effort];
+}
+export function buildTargetInvocation(request) {
+    const readOnly = READ_ONLY_ARGS[request.target];
+    if (readOnly === undefined || readOnly.length === 0) {
+        return undefined;
+    }
+    const shared = [
+        ...readOnly,
+        ...modelArgs(request.target, request.model),
+        ...effortArgs(request.target, request.effort)
+    ];
+    if (request.target === "codex") {
+        // codex writes progress to stderr and leaves stdout as the bare final
+        // message, so it needs no output-format flag and no scratch file.
+        // Without --skip-git-repo-check it refuses to run outside a trusted git
+        // directory, which a caller would otherwise read as "not authenticated".
+        return {
+            command: "codex",
+            args: ["exec", request.prompt, "--skip-git-repo-check", ...shared]
+        };
+    }
+    return {
+        command: request.target,
+        args: ["-p", request.prompt, "--output-format", "json", ...shared]
+    };
+}

package/dist/runtime/src/review/probe.js

@@ -0,0 +1,48 @@
+import { runVerificationCommand } from "../verify/spawn.js";
+import { extractFinalMessage } from "./extract.js";
+import { buildTargetInvocation } from "./invocation.js";
+const PROBE_PROMPT = "Reply with the single word OK and nothing else.";
+/**
+ * Matches the review timeout rather than being "quick": codex at high
+ * reasoning effort answers a trivial prompt in ~20s, and a probe that times out
+ * would otherwise be reported as an authentication failure.
+ */
+const PROBE_TIMEOUT_MS = 120_000;
+/**
+ * The only check that actually proves a target is usable: ask it something
+ * trivial and see whether an answer comes back. A credential-file check can
+ * pass while the token is expired, and self-declaration ("already logged in?")
+ * is not evidence at all.
+ */
+export async function probeReviewTarget(target, options) {
+    const invocation = buildTargetInvocation({ target, prompt: PROBE_PROMPT });
+    if (invocation === undefined) {
+        return "ineligible";
+    }
+    const deep = options.deep === true;
+    const spawned = await runVerificationCommand({
+        id: `review-probe-${target}`,
+        command: invocation.command,
+        args: deep ? [...invocation.args] : ["--version"],
+        cwd: options.cwd,
+        required: true,
+        evidence: { kind: "exit-code" },
+        timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS
+    }, {
+        cwd: options.cwd,
+        ...(options.runner === undefined ? {} : { runner: options.runner })
+    });
+    if (spawned.failureClass === "missing-executable") {
+        return "missing-executable";
+    }
+    if (spawned.timedOut) {
+        return "timeout";
+    }
+    if (!deep) {
+        return spawned.status === "PASS" ? "ok" : "unauthenticated";
+    }
+    return spawned.status === "PASS" &&
+        extractFinalMessage(target, spawned.stdout) !== undefined
+        ? "ok"
+        : "unauthenticated";
+}

package/dist/runtime/src/review/result.js

@@ -14,10 +14,10 @@ export function aggregateReviewResults(requestedCriterionIds, results) {
     if (seen.size !== expected.size) {
         valid = false;
     }
-    const status = valid && results.every((result) => result.status === "PASS")
+    const status = results.every((result) => result.status === "PASS")
         ? "PASS"
         : "FAIL";
-    return { status, results: [...results] };
+    return { status, results: [...results], valid };
 }
 export function summarizeReview(request) {
     return aggregateReviewResults(request.packet.criteria.map((criterion) => criterion.id), request.criterionResults);

package/dist/runtime/src/review/roles.js

@@ -1,3 +1,38 @@
+/**
+ * Default chain order. codex first because its stdout is the bare final message
+ * (nothing to unwrap), then agy's flat envelope. claude is last because it is
+ * the only host we can detect, and `orderChain` would push it back anyway.
+ */
+export const DEFAULT_REVIEW_TARGETS = [
+    "codex",
+    "agy",
+    "claude"
+];
 export function resolveReviewRole(role, configured) {
     return configured.find((item) => item.role === role);
 }
+export function reviewTargets(config, role) {
+    return resolveReviewRole(role, config.reviewRoles ?? [])?.targets ?? [];
+}
+/**
+ * Which review target is hosting this process, when that is knowable. Only
+ * Claude Code publishes a documented marker; guessing the others would produce
+ * a detector that silently fails, which is worse than no detector.
+ */
+export function detectHostTarget(env) {
+    return env.CLAUDECODE === undefined ? undefined : "claude";
+}
+/**
+ * Move the hosting target to the end so an independent reviewer is preferred,
+ * without ever dropping it — a single configured target still runs, self-review
+ * warning and all.
+ */
+export function orderChain(targets, host) {
+    if (host === undefined) {
+        return [...targets];
+    }
+    return [
+        ...targets.filter((target) => target !== host),
+        ...targets.filter((target) => target === host)
+    ];
+}

package/dist/runtime/src/review/runner.js

@@ -1,12 +1,43 @@
 import { aggregateReviewResults } from "./result.js";
 import { redactSecrets } from "../security/redact.js";
 import { safeTaskText } from "../task/render.js";
-function promptFor(invocation) {
+function criterionLine(criterion) {
+    const verified = criterion.verifierIds ?? [];
+    const covered = verified.length === 0
+        ? ""
+        : ` (already machine-verified by: ${verified.join(", ")} —` +
+            " do not re-run those checks)";
+    return `- ${criterion.id}: ${criterion.description}${covered}`;
+}
+/**
+ * The prompt the reviewing CLI actually receives. It stays short on purpose: it
+ * travels through argv, so an embedded diff would risk ARG_MAX and would expose
+ * the diff in `ps` output. The target inspects the repository itself instead,
+ * which its read-only sandbox permits.
+ */
+export function buildReviewPrompt(invocation) {
+    const ids = invocation.packet.criteria.map((criterion) => criterion.id);
+    const shape = ids
+        .map((id) => `{"criterionId":"${id}","status":"PASS|FAIL","evidence":["<reference>"]}`)
+        .join(",");
     return [
-        "Review the requested criteria in read-only mode.",
+        invocation.packet.request,
+        "",
+        "You are a read-only reviewer. Inspect this repository yourself " +
+            "(git diff, git log, reading files); do not modify anything.",
         `Harness: ${invocation.harness}; model: ${invocation.model}; effort: ${invocation.effort}.`,
         `Artifacts: ${invocation.packet.artifactRefs.join(", ") || "none"}.`,
-        `Criteria: ${invocation.packet.criteria.map((criterion) => criterion.id).join(", ") || "none"}.`
+        "",
+        "Criteria:",
+        ...(invocation.packet.criteria.length === 0
+            ? ["- none"]
+            : invocation.packet.criteria.map(criterionLine)),
+        ...invocation.packet.evidenceRequirements.map((requirement) => `- evidence for ${requirement.criterionId}: ${requirement.requirement}`),
+        "",
+        "Reply with exactly one JSON object and nothing else. Name every " +
+            "criterion above exactly once, each with at least one non-empty " +
+            "evidence reference:",
+        `{"results":[${shape}]}`
     ].join("\n");
 }
 function safeResult(result) {
@@ -21,7 +52,7 @@ export async function runIndependentReview(options) {
         harness: options.invocation.harness,
         model: options.invocation.model,
         effort: options.invocation.effort,
-        prompt: promptFor(options.invocation)
+        prompt: buildReviewPrompt(options.invocation)
     };
     if (!options.authorized) {
         return { ...base, status: "NOT_RUN", reason: "authorization-required" };
@@ -37,6 +68,9 @@ export async function runIndependentReview(options) {
         return { ...base, status: result.status, results: result.results };
     }
     const summary = aggregateReviewResults(options.invocation.packet.criteria.map((criterion) => criterion.id), result.results);
+    if (!summary.valid) {
+        return { ...base, status: "NOT_RUN", reason: "unparseable-output" };
+    }
     return {
         ...base,
         status: summary.status,

package/dist/runtime/src/schema/validate.js

@@ -2,15 +2,31 @@ import { CONFIG_SCHEMA_VERSION, EVIDENCE_SCHEMA_VERSION, MANIFEST_SCHEMA_VERSION
 const ID_PATTERN = /^[a-z][a-z0-9-]{0,127}$/;
 const HASH_PATTERN = /^[a-f0-9]{64}$/;
 const WINDOWS_RESERVED_SEGMENT = /^(?:aux|com[1-9]|con|lpt[1-9]|nul|prn)(?:\..*)?$/i;
-const PROFILE_VALUES = new Set(["advisory", "core", "guardrails"]);
+const PROFILE_VALUES = new Set(["advisory", "core", "guardrails", "loop"]);
 const EVIDENCE_KINDS = new Set(["exit-code", "file", "test-count"]);
 const SCOPE_VALUES = new Set(["project", "user"]);
 const HARNESS_VALUES = new Set(["claude", "codex", "opencode"]);
+const REVIEW_ROLE_VALUES = new Set([
+    "deep-reasoning",
+    "implementation",
+    "independent-review",
+    "mechanical"
+]);
+// opencode is absent by design: it has no read-only flag. See
+// docs/plans/2026-08-12-external-review-cli-targets.md.
+const REVIEW_TARGET_VALUES = new Set(["agy", "claude", "codex"]);
 // opencode's plugin is a managed artifact, not a ManagedHookRecord entry.
 const HOOK_HARNESS_VALUES = new Set(["claude", "codex"]);
 const HOOK_EVENT_VALUES = new Set([
     "SessionStart",
+    "UserPromptSubmit",
     "PreToolUse",
+    "PermissionRequest",
+    "PostToolUse",
+    "PreCompact",
+    "PostCompact",
+    "SubagentStart",
+    "SubagentStop",
     "Stop"
 ]);
 const MAX_TIMEOUT_MS = 2_147_483_647;
@@ -281,6 +297,7 @@ export function validateConfig(value) {
         "features",
         "pathMappings",
         "profiles",
+        "reviewRoles",
         "schemaVersion",
         "securityExceptions",
         "verification"
@@ -368,8 +385,60 @@ export function validateConfig(value) {
             return exception;
         }
     }
+    if (root.reviewRoles !== undefined) {
+        if (!Array.isArray(root.reviewRoles)) {
+            return failure("INVALID_TYPE", "$.reviewRoles", "reviewRoles must be an array.");
+        }
+        const roles = new Set();
+        for (const [index, roleValue] of root.reviewRoles.entries()) {
+            const role = validateReviewRole(roleValue, `$.reviewRoles[${index}]`);
+            if (!role.ok) {
+                return role;
+            }
+            if (roles.has(role.value.role)) {
+                return failure("DUPLICATE_ID", "$.reviewRoles", `Duplicate review role: ${role.value.role}`);
+            }
+            roles.add(role.value.role);
+        }
+    }
     return success(root);
 }
+function validateReviewRole(value, path) {
+    if (!isRecord(value)) {
+        return failure("INVALID_TYPE", path, "Expected a review role object.");
+    }
+    const unknown = unknownFieldFailure(value, ["effort", "model", "role", "targets", "timeoutMs"], path);
+    if (unknown !== undefined) {
+        return unknown;
+    }
+    if (typeof value.role !== "string" || !REVIEW_ROLE_VALUES.has(value.role)) {
+        return failure("INVALID_REVIEW_ROLE", `${path}.role`, `Unsupported review role: ${String(value.role)}`);
+    }
+    if (!Array.isArray(value.targets) || value.targets.length === 0) {
+        return failure("INVALID_REVIEW_TARGET", `${path}.targets`, "targets must list at least one review target.");
+    }
+    for (const [index, target] of value.targets.entries()) {
+        if (typeof target !== "string" || !REVIEW_TARGET_VALUES.has(target)) {
+            return failure("INVALID_REVIEW_TARGET", `${path}.targets[${index}]`, `Unsupported review target: ${String(target)}`);
+        }
+    }
+    if (!hasUniqueStrings(value.targets)) {
+        return failure("DUPLICATE_ID", `${path}.targets`, "Review targets must be unique.");
+    }
+    if (value.model !== undefined && !isNonEmptyString(value.model)) {
+        return failure("INVALID_TYPE", `${path}.model`, "model must be a non-empty string.");
+    }
+    if (value.effort !== undefined && !isNonEmptyString(value.effort)) {
+        return failure("INVALID_TYPE", `${path}.effort`, "effort must be a non-empty string.");
+    }
+    if (value.timeoutMs !== undefined &&
+        (!Number.isSafeInteger(value.timeoutMs) ||
+            value.timeoutMs <= 0 ||
+            value.timeoutMs > MAX_TIMEOUT_MS)) {
+        return failure("INVALID_TIMEOUT", `${path}.timeoutMs`, "timeoutMs must be a positive integer.");
+    }
+    return success(value);
+}
 function validateCriterion(value, path) {
     if (!isRecord(value)) {
         return failure("INVALID_TYPE", path, "Expected a criterion object.");

package/dist/runtime/src/task/service.js

@@ -181,6 +181,46 @@ export class TaskService {
             return cloneRecord(completed);
         });
     }
+    /**
+     * Append evidence for some criteria without completing the task. Unlike
+     * `complete`, the input may be partial — an independent review covers the
+     * criteria it was asked about, not necessarily all of them. Only an active
+     * task accepts evidence: a completed record must stay exactly as it was
+     * verified.
+     */
+    async recordEvidence(taskId, evidenceInput) {
+        const now = assertTimestamp(this.#now());
+        return await this.#store.mutate((state) => {
+            const current = findTask(state, taskId);
+            if (current.status !== "active") {
+                throw taskError("TASK_NOT_ACTIVE", "Only an active task can record additional evidence.");
+            }
+            const criterionIds = new Set(current.task.criteria.map((criterion) => criterion.id));
+            const evidence = Object.fromEntries(Object.entries(current.evidence).map(([criterionId, references]) => [
+                criterionId,
+                [...references]
+            ]));
+            for (const [criterionId, references] of Object.entries(evidenceInput)) {
+                if (!criterionIds.has(criterionId)) {
+                    throw taskError("TASK_EVIDENCE_UNKNOWN_CRITERION", `Unknown criterion: ${criterionId}`);
+                }
+                if (references.length === 0 ||
+                    references.some((reference) => typeof reference !== "string" || reference.trim().length === 0)) {
+                    throw taskError("TASK_EVIDENCE_INVALID", `Evidence for ${criterionId} must be non-empty references.`);
+                }
+                evidence[criterionId] = [
+                    ...new Set([...(evidence[criterionId] ?? []), ...references])
+                ];
+            }
+            const updated = {
+                ...current,
+                evidence,
+                updatedAt: now
+            };
+            replaceTask(state, updated);
+            return cloneRecord(updated);
+        });
+    }
     async archive(taskId) {
         const now = assertTimestamp(this.#now());
         return await this.#store.mutate((state) => {