@kylecheng3146/agent-ops 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/packages/cli/src/args.js +32 -0
- package/dist/packages/cli/src/bin.js +38 -3
- package/dist/packages/cli/src/cli.js +12 -1
- package/dist/packages/cli/src/commands/init.js +4 -1
- package/dist/packages/cli/src/commands/review.js +97 -10
- package/dist/packages/cli/src/version.js +1 -1
- package/dist/packages/cli/src/wizard.js +62 -3
- package/dist/runtime/src/config/merge.js +17 -2
- package/dist/runtime/src/install/doctor.js +42 -1
- package/dist/runtime/src/install/plan.js +11 -5
- package/dist/runtime/src/review/execute.js +120 -0
- package/dist/runtime/src/review/extract.js +71 -0
- package/dist/runtime/src/review/invocation.js +52 -0
- package/dist/runtime/src/review/probe.js +48 -0
- package/dist/runtime/src/review/result.js +2 -2
- package/dist/runtime/src/review/roles.js +35 -0
- package/dist/runtime/src/review/runner.js +38 -4
- package/dist/runtime/src/schema/validate.js +62 -0
- package/dist/runtime/src/task/service.js +40 -0
- package/docs/en/guides/configuration.md +60 -0
- package/docs/en/spec/review.md +37 -4
- package/docs/zh-TW/guides/configuration.md +53 -0
- package/docs/zh-TW/spec/review.md +33 -3
- package/package.json +1 -1
- package/schemas/config.schema.json +29 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { runVerificationCommand } from "../verify/spawn.js";
|
|
2
|
+
import { extractFinalMessage, extractJsonObject } from "./extract.js";
|
|
3
|
+
import { buildTargetInvocation } from "./invocation.js";
|
|
4
|
+
import { detectHostTarget, orderChain } from "./roles.js";
|
|
5
|
+
import { buildReviewPrompt } from "./runner.js";
|
|
6
|
+
/**
|
|
7
|
+
* Deliberately below the five-minute `spawn.ts` default: a timeout advances the
|
|
8
|
+
* chain, so the worst case is targets x timeout.
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULT_REVIEW_TIMEOUT_MS = 120_000;
|
|
11
|
+
/**
|
|
12
|
+
* Failure classes that mean no review happened, so trying the next target is
|
|
13
|
+
* not review shopping. Everything else — including FAIL — is terminal.
|
|
14
|
+
*/
|
|
15
|
+
const ADVANCING = new Set([
|
|
16
|
+
"missing-executable",
|
|
17
|
+
"spawn-failed",
|
|
18
|
+
"timeout"
|
|
19
|
+
]);
|
|
20
|
+
function statusOf(value) {
|
|
21
|
+
return value === "PASS" || value === "FAIL" ? value : undefined;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The response must name every requested criterion exactly once, with at least
|
|
25
|
+
* one non-blank evidence reference. A response that breaks the contract is
|
|
26
|
+
* unparseable output, never a FAIL verdict: FAIL has to keep meaning "the
|
|
27
|
+
* reviewer looked and judged it inadequate".
|
|
28
|
+
*/
|
|
29
|
+
function parseResults(payload, expected) {
|
|
30
|
+
const raw = payload.results;
|
|
31
|
+
if (!Array.isArray(raw) || raw.length !== expected.length) {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
const results = [];
|
|
35
|
+
const seen = new Set();
|
|
36
|
+
for (const entry of raw) {
|
|
37
|
+
if (typeof entry !== "object" || entry === null) {
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
const item = entry;
|
|
41
|
+
const criterionId = item.criterionId;
|
|
42
|
+
const status = statusOf(item.status);
|
|
43
|
+
if (typeof criterionId !== "string" ||
|
|
44
|
+
status === undefined ||
|
|
45
|
+
!expected.includes(criterionId) ||
|
|
46
|
+
seen.has(criterionId) ||
|
|
47
|
+
!Array.isArray(item.evidence) ||
|
|
48
|
+
item.evidence.length === 0 ||
|
|
49
|
+
!item.evidence.every((reference) => typeof reference === "string" && reference.trim().length > 0)) {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
seen.add(criterionId);
|
|
53
|
+
results.push({
|
|
54
|
+
criterionId,
|
|
55
|
+
status,
|
|
56
|
+
evidence: item.evidence.map((reference) => String(reference))
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return results;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Builds the `execute` callback `runIndependentReview` expects: walk the
|
|
63
|
+
* configured targets in order and return the first real verdict.
|
|
64
|
+
*/
|
|
65
|
+
export function createReviewExecutor(options) {
|
|
66
|
+
const report = options.onProgress ?? (() => { });
|
|
67
|
+
const host = detectHostTarget(options.env ?? process.env);
|
|
68
|
+
const chain = orderChain(options.targets, host);
|
|
69
|
+
return async (request) => {
|
|
70
|
+
const expected = request.invocation.packet.criteria.map((criterion) => criterion.id);
|
|
71
|
+
const prompt = buildReviewPrompt(request.invocation);
|
|
72
|
+
for (const [index, target] of chain.entries()) {
|
|
73
|
+
const invocation = buildTargetInvocation({
|
|
74
|
+
target,
|
|
75
|
+
prompt,
|
|
76
|
+
...(options.model === undefined ? {} : { model: options.model }),
|
|
77
|
+
...(options.effort === undefined ? {} : { effort: options.effort })
|
|
78
|
+
});
|
|
79
|
+
if (invocation === undefined) {
|
|
80
|
+
report(`${target}: no read-only mode available → skipping`);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (target === host) {
|
|
84
|
+
report(`${target}: reviewer == host; no independent target configured`);
|
|
85
|
+
}
|
|
86
|
+
const spawned = await runVerificationCommand({
|
|
87
|
+
id: `review-${target}-${index}`,
|
|
88
|
+
command: invocation.command,
|
|
89
|
+
args: [...invocation.args],
|
|
90
|
+
cwd: options.cwd,
|
|
91
|
+
required: true,
|
|
92
|
+
evidence: { kind: "exit-code" },
|
|
93
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
|
|
94
|
+
}, {
|
|
95
|
+
cwd: options.cwd,
|
|
96
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
97
|
+
...(options.outputLimitBytes === undefined
|
|
98
|
+
? {}
|
|
99
|
+
: { outputLimitBytes: options.outputLimitBytes })
|
|
100
|
+
});
|
|
101
|
+
if (ADVANCING.has(spawned.failureClass)) {
|
|
102
|
+
report(`${target}: ${spawned.failureClass} → trying next target`);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (spawned.stdoutTruncated) {
|
|
106
|
+
return { status: "NOT_RUN", reason: "unparseable-output" };
|
|
107
|
+
}
|
|
108
|
+
const message = extractFinalMessage(target, spawned.stdout);
|
|
109
|
+
const payload = message === undefined ? undefined : extractJsonObject(message);
|
|
110
|
+
const results = payload === undefined ? undefined : parseResults(payload, expected);
|
|
111
|
+
if (results === undefined) {
|
|
112
|
+
return { status: "NOT_RUN", reason: "unparseable-output" };
|
|
113
|
+
}
|
|
114
|
+
return results.every((result) => result.status === "PASS")
|
|
115
|
+
? { status: "PASS", results }
|
|
116
|
+
: { status: "FAIL", results };
|
|
117
|
+
}
|
|
118
|
+
return { status: "NOT_RUN", reason: "missing-cli" };
|
|
119
|
+
};
|
|
120
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The key holding the model's answer differs in every envelope, so every
|
|
3
|
+
* target gets its own branch. Verified against tests/fixtures/review/.
|
|
4
|
+
*/
|
|
5
|
+
const ENVELOPE_KEYS = {
|
|
6
|
+
agy: "response",
|
|
7
|
+
claude: "result"
|
|
8
|
+
};
|
|
9
|
+
function parseObject(text) {
|
|
10
|
+
let parsed;
|
|
11
|
+
try {
|
|
12
|
+
parsed = JSON.parse(text);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
|
18
|
+
? parsed
|
|
19
|
+
: undefined;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The model's answer as text, before any JSON contract is applied. Returns
|
|
23
|
+
* undefined rather than throwing so the caller can report
|
|
24
|
+
* `unparseable-output` for every transport failure through one path.
|
|
25
|
+
*/
|
|
26
|
+
export function extractFinalMessage(target, stdout) {
|
|
27
|
+
const key = ENVELOPE_KEYS[target];
|
|
28
|
+
if (key === undefined) {
|
|
29
|
+
// codex: stdout is the final message itself.
|
|
30
|
+
const trimmed = stdout.trim();
|
|
31
|
+
return trimmed.length === 0 ? undefined : trimmed;
|
|
32
|
+
}
|
|
33
|
+
const envelope = parseObject(stdout);
|
|
34
|
+
const value = envelope?.[key];
|
|
35
|
+
if (typeof value !== "string") {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
const trimmed = value.trim();
|
|
39
|
+
return trimmed.length === 0 ? undefined : trimmed;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The last balanced JSON object in a block of model text. Scanning backwards
|
|
43
|
+
* matters: models often restate the schema before answering, and the answer is
|
|
44
|
+
* what comes last. This only ever runs on the extracted final message, never on
|
|
45
|
+
* raw stdout, so it cannot capture a transport envelope.
|
|
46
|
+
*/
|
|
47
|
+
export function extractJsonObject(text) {
|
|
48
|
+
for (let end = text.lastIndexOf("}"); end !== -1; end = text.lastIndexOf("}", end - 1)) {
|
|
49
|
+
let depth = 0;
|
|
50
|
+
for (let start = end; start >= 0; start -= 1) {
|
|
51
|
+
const character = text[start];
|
|
52
|
+
if (character === "}") {
|
|
53
|
+
depth += 1;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (character !== "{") {
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
depth -= 1;
|
|
60
|
+
if (depth !== 0) {
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const candidate = parseObject(text.slice(start, end + 1));
|
|
64
|
+
if (candidate !== undefined) {
|
|
65
|
+
return candidate;
|
|
66
|
+
}
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only enforcement per target, verified against each CLI's own help
|
|
3
|
+
* output. A target absent from this table is ineligible: review never runs an
|
|
4
|
+
* agent that can edit the code it is reviewing. This is what excludes
|
|
5
|
+
* opencode, whose `--agent plan` is rejected as a subagent and silently falls
|
|
6
|
+
* back to a writable agent.
|
|
7
|
+
*/
|
|
8
|
+
export const READ_ONLY_ARGS = {
|
|
9
|
+
agy: ["--sandbox", "--mode", "plan"],
|
|
10
|
+
claude: ["--permission-mode", "plan"],
|
|
11
|
+
codex: ["-s", "read-only"]
|
|
12
|
+
};
|
|
13
|
+
function modelArgs(target, model) {
|
|
14
|
+
if (model === undefined) {
|
|
15
|
+
return [];
|
|
16
|
+
}
|
|
17
|
+
return target === "codex" ? ["-m", model] : ["--model", model];
|
|
18
|
+
}
|
|
19
|
+
function effortArgs(target, effort) {
|
|
20
|
+
if (effort === undefined) {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
// codex has no --effort flag; reasoning effort is a config override.
|
|
24
|
+
return target === "codex"
|
|
25
|
+
? ["-c", `model_reasoning_effort=${effort}`]
|
|
26
|
+
: ["--effort", effort];
|
|
27
|
+
}
|
|
28
|
+
export function buildTargetInvocation(request) {
|
|
29
|
+
const readOnly = READ_ONLY_ARGS[request.target];
|
|
30
|
+
if (readOnly === undefined || readOnly.length === 0) {
|
|
31
|
+
return undefined;
|
|
32
|
+
}
|
|
33
|
+
const shared = [
|
|
34
|
+
...readOnly,
|
|
35
|
+
...modelArgs(request.target, request.model),
|
|
36
|
+
...effortArgs(request.target, request.effort)
|
|
37
|
+
];
|
|
38
|
+
if (request.target === "codex") {
|
|
39
|
+
// codex writes progress to stderr and leaves stdout as the bare final
|
|
40
|
+
// message, so it needs no output-format flag and no scratch file.
|
|
41
|
+
// Without --skip-git-repo-check it refuses to run outside a trusted git
|
|
42
|
+
// directory, which a caller would otherwise read as "not authenticated".
|
|
43
|
+
return {
|
|
44
|
+
command: "codex",
|
|
45
|
+
args: ["exec", request.prompt, "--skip-git-repo-check", ...shared]
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
return {
|
|
49
|
+
command: request.target,
|
|
50
|
+
args: ["-p", request.prompt, "--output-format", "json", ...shared]
|
|
51
|
+
};
|
|
52
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { runVerificationCommand } from "../verify/spawn.js";
|
|
2
|
+
import { extractFinalMessage } from "./extract.js";
|
|
3
|
+
import { buildTargetInvocation } from "./invocation.js";
|
|
4
|
+
const PROBE_PROMPT = "Reply with the single word OK and nothing else.";
|
|
5
|
+
/**
|
|
6
|
+
* Matches the review timeout rather than being "quick": codex at high
|
|
7
|
+
* reasoning effort answers a trivial prompt in ~20s, and a probe that times out
|
|
8
|
+
* would otherwise be reported as an authentication failure.
|
|
9
|
+
*/
|
|
10
|
+
const PROBE_TIMEOUT_MS = 120_000;
|
|
11
|
+
/**
|
|
12
|
+
* The only check that actually proves a target is usable: ask it something
|
|
13
|
+
* trivial and see whether an answer comes back. A credential-file check can
|
|
14
|
+
* pass while the token is expired, and self-declaration ("already logged in?")
|
|
15
|
+
* is not evidence at all.
|
|
16
|
+
*/
|
|
17
|
+
export async function probeReviewTarget(target, options) {
|
|
18
|
+
const invocation = buildTargetInvocation({ target, prompt: PROBE_PROMPT });
|
|
19
|
+
if (invocation === undefined) {
|
|
20
|
+
return "ineligible";
|
|
21
|
+
}
|
|
22
|
+
const deep = options.deep === true;
|
|
23
|
+
const spawned = await runVerificationCommand({
|
|
24
|
+
id: `review-probe-${target}`,
|
|
25
|
+
command: invocation.command,
|
|
26
|
+
args: deep ? [...invocation.args] : ["--version"],
|
|
27
|
+
cwd: options.cwd,
|
|
28
|
+
required: true,
|
|
29
|
+
evidence: { kind: "exit-code" },
|
|
30
|
+
timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS
|
|
31
|
+
}, {
|
|
32
|
+
cwd: options.cwd,
|
|
33
|
+
...(options.runner === undefined ? {} : { runner: options.runner })
|
|
34
|
+
});
|
|
35
|
+
if (spawned.failureClass === "missing-executable") {
|
|
36
|
+
return "missing-executable";
|
|
37
|
+
}
|
|
38
|
+
if (spawned.timedOut) {
|
|
39
|
+
return "timeout";
|
|
40
|
+
}
|
|
41
|
+
if (!deep) {
|
|
42
|
+
return spawned.status === "PASS" ? "ok" : "unauthenticated";
|
|
43
|
+
}
|
|
44
|
+
return spawned.status === "PASS" &&
|
|
45
|
+
extractFinalMessage(target, spawned.stdout) !== undefined
|
|
46
|
+
? "ok"
|
|
47
|
+
: "unauthenticated";
|
|
48
|
+
}
|
|
@@ -14,10 +14,10 @@ export function aggregateReviewResults(requestedCriterionIds, results) {
|
|
|
14
14
|
if (seen.size !== expected.size) {
|
|
15
15
|
valid = false;
|
|
16
16
|
}
|
|
17
|
-
const status =
|
|
17
|
+
const status = results.every((result) => result.status === "PASS")
|
|
18
18
|
? "PASS"
|
|
19
19
|
: "FAIL";
|
|
20
|
-
return { status, results: [...results] };
|
|
20
|
+
return { status, results: [...results], valid };
|
|
21
21
|
}
|
|
22
22
|
export function summarizeReview(request) {
|
|
23
23
|
return aggregateReviewResults(request.packet.criteria.map((criterion) => criterion.id), request.criterionResults);
|
|
@@ -1,3 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default chain order. codex first because its stdout is the bare final message
|
|
3
|
+
* (nothing to unwrap), then agy's flat envelope. claude is last because it is
|
|
4
|
+
* the only host we can detect, and `orderChain` would push it back anyway.
|
|
5
|
+
*/
|
|
6
|
+
export const DEFAULT_REVIEW_TARGETS = [
|
|
7
|
+
"codex",
|
|
8
|
+
"agy",
|
|
9
|
+
"claude"
|
|
10
|
+
];
|
|
1
11
|
export function resolveReviewRole(role, configured) {
|
|
2
12
|
return configured.find((item) => item.role === role);
|
|
3
13
|
}
|
|
14
|
+
export function reviewTargets(config, role) {
|
|
15
|
+
return resolveReviewRole(role, config.reviewRoles ?? [])?.targets ?? [];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Which review target is hosting this process, when that is knowable. Only
|
|
19
|
+
* Claude Code publishes a documented marker; guessing the others would produce
|
|
20
|
+
* a detector that silently fails, which is worse than no detector.
|
|
21
|
+
*/
|
|
22
|
+
export function detectHostTarget(env) {
|
|
23
|
+
return env.CLAUDECODE === undefined ? undefined : "claude";
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Move the hosting target to the end so an independent reviewer is preferred,
|
|
27
|
+
* without ever dropping it — a single configured target still runs, self-review
|
|
28
|
+
* warning and all.
|
|
29
|
+
*/
|
|
30
|
+
export function orderChain(targets, host) {
|
|
31
|
+
if (host === undefined) {
|
|
32
|
+
return [...targets];
|
|
33
|
+
}
|
|
34
|
+
return [
|
|
35
|
+
...targets.filter((target) => target !== host),
|
|
36
|
+
...targets.filter((target) => target === host)
|
|
37
|
+
];
|
|
38
|
+
}
|
|
@@ -1,12 +1,43 @@
|
|
|
1
1
|
import { aggregateReviewResults } from "./result.js";
|
|
2
2
|
import { redactSecrets } from "../security/redact.js";
|
|
3
3
|
import { safeTaskText } from "../task/render.js";
|
|
4
|
-
function
|
|
4
|
+
function criterionLine(criterion) {
|
|
5
|
+
const verified = criterion.verifierIds ?? [];
|
|
6
|
+
const covered = verified.length === 0
|
|
7
|
+
? ""
|
|
8
|
+
: ` (already machine-verified by: ${verified.join(", ")} —` +
|
|
9
|
+
" do not re-run those checks)";
|
|
10
|
+
return `- ${criterion.id}: ${criterion.description}${covered}`;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The prompt the reviewing CLI actually receives. It stays short on purpose: it
|
|
14
|
+
* travels through argv, so an embedded diff would risk ARG_MAX and would expose
|
|
15
|
+
* the diff in `ps` output. The target inspects the repository itself instead,
|
|
16
|
+
* which its read-only sandbox permits.
|
|
17
|
+
*/
|
|
18
|
+
export function buildReviewPrompt(invocation) {
|
|
19
|
+
const ids = invocation.packet.criteria.map((criterion) => criterion.id);
|
|
20
|
+
const shape = ids
|
|
21
|
+
.map((id) => `{"criterionId":"${id}","status":"PASS|FAIL","evidence":["<reference>"]}`)
|
|
22
|
+
.join(",");
|
|
5
23
|
return [
|
|
6
|
-
|
|
24
|
+
invocation.packet.request,
|
|
25
|
+
"",
|
|
26
|
+
"You are a read-only reviewer. Inspect this repository yourself " +
|
|
27
|
+
"(git diff, git log, reading files); do not modify anything.",
|
|
7
28
|
`Harness: ${invocation.harness}; model: ${invocation.model}; effort: ${invocation.effort}.`,
|
|
8
29
|
`Artifacts: ${invocation.packet.artifactRefs.join(", ") || "none"}.`,
|
|
9
|
-
|
|
30
|
+
"",
|
|
31
|
+
"Criteria:",
|
|
32
|
+
...(invocation.packet.criteria.length === 0
|
|
33
|
+
? ["- none"]
|
|
34
|
+
: invocation.packet.criteria.map(criterionLine)),
|
|
35
|
+
...invocation.packet.evidenceRequirements.map((requirement) => `- evidence for ${requirement.criterionId}: ${requirement.requirement}`),
|
|
36
|
+
"",
|
|
37
|
+
"Reply with exactly one JSON object and nothing else. Name every " +
|
|
38
|
+
"criterion above exactly once, each with at least one non-empty " +
|
|
39
|
+
"evidence reference:",
|
|
40
|
+
`{"results":[${shape}]}`
|
|
10
41
|
].join("\n");
|
|
11
42
|
}
|
|
12
43
|
function safeResult(result) {
|
|
@@ -21,7 +52,7 @@ export async function runIndependentReview(options) {
|
|
|
21
52
|
harness: options.invocation.harness,
|
|
22
53
|
model: options.invocation.model,
|
|
23
54
|
effort: options.invocation.effort,
|
|
24
|
-
prompt:
|
|
55
|
+
prompt: buildReviewPrompt(options.invocation)
|
|
25
56
|
};
|
|
26
57
|
if (!options.authorized) {
|
|
27
58
|
return { ...base, status: "NOT_RUN", reason: "authorization-required" };
|
|
@@ -37,6 +68,9 @@ export async function runIndependentReview(options) {
|
|
|
37
68
|
return { ...base, status: result.status, results: result.results };
|
|
38
69
|
}
|
|
39
70
|
const summary = aggregateReviewResults(options.invocation.packet.criteria.map((criterion) => criterion.id), result.results);
|
|
71
|
+
if (!summary.valid) {
|
|
72
|
+
return { ...base, status: "NOT_RUN", reason: "unparseable-output" };
|
|
73
|
+
}
|
|
40
74
|
return {
|
|
41
75
|
...base,
|
|
42
76
|
status: summary.status,
|
|
@@ -6,6 +6,15 @@ const PROFILE_VALUES = new Set(["advisory", "core", "guardrails", "loop"]);
|
|
|
6
6
|
const EVIDENCE_KINDS = new Set(["exit-code", "file", "test-count"]);
|
|
7
7
|
const SCOPE_VALUES = new Set(["project", "user"]);
|
|
8
8
|
const HARNESS_VALUES = new Set(["claude", "codex", "opencode"]);
|
|
9
|
+
const REVIEW_ROLE_VALUES = new Set([
|
|
10
|
+
"deep-reasoning",
|
|
11
|
+
"implementation",
|
|
12
|
+
"independent-review",
|
|
13
|
+
"mechanical"
|
|
14
|
+
]);
|
|
15
|
+
// opencode is absent by design: it has no read-only flag. See
|
|
16
|
+
// docs/plans/2026-08-12-external-review-cli-targets.md.
|
|
17
|
+
const REVIEW_TARGET_VALUES = new Set(["agy", "claude", "codex"]);
|
|
9
18
|
// opencode's plugin is a managed artifact, not a ManagedHookRecord entry.
|
|
10
19
|
const HOOK_HARNESS_VALUES = new Set(["claude", "codex"]);
|
|
11
20
|
const HOOK_EVENT_VALUES = new Set([
|
|
@@ -288,6 +297,7 @@ export function validateConfig(value) {
|
|
|
288
297
|
"features",
|
|
289
298
|
"pathMappings",
|
|
290
299
|
"profiles",
|
|
300
|
+
"reviewRoles",
|
|
291
301
|
"schemaVersion",
|
|
292
302
|
"securityExceptions",
|
|
293
303
|
"verification"
|
|
@@ -375,8 +385,60 @@ export function validateConfig(value) {
|
|
|
375
385
|
return exception;
|
|
376
386
|
}
|
|
377
387
|
}
|
|
388
|
+
if (root.reviewRoles !== undefined) {
|
|
389
|
+
if (!Array.isArray(root.reviewRoles)) {
|
|
390
|
+
return failure("INVALID_TYPE", "$.reviewRoles", "reviewRoles must be an array.");
|
|
391
|
+
}
|
|
392
|
+
const roles = new Set();
|
|
393
|
+
for (const [index, roleValue] of root.reviewRoles.entries()) {
|
|
394
|
+
const role = validateReviewRole(roleValue, `$.reviewRoles[${index}]`);
|
|
395
|
+
if (!role.ok) {
|
|
396
|
+
return role;
|
|
397
|
+
}
|
|
398
|
+
if (roles.has(role.value.role)) {
|
|
399
|
+
return failure("DUPLICATE_ID", "$.reviewRoles", `Duplicate review role: ${role.value.role}`);
|
|
400
|
+
}
|
|
401
|
+
roles.add(role.value.role);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
378
404
|
return success(root);
|
|
379
405
|
}
|
|
406
|
+
function validateReviewRole(value, path) {
|
|
407
|
+
if (!isRecord(value)) {
|
|
408
|
+
return failure("INVALID_TYPE", path, "Expected a review role object.");
|
|
409
|
+
}
|
|
410
|
+
const unknown = unknownFieldFailure(value, ["effort", "model", "role", "targets", "timeoutMs"], path);
|
|
411
|
+
if (unknown !== undefined) {
|
|
412
|
+
return unknown;
|
|
413
|
+
}
|
|
414
|
+
if (typeof value.role !== "string" || !REVIEW_ROLE_VALUES.has(value.role)) {
|
|
415
|
+
return failure("INVALID_REVIEW_ROLE", `${path}.role`, `Unsupported review role: ${String(value.role)}`);
|
|
416
|
+
}
|
|
417
|
+
if (!Array.isArray(value.targets) || value.targets.length === 0) {
|
|
418
|
+
return failure("INVALID_REVIEW_TARGET", `${path}.targets`, "targets must list at least one review target.");
|
|
419
|
+
}
|
|
420
|
+
for (const [index, target] of value.targets.entries()) {
|
|
421
|
+
if (typeof target !== "string" || !REVIEW_TARGET_VALUES.has(target)) {
|
|
422
|
+
return failure("INVALID_REVIEW_TARGET", `${path}.targets[${index}]`, `Unsupported review target: ${String(target)}`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
if (!hasUniqueStrings(value.targets)) {
|
|
426
|
+
return failure("DUPLICATE_ID", `${path}.targets`, "Review targets must be unique.");
|
|
427
|
+
}
|
|
428
|
+
if (value.model !== undefined && !isNonEmptyString(value.model)) {
|
|
429
|
+
return failure("INVALID_TYPE", `${path}.model`, "model must be a non-empty string.");
|
|
430
|
+
}
|
|
431
|
+
if (value.effort !== undefined && !isNonEmptyString(value.effort)) {
|
|
432
|
+
return failure("INVALID_TYPE", `${path}.effort`, "effort must be a non-empty string.");
|
|
433
|
+
}
|
|
434
|
+
if (value.timeoutMs !== undefined &&
|
|
435
|
+
(!Number.isSafeInteger(value.timeoutMs) ||
|
|
436
|
+
value.timeoutMs <= 0 ||
|
|
437
|
+
value.timeoutMs > MAX_TIMEOUT_MS)) {
|
|
438
|
+
return failure("INVALID_TIMEOUT", `${path}.timeoutMs`, "timeoutMs must be a positive integer.");
|
|
439
|
+
}
|
|
440
|
+
return success(value);
|
|
441
|
+
}
|
|
380
442
|
function validateCriterion(value, path) {
|
|
381
443
|
if (!isRecord(value)) {
|
|
382
444
|
return failure("INVALID_TYPE", path, "Expected a criterion object.");
|
|
@@ -181,6 +181,46 @@ export class TaskService {
|
|
|
181
181
|
return cloneRecord(completed);
|
|
182
182
|
});
|
|
183
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Append evidence for some criteria without completing the task. Unlike
|
|
186
|
+
* `complete`, the input may be partial — an independent review covers the
|
|
187
|
+
* criteria it was asked about, not necessarily all of them. Only an active
|
|
188
|
+
* task accepts evidence: a completed record must stay exactly as it was
|
|
189
|
+
* verified.
|
|
190
|
+
*/
|
|
191
|
+
async recordEvidence(taskId, evidenceInput) {
|
|
192
|
+
const now = assertTimestamp(this.#now());
|
|
193
|
+
return await this.#store.mutate((state) => {
|
|
194
|
+
const current = findTask(state, taskId);
|
|
195
|
+
if (current.status !== "active") {
|
|
196
|
+
throw taskError("TASK_NOT_ACTIVE", "Only an active task can record additional evidence.");
|
|
197
|
+
}
|
|
198
|
+
const criterionIds = new Set(current.task.criteria.map((criterion) => criterion.id));
|
|
199
|
+
const evidence = Object.fromEntries(Object.entries(current.evidence).map(([criterionId, references]) => [
|
|
200
|
+
criterionId,
|
|
201
|
+
[...references]
|
|
202
|
+
]));
|
|
203
|
+
for (const [criterionId, references] of Object.entries(evidenceInput)) {
|
|
204
|
+
if (!criterionIds.has(criterionId)) {
|
|
205
|
+
throw taskError("TASK_EVIDENCE_UNKNOWN_CRITERION", `Unknown criterion: ${criterionId}`);
|
|
206
|
+
}
|
|
207
|
+
if (references.length === 0 ||
|
|
208
|
+
references.some((reference) => typeof reference !== "string" || reference.trim().length === 0)) {
|
|
209
|
+
throw taskError("TASK_EVIDENCE_INVALID", `Evidence for ${criterionId} must be non-empty references.`);
|
|
210
|
+
}
|
|
211
|
+
evidence[criterionId] = [
|
|
212
|
+
...new Set([...(evidence[criterionId] ?? []), ...references])
|
|
213
|
+
];
|
|
214
|
+
}
|
|
215
|
+
const updated = {
|
|
216
|
+
...current,
|
|
217
|
+
evidence,
|
|
218
|
+
updatedAt: now
|
|
219
|
+
};
|
|
220
|
+
replaceTask(state, updated);
|
|
221
|
+
return cloneRecord(updated);
|
|
222
|
+
});
|
|
223
|
+
}
|
|
184
224
|
async archive(taskId) {
|
|
185
225
|
const now = assertTimestamp(this.#now());
|
|
186
226
|
return await this.#store.mutate((state) => {
|
|
@@ -31,6 +31,66 @@ implies them. Advisory runs through the real SessionStart path and is
|
|
|
31
31
|
fail-open. Claude and Codex lifecycle support is `supported`; OpenCode begins
|
|
32
32
|
at app initialization and is honestly reported as `degraded`.
|
|
33
33
|
|
|
34
|
+
### External review targets
|
|
35
|
+
|
|
36
|
+
`agent-ops review` can call another agent CLI to review your work. It is
|
|
37
|
+
disabled by default: an absent `reviewRoles` field, an absent
|
|
38
|
+
`--review-target` flag, and the interactive question's default all mean off.
|
|
39
|
+
Enable it during `agent-ops init`, or by hand:
|
|
40
|
+
|
|
41
|
+
```json
|
|
42
|
+
{
|
|
43
|
+
"reviewRoles": [
|
|
44
|
+
{ "role": "independent-review", "targets": ["codex", "agy"] }
|
|
45
|
+
]
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
`targets` is an **ordered fallback chain**. Supported targets and the read-only
|
|
50
|
+
flags they are launched with:
|
|
51
|
+
|
|
52
|
+
| Target | Invocation | Read-only |
|
|
53
|
+
| --- | --- | --- |
|
|
54
|
+
| `codex` | `codex exec` | `-s read-only` |
|
|
55
|
+
| `agy` (Antigravity) | `agy -p` | `--sandbox --mode plan` |
|
|
56
|
+
| `claude` | `claude -p` | `--permission-mode plan` |
|
|
57
|
+
|
|
58
|
+
`opencode` is **not** a review target even though it is a supported harness.
|
|
59
|
+
Its `--agent plan` is rejected as a subagent and silently falls back to a
|
|
60
|
+
writable agent, so it cannot satisfy the read-only precondition. A target with
|
|
61
|
+
no read-only flag is skipped rather than run unsandboxed.
|
|
62
|
+
|
|
63
|
+
The chain advances only when no review happened — the executable is missing,
|
|
64
|
+
the spawn failed, or the attempt timed out (120s per target by default,
|
|
65
|
+
overridable with `timeoutMs`). A `FAIL` verdict is **terminal**: the chain
|
|
66
|
+
never retries another target after a real verdict, because that would be
|
|
67
|
+
automated review shopping. Unparseable output is terminal too, since it points
|
|
68
|
+
at a prompt or CLI-version mismatch worth surfacing.
|
|
69
|
+
|
|
70
|
+
If Claude Code is the host (`CLAUDECODE` is set), `claude` is moved to the end
|
|
71
|
+
of the chain. It still runs when it is the only configured target, with a
|
|
72
|
+
`reviewer == host` warning.
|
|
73
|
+
|
|
74
|
+
Criterion descriptions come from the task bound to the current session, so a
|
|
75
|
+
review needs an attached task; `--criterion` filters those ids. Results are
|
|
76
|
+
appended to the task's evidence with a `review:<target>:` prefix, and only
|
|
77
|
+
while the task is active — a completed task is printed, never rewritten.
|
|
78
|
+
|
|
79
|
+
`--yes` is still required for every review run: init selection decides which
|
|
80
|
+
targets are permitted, `--yes` decides whether to spend money now.
|
|
81
|
+
|
|
82
|
+
Because target authentication is not sniffed from stderr, an unauthenticated
|
|
83
|
+
CLI surfaces as one review failure. Diagnose it with:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
agent-ops doctor # presence only: no tokens, no network
|
|
87
|
+
agent-ops doctor --check-auth # one real print call per target
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`--check-auth` is a dedicated flag; `--yes` stays inert for doctor. Doctor
|
|
91
|
+
reports what to do but never fixes it: every target authenticates through
|
|
92
|
+
interactive OAuth, so there is no `--fix`. Run `<target> login` yourself.
|
|
93
|
+
|
|
34
94
|
### Project-local loop profile
|
|
35
95
|
|
|
36
96
|
`--profile loop` is an opt-in project-scope profile. Select `codex`, `claude`,
|
package/docs/en/spec/review.md
CHANGED
|
@@ -22,11 +22,44 @@ A review result MUST preserve PASS, FAIL, or NOT_RUN and MUST NOT convert NOT_RU
|
|
|
22
22
|
|
|
23
23
|
## REVIEW-HARNESS-001
|
|
24
24
|
|
|
25
|
-
A review invocation MUST resolve to exactly one concrete
|
|
26
|
-
installation supports multiple harnesses.
|
|
25
|
+
A review invocation MUST resolve to exactly one concrete review target, even
|
|
26
|
+
when an installation supports multiple harnesses.
|
|
27
27
|
|
|
28
28
|
- Trigger: Running `review` with a harness selection.
|
|
29
|
-
- Action: Select one of `codex`, `
|
|
29
|
+
- Action: Select one of `codex`, `agy`, or `claude`; keep multi-harness installation separate from review execution.
|
|
30
30
|
- Evidence: Argument parsing rejects `all`, `both`, and comma-separated multi-harness values for review.
|
|
31
|
-
- Positive: `review --harness
|
|
31
|
+
- Positive: `review --harness claude` resolves one target.
|
|
32
32
|
- Negative: `Run one review invocation against every installed harness implicitly.`
|
|
33
|
+
|
|
34
|
+
## REVIEW-READONLY-001
|
|
35
|
+
|
|
36
|
+
A review target MUST be launched with its own read-only mechanism, and a target
|
|
37
|
+
without one MUST be skipped rather than run unsandboxed.
|
|
38
|
+
|
|
39
|
+
- Trigger: Building a review invocation for a configured target.
|
|
40
|
+
- Action: Pass `-s read-only` (codex), `--sandbox --mode plan` (agy), or `--permission-mode plan` (claude); treat any other target as ineligible.
|
|
41
|
+
- Evidence: The spawned argv contains the target's read-only flags.
|
|
42
|
+
- Positive: `opencode is not a review target: --agent plan silently falls back to a writable agent.`
|
|
43
|
+
- Negative: `Trust the prompt to stop the reviewer from editing files.`
|
|
44
|
+
|
|
45
|
+
## REVIEW-CHAIN-001
|
|
46
|
+
|
|
47
|
+
Configured targets form an ordered fallback chain that MUST advance only when
|
|
48
|
+
no review happened, and MUST NOT advance past a verdict.
|
|
49
|
+
|
|
50
|
+
- Trigger: A configured target is missing, fails to spawn, or times out.
|
|
51
|
+
- Action: Try the next target; on PASS, FAIL, or unparseable output, stop and report that outcome.
|
|
52
|
+
- Evidence: The number of spawned attempts matches the failures that preceded the verdict.
|
|
53
|
+
- Positive: `codex FAIL is final; agy is never asked for a second opinion.`
|
|
54
|
+
- Negative: `Retry other targets after a FAIL until one reports PASS.`
|
|
55
|
+
|
|
56
|
+
## REVIEW-CONTRACT-001
|
|
57
|
+
|
|
58
|
+
A response that breaks the reply contract MUST be reported as NOT_RUN, not as
|
|
59
|
+
FAIL.
|
|
60
|
+
|
|
61
|
+
- Trigger: The reviewer omits, duplicates, or invents a criterion, or returns blank evidence.
|
|
62
|
+
- Action: Report `NOT_RUN` with reason `unparseable-output`, write no evidence, and keep FAIL for judged inadequacy.
|
|
63
|
+
- Evidence: The result reason distinguishes a protocol violation from a verdict.
|
|
64
|
+
- Positive: `NOT_RUN: unparseable-output; one criterion was missing.`
|
|
65
|
+
- Negative: `Record a failed review because the model's JSON was malformed.`
|