@kylecheng3146/agent-ops 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +91 -21
- package/README.md +39 -1
- package/dist/packages/cli/src/args.js +46 -6
- package/dist/packages/cli/src/bin.js +135 -70
- package/dist/packages/cli/src/cli.js +27 -3
- package/dist/packages/cli/src/commands/batch.js +106 -0
- package/dist/packages/cli/src/commands/review.js +22 -0
- package/dist/runtime/src/install/harness.js +1 -1
- package/dist/runtime/src/install/preauth.js +49 -6
- package/dist/runtime/src/review/batch-guard.js +11 -0
- package/dist/runtime/src/review/batch-select.js +37 -0
- package/dist/runtime/src/review/batch.js +202 -0
- package/dist/runtime/src/review/invocation.js +15 -2
- package/dist/runtime/src/review/probe.js +13 -0
- package/docs/en/guides/configuration.md +6 -3
- package/docs/en/spec/review.md +70 -0
- package/docs/zh-TW/guides/configuration.md +8 -5
- package/docs/zh-TW/spec/review.md +71 -1
- package/package.json +2 -2
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { AgentOpsError } from "../../../../runtime/src/fs/paths.js";
|
|
2
|
+
import { DEFAULT_BATCH_WIDTH, runBatch } from "../../../../runtime/src/review/batch.js";
|
|
3
|
+
import { resolveBatchBase, selectBatchTasks } from "../../../../runtime/src/review/batch-select.js";
|
|
4
|
+
import { redactSecrets } from "../../../../runtime/src/security/redact.js";
|
|
5
|
+
import { safeTaskText } from "../../../../runtime/src/task/render.js";
|
|
6
|
+
import { errorEnvelope } from "../output.js";
|
|
7
|
+
function outcomeLabel(task) {
|
|
8
|
+
if (task.aborted) {
|
|
9
|
+
return "aborted";
|
|
10
|
+
}
|
|
11
|
+
if (task.review === null) {
|
|
12
|
+
return task.verify === "FAIL" ? "no review (verify failed)" : "not reviewed";
|
|
13
|
+
}
|
|
14
|
+
const reason = task.review.reason === undefined ? "" : ` ${task.review.reason}`;
|
|
15
|
+
return `${task.review.status}${reason}${task.review.reused === true ? " (reused)" : ""}${task.retried ? " (retried)" : ""}`;
|
|
16
|
+
}
|
|
17
|
+
function renderBatch(data) {
|
|
18
|
+
const lines = [
|
|
19
|
+
`Batch of ${data.tasks.length} task(s) under ${data.parentTaskId}, width ${data.width}.`,
|
|
20
|
+
"",
|
|
21
|
+
...data.tasks.map((task) => `- ${task.taskId} verify ${task.verify} review ${outcomeLabel(task)} base ${task.base.slice(0, 12)} (${task.baseSource}) ${safeTaskText(redactSecrets(task.title))}`)
|
|
22
|
+
];
|
|
23
|
+
if (data.tasks.some((task) => task.aborted)) {
|
|
24
|
+
lines.push("", "The source changed or the run was interrupted; unfinished tasks were aborted. Run the batch again.");
|
|
25
|
+
}
|
|
26
|
+
return lines.join("\n");
|
|
27
|
+
}
|
|
28
|
+
function isPass(task) {
|
|
29
|
+
return !task.aborted && task.review?.status === "PASS";
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The envelope's code follows the worst outcome that needs a human: a failed
|
|
33
|
+
* verify or review (fix the code) before a run that could not finish (run it
|
|
34
|
+
* again), so one defect is never hidden behind an unrelated NOT_RUN.
|
|
35
|
+
*/
|
|
36
|
+
function classify(tasks) {
|
|
37
|
+
if (tasks.every(isPass)) {
|
|
38
|
+
return "BATCH_RESULT";
|
|
39
|
+
}
|
|
40
|
+
return tasks.some((task) => task.verify === "FAIL" || task.review?.status === "FAIL")
|
|
41
|
+
? "BATCH_FAILED"
|
|
42
|
+
: "BATCH_NOT_RUN";
|
|
43
|
+
}
|
|
44
|
+
export async function runBatchCommand(options) {
|
|
45
|
+
const { args } = options;
|
|
46
|
+
const parentTaskId = args.parentTaskId;
|
|
47
|
+
if (parentTaskId === undefined) {
|
|
48
|
+
throw new AgentOpsError("BATCH_PARENT_REQUIRED", "Batch requires --parent.");
|
|
49
|
+
}
|
|
50
|
+
const selected = selectBatchTasks(await options.tasks.list(), parentTaskId);
|
|
51
|
+
const plan = selected.map((record) => ({
|
|
52
|
+
record,
|
|
53
|
+
...resolveBatchBase(record, record.task.id === parentTaskId, {
|
|
54
|
+
...(args.base === undefined ? {} : { base: args.base }),
|
|
55
|
+
...(args.parentBase === undefined ? {} : { parentBase: args.parentBase }),
|
|
56
|
+
...(options.worktreeBase === undefined ? {} : { worktreeBase: options.worktreeBase })
|
|
57
|
+
})
|
|
58
|
+
}));
|
|
59
|
+
const baseOf = new Map(plan.map((item) => [item.record.task.id, item.base]));
|
|
60
|
+
const base = (id) => baseOf.get(id);
|
|
61
|
+
// The first answer is the clean-tree check: a batch reviews committed ranges.
|
|
62
|
+
if (!await options.guard()) {
|
|
63
|
+
return {
|
|
64
|
+
...errorEnvelope("BATCH_DIRTY_WORKTREE", "Batch needs a clean working tree. Commit or discard your changes first."),
|
|
65
|
+
data: null
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
const width = args.width ?? DEFAULT_BATCH_WIDTH;
|
|
69
|
+
const outcomes = await runBatch(plan.map((item) => item.record.task.id), {
|
|
70
|
+
isVerified: (id) => options.isVerified(id, base(id)),
|
|
71
|
+
verify: (id, signal) => options.verify(id, base(id), signal),
|
|
72
|
+
review: (id, signal) => options.review(id, base(id), signal),
|
|
73
|
+
guard: options.guard
|
|
74
|
+
}, {
|
|
75
|
+
width,
|
|
76
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
77
|
+
...(options.onProgress === undefined ? {} : { onProgress: options.onProgress })
|
|
78
|
+
});
|
|
79
|
+
const byId = new Map(outcomes.map((outcome) => [outcome.id, outcome]));
|
|
80
|
+
const tasks = plan.map((item) => {
|
|
81
|
+
const outcome = byId.get(item.record.task.id);
|
|
82
|
+
return {
|
|
83
|
+
taskId: item.record.task.id,
|
|
84
|
+
title: item.record.task.title,
|
|
85
|
+
base: item.base,
|
|
86
|
+
baseSource: item.source,
|
|
87
|
+
verify: outcome.verify,
|
|
88
|
+
review: outcome.review,
|
|
89
|
+
retried: outcome.retried,
|
|
90
|
+
aborted: outcome.aborted
|
|
91
|
+
};
|
|
92
|
+
});
|
|
93
|
+
const code = classify(tasks);
|
|
94
|
+
const passed = tasks.filter(isPass).length;
|
|
95
|
+
const message = code === "BATCH_RESULT"
|
|
96
|
+
? `Batch passed: ${passed} of ${tasks.length} task(s).`
|
|
97
|
+
: code === "BATCH_FAILED"
|
|
98
|
+
? `Batch failed: ${passed} of ${tasks.length} task(s) passed.`
|
|
99
|
+
: `Batch was not completed: ${passed} of ${tasks.length} task(s) passed.`;
|
|
100
|
+
const summary = { message, parentTaskId, width, tasks };
|
|
101
|
+
const data = { ...summary, text: `${message}\n\n${renderBatch(summary)}` };
|
|
102
|
+
if (code === "BATCH_RESULT") {
|
|
103
|
+
return { code, status: "ok", data, errors: [] };
|
|
104
|
+
}
|
|
105
|
+
return { code, status: "error", data, errors: [{ code, message }] };
|
|
106
|
+
}
|
|
@@ -196,6 +196,28 @@ async function preflightReview(options, context, sourceFingerprint) {
|
|
|
196
196
|
summary: { status: "PASS", sourceFingerprint, commands }
|
|
197
197
|
};
|
|
198
198
|
}
|
|
199
|
+
/**
|
|
200
|
+
* Whether the task already holds current PASS verification evidence for the
|
|
201
|
+
* source a review of `base` (or the working tree) would measure. This is the
|
|
202
|
+
* same preflight `review` runs before it spawns, so a batch that skips verify
|
|
203
|
+
* on a true answer never reaches a review that then reports stale evidence.
|
|
204
|
+
*/
|
|
205
|
+
export async function hasFreshVerification(options, taskId, base) {
|
|
206
|
+
if (options.root === undefined || options.gitRunner === undefined) {
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
const context = await taskContext({ ...options, taskId });
|
|
210
|
+
if (context === undefined) {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
const scope = await resolveReviewScope({
|
|
214
|
+
root: options.root,
|
|
215
|
+
runner: options.gitRunner,
|
|
216
|
+
...(base === undefined ? {} : { base })
|
|
217
|
+
});
|
|
218
|
+
const fingerprint = await calculateSourceFingerprint(options.root, scope, options.gitRunner);
|
|
219
|
+
return (await preflightReview(options, context, fingerprint)).ok;
|
|
220
|
+
}
|
|
199
221
|
function scopeReason(error) {
|
|
200
222
|
if (!(error instanceof AgentOpsError)) {
|
|
201
223
|
return undefined;
|
|
@@ -322,7 +322,7 @@ export function managedRules(descriptor, context) {
|
|
|
322
322
|
lines.push("For every change:", "", "1. Define two to five mechanically verifiable acceptance criteria.", "2. Inspect the smallest relevant scope and preserve unrelated changes.", "3. Apply the smallest safe change.", "4. Run evidence-producing verification for every criterion.", "5. Obtain complete independent review before claiming completion, via", " `agent-ops review --task <task-id> --yes` (or the CLI's equivalent invocation).", " Never call a", " review-target CLI (agy, codex, claude) directly — direct calls skip", " the enforced read-only sandbox flags and can hang or fail on command", " permission prompts.", " Start this command through the trusted outer host runner as its first", " and only invocation. On Codex, the outer tool request must use", " `sandbox_permissions: \"require_escalated\"` so network and loopback", " are available; in that same request, unset", " `CODEX_SANDBOX_NETWORK_DISABLED` and set `AGENT_OPS_HOST` when three", " targets are configured. The Codex shell form is", " `env -u CODEX_SANDBOX_NETWORK_DISABLED AGENT_OPS_HOST=<current-host> agent-ops review --task <task-id> --yes`.", " Keep it a plain command line: to keep the output add `--output <file>`", " (the JSON envelope is written there and one line is printed) instead of", " a shell redirect, which the host can only allow by matching the whole", " command string, task id included.", " Do not run it once in the restricted sandbox and retry; changing the", " variable inside that sandbox is not elevation.", " Review runs exactly two fresh sessions: with three configured targets,", " set `AGENT_OPS_HOST` to the current CLI id so it is excluded; with two", " targets both run in configured order; with one target it runs twice.", " The first session is necessary; a PASS then requires an adversarial", " re-check carrying the first report. A first FAIL or NOT_RUN stops.", "", "Treat `.agent-ops/config.json` as verifier authority. Discovery output is", "only a proposal until a user confirms it. Repository commands require an", "exact matching trust record. Confirmed project init/update grants it", "automatically when verification commands are configured.", "");
|
|
323
323
|
}
|
|
324
324
|
if (context.capabilities.includes("task")) {
|
|
325
|
-
lines.push("Split a change that exceeds five acceptance criteria into subtasks:", "`agent-ops task create --parent <task-id>` records one, and", "`agent-ops task status --parent <task-id>` lists them. Each subtask", "carries its own criteria, verification, and independent review;", "completing one never completes its parent.", "", "Delegate a subtask whose scope spans unknown files, broad scans, or an", "independent workstream instead of doing it inline. Name the bounded", "scope, the artifact to return, and the acceptance criteria it serves.", subagentInstruction(descriptor), "Read every returned artifact back yourself: integration, verification,", "and review stay with the coordinator.", "", "One writer at a time. `agent-ops verify` and `agent-ops review`", "fingerprint the working tree and report UNKNOWN when anything else", "writes it during the run, so delegate edits to one subagent at a time,", "or give each writer its own Git worktree and merge before verifying.", "Read-only delegation has no such limit and may run in parallel.", "", "Parallel conversations in one repository each need their own worktree.", "When `.agent-ops/config.json` sets `worktree.mode` to `auto`, run", "`agent-ops task create` from the main checkout before your first edit:", "it creates this session's worktree and puts the task there (`agent-ops", "worktree add <name> --session <session-id>` does the same without a", "task). Work only inside the printed path", "(Claude Code: EnterWorktree with that path; otherwise use it as every", "command's working directory). Commit, then run verify and review for", "each task with the printed `--base`; do not run `task complete` in the", "worktree. Leave it (Claude Code: ExitWorktree with action keep) and run", "`agent-ops worktree finish <name>` from the main checkout: it completes", "every task in the worktree, subtasks first, then merges. A finish that", "reports an incomplete task names it; verify or review that task again", "in the worktree and finish again. A finish that reports a conflict is", "resolved once, as a new task whose criteria cover both intents; if its", "verification or review fails, reset as instructed and report.", "");
|
|
325
|
+
lines.push("Split a change that exceeds five acceptance criteria into subtasks:", "`agent-ops task create --parent <task-id>` records one, and", "`agent-ops task status --parent <task-id>` lists them. Each subtask", "carries its own criteria, verification, and independent review;", "completing one never completes its parent.", "", "To review every active subtask of a parent together, run", "`agent-ops batch --parent <task-id> --yes` instead of one review per", "subtask. It needs what review needs: start it through the trusted outer", "host runner as its first and only invocation, with the same Codex", "elevation, and keep `--output <file>` instead of a shell redirect. It", "verifies, one task at a time, each task that lacks fresh evidence, and", "reviews a few tasks at once. Do not commit or edit the worktree while it", "runs: a moved HEAD voids every verify and review in the batch.", "", "Delegate a subtask whose scope spans unknown files, broad scans, or an", "independent workstream instead of doing it inline. Name the bounded", "scope, the artifact to return, and the acceptance criteria it serves.", subagentInstruction(descriptor), "Read every returned artifact back yourself: integration, verification,", "and review stay with the coordinator.", "", "One writer at a time. `agent-ops verify` and `agent-ops review`", "fingerprint the working tree and report UNKNOWN when anything else", "writes it during the run, so delegate edits to one subagent at a time,", "or give each writer its own Git worktree and merge before verifying.", "Read-only delegation has no such limit and may run in parallel.", "", "Parallel conversations in one repository each need their own worktree.", "When `.agent-ops/config.json` sets `worktree.mode` to `auto`, run", "`agent-ops task create` from the main checkout before your first edit:", "it creates this session's worktree and puts the task there (`agent-ops", "worktree add <name> --session <session-id>` does the same without a", "task). Work only inside the printed path", "(Claude Code: EnterWorktree with that path; otherwise use it as every", "command's working directory). Commit, then run verify and review for", "each task with the printed `--base`; do not run `task complete` in the", "worktree. Leave it (Claude Code: ExitWorktree with action keep) and run", "`agent-ops worktree finish <name>` from the main checkout: it completes", "every task in the worktree, subtasks first, then merges. A finish that", "reports an incomplete task names it; verify or review that task again", "in the worktree and finish again. A finish that reports a conflict is", "resolved once, as a new task whose criteria cover both intents; if its", "verification or review fails, reset as instructed and report.", "");
|
|
326
326
|
}
|
|
327
327
|
if (context.capabilities.includes("lifecycle-summary")) {
|
|
328
328
|
lines.push("Advisory lifecycle summaries and local logs are informational. Advisory", "failures must remain fail-open and cannot become verification evidence.", "");
|
|
@@ -15,13 +15,28 @@ export const CLAUDE_PREAUTH_ALLOW = [
|
|
|
15
15
|
"Bash(agent-ops task *)",
|
|
16
16
|
"Bash(agent-ops verify *)",
|
|
17
17
|
"Bash(agent-ops review *)",
|
|
18
|
+
"Bash(agent-ops batch *)",
|
|
18
19
|
"Bash(agent-ops worktree *)",
|
|
19
|
-
"Bash(agent-ops doctor *)"
|
|
20
|
+
"Bash(agent-ops doctor *)",
|
|
21
|
+
// The PreToolUse hook still blocks a path this session does not own.
|
|
22
|
+
"EnterWorktree",
|
|
23
|
+
// A worktree commit runs the project's pre-commit hook, usually this.
|
|
24
|
+
"Bash(npx lint-staged *)"
|
|
20
25
|
];
|
|
21
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* review and batch talk to the reviewer's API, which the sandbox withholds;
|
|
28
|
+
* worktree commit and lint-staged write the main .git, which a worktree's
|
|
29
|
+
* sandbox cannot.
|
|
30
|
+
*/
|
|
22
31
|
export const CLAUDE_PREAUTH_UNSANDBOXED = [
|
|
23
32
|
"agent-ops review",
|
|
24
|
-
"agent-ops review *"
|
|
33
|
+
"agent-ops review *",
|
|
34
|
+
"agent-ops batch",
|
|
35
|
+
"agent-ops batch *",
|
|
36
|
+
"agent-ops worktree commit",
|
|
37
|
+
"agent-ops worktree commit *",
|
|
38
|
+
"npx lint-staged",
|
|
39
|
+
"npx lint-staged *"
|
|
25
40
|
];
|
|
26
41
|
export const CODEX_RULES_PATH = "rules/agent-ops.rules";
|
|
27
42
|
export const CODEX_RULES_MARKER = "# agent-ops managed: regenerated by agent-ops init and update; do not edit.";
|
|
@@ -51,11 +66,32 @@ function rule(options) {
|
|
|
51
66
|
}
|
|
52
67
|
/**
|
|
53
68
|
* Prefix rules for the exact shapes `.agent-ops/AGENTS.md` tells Codex to run,
|
|
54
|
-
* with and without AGENT_OPS_HOST, for review and the auth probe.
|
|
55
|
-
* by argv prefix, so a shell redirect would defeat it: use
|
|
69
|
+
* with and without AGENT_OPS_HOST, for review, batch and the auth probe.
|
|
70
|
+
* Matching is by argv prefix, so a shell redirect would defeat it: use
|
|
71
|
+
* `--output`.
|
|
56
72
|
*/
|
|
57
73
|
export const CODEX_RULES_CONTENT = `${[
|
|
58
74
|
CODEX_RULES_MARKER,
|
|
75
|
+
...[
|
|
76
|
+
{
|
|
77
|
+
pattern: ["agent-ops", "worktree", "commit"],
|
|
78
|
+
justification: "agent-ops worktree commit runs git hooks, which write the main .git outside the sandbox.",
|
|
79
|
+
match: "agent-ops worktree commit -m message"
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
pattern: ["npx", "lint-staged"],
|
|
83
|
+
justification: "lint-staged backs up through the main .git, which the sandbox cannot write from a worktree.",
|
|
84
|
+
match: "npx lint-staged --debug"
|
|
85
|
+
}
|
|
86
|
+
].map((entry) => [
|
|
87
|
+
"prefix_rule(",
|
|
88
|
+
` pattern = ${starlarkList(entry.pattern)},`,
|
|
89
|
+
' decision = "allow",',
|
|
90
|
+
` justification = ${JSON.stringify(entry.justification)},`,
|
|
91
|
+
` match = [${JSON.stringify(entry.match)}],`,
|
|
92
|
+
' not_match = ["agent-ops trust grant --scope project --yes"]',
|
|
93
|
+
")"
|
|
94
|
+
].join("\n")),
|
|
59
95
|
...[true, false].flatMap((withHost) => {
|
|
60
96
|
const host = withHost ? "AGENT_OPS_HOST=codex " : "";
|
|
61
97
|
const env = `env -u CODEX_SANDBOX_NETWORK_DISABLED ${host}agent-ops`;
|
|
@@ -67,6 +103,13 @@ export const CODEX_RULES_CONTENT = `${[
|
|
|
67
103
|
match: `${env} review --task task-x --yes --output review.json`,
|
|
68
104
|
notMatch: `${env} trust grant --scope project --yes`
|
|
69
105
|
}),
|
|
106
|
+
rule({
|
|
107
|
+
withHost,
|
|
108
|
+
command: ["batch"],
|
|
109
|
+
justification: "agent-ops batch runs reviews, which need network and loopback outside the sandbox.",
|
|
110
|
+
match: `${env} batch --parent task-x --yes --output batch.json`,
|
|
111
|
+
notMatch: `${env} trust grant --scope project --yes`
|
|
112
|
+
}),
|
|
70
113
|
rule({
|
|
71
114
|
withHost,
|
|
72
115
|
command: ["doctor", "--check-auth"],
|
|
@@ -229,7 +272,7 @@ async function codexChange(codexHome, desired) {
|
|
|
229
272
|
content: CODEX_RULES_CONTENT,
|
|
230
273
|
expectedHash: current?.hash ?? null
|
|
231
274
|
},
|
|
232
|
-
summary: `Writes ${join(codexHome, CODEX_RULES_PATH)}: allow rules for agent-ops review
|
|
275
|
+
summary: `Writes ${join(codexHome, CODEX_RULES_PATH)}: allow rules for agent-ops review, batch, doctor --check-auth, worktree commit and lint-staged.`,
|
|
233
276
|
ownedContent: true
|
|
234
277
|
};
|
|
235
278
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { collectChangeSurface, resolveGitCommit } from "../verify/change-surface.js";
|
|
2
|
+
/**
|
|
3
|
+
* Remembers HEAD now and answers whether the batch may go on: HEAD is still
|
|
4
|
+
* that commit and the working tree has no change. The first answer doubles as
|
|
5
|
+
* the clean-tree check, since a batch reviews committed ranges only.
|
|
6
|
+
*/
|
|
7
|
+
export async function createSourceGuard(runner) {
|
|
8
|
+
const head = await resolveGitCommit(runner, "HEAD");
|
|
9
|
+
return async () => await resolveGitCommit(runner, "HEAD") === head &&
|
|
10
|
+
(await collectChangeSurface(runner)).paths.length === 0;
|
|
11
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { AgentOpsError } from "../fs/paths.js";
|
|
2
|
+
/**
|
|
3
|
+
* The tasks one `batch --parent` covers: the parent's active subtasks in
|
|
4
|
+
* creation order, then the parent itself. Archived and completed tasks, and
|
|
5
|
+
* tasks under another parent, are left out.
|
|
6
|
+
*/
|
|
7
|
+
export function selectBatchTasks(records, parentId) {
|
|
8
|
+
const parent = records.find((record) => record.task.id === parentId);
|
|
9
|
+
if (parent === undefined) {
|
|
10
|
+
throw new AgentOpsError("BATCH_PARENT_NOT_FOUND", `Parent task not found: ${parentId}`);
|
|
11
|
+
}
|
|
12
|
+
if (parent.status !== "active") {
|
|
13
|
+
throw new AgentOpsError("BATCH_PARENT_NOT_ACTIVE", `Parent task is not active: ${parentId}`);
|
|
14
|
+
}
|
|
15
|
+
const subtasks = records
|
|
16
|
+
.filter((record) => record.task.parentTaskId === parentId && record.status === "active")
|
|
17
|
+
.sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.task.id.localeCompare(right.task.id));
|
|
18
|
+
return [...subtasks, parent];
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* An explicit flag wins, then the commit the task's last PASS review measured
|
|
22
|
+
* against (so a rerun reproduces the same fingerprint), then the worktree base.
|
|
23
|
+
* A parent's base is a human choice that cannot be derived from the repository.
|
|
24
|
+
*/
|
|
25
|
+
export function resolveBatchBase(record, isParent, input) {
|
|
26
|
+
const flag = isParent ? input.parentBase : input.base;
|
|
27
|
+
if (flag !== undefined) {
|
|
28
|
+
return { base: flag, source: "flag" };
|
|
29
|
+
}
|
|
30
|
+
if (record.reviewBase !== undefined) {
|
|
31
|
+
return { base: record.reviewBase, source: "recorded" };
|
|
32
|
+
}
|
|
33
|
+
if (input.worktreeBase !== undefined) {
|
|
34
|
+
return { base: input.worktreeBase, source: "worktree" };
|
|
35
|
+
}
|
|
36
|
+
throw new AgentOpsError("BATCH_BASE_UNKNOWN", `No base for task ${record.task.id}: pass ${isParent ? "--parent-base" : "--base"} or run from an agent-ops worktree.`);
|
|
37
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import { AgentOpsError } from "../fs/paths.js";
|
|
2
|
+
/**
|
|
3
|
+
* NOT_RUN reasons that mean the reviewer was briefly unavailable. Anything
|
|
4
|
+
* else (a sandboxed host, stale verification, a changed policy) fails the same
|
|
5
|
+
* way on a second attempt, so it is reported instead of retried.
|
|
6
|
+
*/
|
|
7
|
+
export const TRANSIENT_REVIEW_REASONS = new Set([
|
|
8
|
+
"probe-failed",
|
|
9
|
+
"stalled",
|
|
10
|
+
"timeout",
|
|
11
|
+
"quota-exhausted",
|
|
12
|
+
"network-unreachable"
|
|
13
|
+
]);
|
|
14
|
+
/** Reviews allowed at once when the caller names no width. */
|
|
15
|
+
export const DEFAULT_BATCH_WIDTH = 2;
|
|
16
|
+
/**
|
|
17
|
+
* Verify runs one task at a time (a full test run per task; several at once
|
|
18
|
+
* starve each other of CPU and fake a failure). Each task's review starts as
|
|
19
|
+
* soon as its own verify has passed, so the next verify overlaps it. Reviews
|
|
20
|
+
* run up to `width` at a time; a transient NOT_RUN drops the width to 1 for
|
|
21
|
+
* good and that task gets one more try alone after everything else finished.
|
|
22
|
+
*/
|
|
23
|
+
export async function runBatch(ids, deps, options) {
|
|
24
|
+
if (!Number.isInteger(options.width) || options.width < 1) {
|
|
25
|
+
throw new AgentOpsError("BATCH_INVALID_WIDTH", "Batch width must be a positive integer.");
|
|
26
|
+
}
|
|
27
|
+
const progress = options.onProgress ?? (() => { });
|
|
28
|
+
const controller = new AbortController();
|
|
29
|
+
const outer = options.signal;
|
|
30
|
+
if (outer?.aborted === true) {
|
|
31
|
+
controller.abort();
|
|
32
|
+
}
|
|
33
|
+
else {
|
|
34
|
+
outer?.addEventListener("abort", () => controller.abort(), { once: true });
|
|
35
|
+
}
|
|
36
|
+
const slots = new Map(ids.map((id) => [
|
|
37
|
+
id,
|
|
38
|
+
{ id, verify: "not-run", review: null, retried: false }
|
|
39
|
+
]));
|
|
40
|
+
const slot = (id) => slots.get(id);
|
|
41
|
+
const stop = (why) => {
|
|
42
|
+
if (!controller.signal.aborted) {
|
|
43
|
+
progress(why);
|
|
44
|
+
controller.abort();
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
const guard = async () => {
|
|
48
|
+
try {
|
|
49
|
+
return await deps.guard();
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
let width = options.width;
|
|
56
|
+
let running = 0;
|
|
57
|
+
let verifying = true;
|
|
58
|
+
const ready = [];
|
|
59
|
+
const retry = [];
|
|
60
|
+
let finish;
|
|
61
|
+
const done = new Promise((resolve) => {
|
|
62
|
+
finish = resolve;
|
|
63
|
+
});
|
|
64
|
+
const review = async (id, final) => {
|
|
65
|
+
if (!await guard()) {
|
|
66
|
+
stop("source changed; aborting the batch");
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
const entry = slot(id);
|
|
70
|
+
if (final) {
|
|
71
|
+
entry.retried = true;
|
|
72
|
+
}
|
|
73
|
+
let result;
|
|
74
|
+
try {
|
|
75
|
+
result = await deps.review(id, controller.signal);
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
progress(`review ${id} failed to run: ${error instanceof Error ? error.message : String(error)}`);
|
|
79
|
+
result = { status: "NOT_RUN", reason: "review-error" };
|
|
80
|
+
}
|
|
81
|
+
if (controller.signal.aborted) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
entry.review = result;
|
|
85
|
+
if (!final &&
|
|
86
|
+
result.status === "NOT_RUN" &&
|
|
87
|
+
result.reason !== undefined &&
|
|
88
|
+
TRANSIENT_REVIEW_REASONS.has(result.reason)) {
|
|
89
|
+
if (width > 1) {
|
|
90
|
+
width = 1;
|
|
91
|
+
progress(`degrade: review width -> 1 (${id}: ${result.reason})`);
|
|
92
|
+
}
|
|
93
|
+
retry.push(id);
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
const pump = () => {
|
|
97
|
+
while (!controller.signal.aborted && running < width && ready.length > 0) {
|
|
98
|
+
const id = ready.shift();
|
|
99
|
+
running += 1;
|
|
100
|
+
void review(id, false).finally(() => {
|
|
101
|
+
running -= 1;
|
|
102
|
+
pump();
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
if (!verifying && running === 0 && (ready.length === 0 || controller.signal.aborted)) {
|
|
106
|
+
finish();
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
const fresh = await Promise.all(ids.map((id) => deps.isVerified(id).catch(() => false)));
|
|
110
|
+
const stale = [];
|
|
111
|
+
ids.forEach((id, index) => {
|
|
112
|
+
if (fresh[index] === true) {
|
|
113
|
+
slot(id).verify = "skipped";
|
|
114
|
+
ready.push(id);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
stale.push(id);
|
|
118
|
+
}
|
|
119
|
+
});
|
|
120
|
+
const verifyAll = async () => {
|
|
121
|
+
for (const id of stale) {
|
|
122
|
+
if (controller.signal.aborted) {
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
if (!await guard()) {
|
|
126
|
+
stop("source changed; aborting the batch");
|
|
127
|
+
break;
|
|
128
|
+
}
|
|
129
|
+
let status;
|
|
130
|
+
try {
|
|
131
|
+
status = await deps.verify(id, controller.signal);
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
progress(`verify ${id} failed to run: ${error instanceof Error ? error.message : String(error)}`);
|
|
135
|
+
status = "FAIL";
|
|
136
|
+
}
|
|
137
|
+
if (controller.signal.aborted) {
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
slot(id).verify = status;
|
|
141
|
+
if (status === "PASS") {
|
|
142
|
+
ready.push(id);
|
|
143
|
+
pump();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
verifying = false;
|
|
147
|
+
pump();
|
|
148
|
+
};
|
|
149
|
+
void verifyAll();
|
|
150
|
+
pump();
|
|
151
|
+
await done;
|
|
152
|
+
if (!controller.signal.aborted) {
|
|
153
|
+
for (const id of retry) {
|
|
154
|
+
if (controller.signal.aborted) {
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
await review(id, true);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
const interrupted = controller.signal.aborted;
|
|
161
|
+
return ids.map((id) => {
|
|
162
|
+
const entry = slot(id);
|
|
163
|
+
const pendingRetry = retry.includes(id) && !entry.retried;
|
|
164
|
+
const unfinished = entry.review === null
|
|
165
|
+
? entry.verify !== "FAIL"
|
|
166
|
+
: pendingRetry;
|
|
167
|
+
return {
|
|
168
|
+
id,
|
|
169
|
+
verify: entry.verify,
|
|
170
|
+
review: entry.review,
|
|
171
|
+
retried: entry.retried,
|
|
172
|
+
aborted: interrupted && unfinished
|
|
173
|
+
};
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* One probe per target for a whole batch. Concurrent callers share the probe
|
|
178
|
+
* in flight and a success is kept, but a failure is forgotten so the retry of
|
|
179
|
+
* a task re-probes instead of inheriting the failure.
|
|
180
|
+
*/
|
|
181
|
+
export function memoizePreflight(probe) {
|
|
182
|
+
const cache = new Map();
|
|
183
|
+
return (target, budget) => {
|
|
184
|
+
const cached = cache.get(target);
|
|
185
|
+
if (cached !== undefined) {
|
|
186
|
+
return cached;
|
|
187
|
+
}
|
|
188
|
+
const pending = probe(target, budget);
|
|
189
|
+
cache.set(target, pending);
|
|
190
|
+
const forget = () => {
|
|
191
|
+
if (cache.get(target) === pending) {
|
|
192
|
+
cache.delete(target);
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
pending.then((answer) => {
|
|
196
|
+
if ((typeof answer === "string" ? answer : answer.result) !== "ok") {
|
|
197
|
+
forget();
|
|
198
|
+
}
|
|
199
|
+
}, forget);
|
|
200
|
+
return pending;
|
|
201
|
+
};
|
|
202
|
+
}
|
|
@@ -161,11 +161,24 @@ export function buildTargetInvocation(request) {
|
|
|
161
161
|
};
|
|
162
162
|
}
|
|
163
163
|
/** A deep doctor probe uses stdin but keeps its simple text response contract. */
|
|
164
|
+
/**
|
|
165
|
+
* What the probe runs with where it differs from a review. The probe asks for
|
|
166
|
+
* one word, not a review, and agy's plan mode tells the model to research the
|
|
167
|
+
* task first: on a real project it explores, runs a command the headless user
|
|
168
|
+
* never allowed, is soft-denied, and prints nothing, which reads as
|
|
169
|
+
* `probe-failed`. In 40 replays, plan mode failed 4 of 20 and without it all 20
|
|
170
|
+
* answered in exactly two model calls. Only the probe drops it; a review keeps
|
|
171
|
+
* `READ_ONLY_ARGS`, and the sandbox stays on for both.
|
|
172
|
+
*/
|
|
173
|
+
const PROBE_ARGS = {
|
|
174
|
+
agy: ["--sandbox"]
|
|
175
|
+
};
|
|
164
176
|
export function buildProbeInvocation(request) {
|
|
165
|
-
const
|
|
166
|
-
if (
|
|
177
|
+
const eligible = READ_ONLY_ARGS[request.target];
|
|
178
|
+
if (eligible === undefined || eligible.length === 0) {
|
|
167
179
|
return undefined;
|
|
168
180
|
}
|
|
181
|
+
const readOnly = PROBE_ARGS[request.target] ?? eligible;
|
|
169
182
|
if (request.target === "codex") {
|
|
170
183
|
return {
|
|
171
184
|
command: "codex",
|
|
@@ -37,7 +37,20 @@ export function probeTimeoutMs(requested) {
|
|
|
37
37
|
export async function probeReviewTarget(target, options) {
|
|
38
38
|
return (await probeReviewTargetDetailed(target, options)).result;
|
|
39
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* A deep probe is one model call, and a model call can fail for reasons that
|
|
42
|
+
* say nothing about the install. `probe-failed` is the answer for "it did not
|
|
43
|
+
* work and did not say why", so it gets one more try; every other answer is
|
|
44
|
+
* specific (not logged in, no network, timed out, not installed) and repeating
|
|
45
|
+
* it would only spend more of the budget on the same result.
|
|
46
|
+
*/
|
|
40
47
|
export async function probeReviewTargetDetailed(target, options) {
|
|
48
|
+
const first = await probeOnce(target, options);
|
|
49
|
+
return options.deep === true && first.result === "probe-failed"
|
|
50
|
+
? await probeOnce(target, options)
|
|
51
|
+
: first;
|
|
52
|
+
}
|
|
53
|
+
async function probeOnce(target, options) {
|
|
41
54
|
const deep = options.deep === true;
|
|
42
55
|
const directory = deep
|
|
43
56
|
? await mkdtemp(join(tmpdir(), "agent-ops-review-probe-"))
|
|
@@ -236,12 +236,15 @@ evidence. Give each editing conversation its own worktree instead:
|
|
|
236
236
|
- With `worktree.mode` auto, `init` and `update` also pre-authorize what a
|
|
237
237
|
session runs itself, and the plan lists it before you confirm: in
|
|
238
238
|
`.claude/settings.local.json`, `permissions.allow` for `agent-ops task`,
|
|
239
|
-
`verify`, `review`, `worktree` and `doctor
|
|
240
|
-
|
|
239
|
+
`verify`, `review`, `worktree` and `doctor`, plus `EnterWorktree` (the hook
|
|
240
|
+
still blocks a path the session does not own) and `npx lint-staged`, and
|
|
241
|
+
`sandbox.excludedCommands` for `agent-ops review`, `batch`, `worktree commit`
|
|
242
|
+
and `npx lint-staged` (your other entries are kept, and the plan describes
|
|
241
243
|
the file instead of printing it); and, when the CLI can see Codex's home
|
|
242
244
|
(`CODEX_HOME`, else `~/.codex`), `rules/agent-ops.rules` with allow rules
|
|
243
245
|
for `agent-ops review` and `doctor --check-auth`, with and without
|
|
244
|
-
`AGENT_OPS_HOST
|
|
246
|
+
`AGENT_OPS_HOST=`, and for `agent-ops worktree commit` and `npx lint-staged`,
|
|
247
|
+
so a pre-commit hook can write the main `.git`. `trust`, `allow-stop`, `init`, `update` and `uninstall`
|
|
245
248
|
are never pre-authorized. Turning auto mode off, or `uninstall`, takes
|
|
246
249
|
exactly those entries back out; a rules file of that name agent-ops did not
|
|
247
250
|
write is left alone.
|
package/docs/en/spec/review.md
CHANGED
|
@@ -125,3 +125,73 @@ FAIL.
|
|
|
125
125
|
- Evidence: The result reason distinguishes a protocol violation from a verdict.
|
|
126
126
|
- Positive: `NOT_RUN: unparseable-output; one criterion was missing.`
|
|
127
127
|
- Negative: `Record a failed review because the model's JSON was malformed.`
|
|
128
|
+
|
|
129
|
+
## REVIEW-BATCH-001
|
|
130
|
+
|
|
131
|
+
A batch review MUST cover exactly the active subtasks of the named parent and the parent itself.
|
|
132
|
+
|
|
133
|
+
- Trigger: Running `batch --parent <task-id>`.
|
|
134
|
+
- Action: Select the parent's active subtasks in creation order, then the parent. Leave out archived and completed tasks and tasks under another parent. Reject an unknown parent with `BATCH_PARENT_NOT_FOUND` and an inactive one with `BATCH_PARENT_NOT_ACTIVE`.
|
|
135
|
+
- Evidence: `data.tasks` lists exactly those task ids, in that order.
|
|
136
|
+
- Positive: `Two active subtasks and the parent are covered; the archived subtask is not.`
|
|
137
|
+
- Negative: `Cover every task in the worktree whatever its parent.`
|
|
138
|
+
|
|
139
|
+
## REVIEW-BATCH-002
|
|
140
|
+
|
|
141
|
+
Each task's base MUST come from an explicit flag, then its recorded review base, then the worktree base.
|
|
142
|
+
|
|
143
|
+
- Trigger: Resolving the range a batch task is verified and reviewed against.
|
|
144
|
+
- Action: Use `--base` for subtasks and `--parent-base` for the parent; otherwise the task's recorded `reviewBase`; otherwise the worktree base. Report `BATCH_BASE_UNKNOWN` when none exists.
|
|
145
|
+
- Evidence: `data.tasks[].base` names the base each task used.
|
|
146
|
+
- Positive: `A rerun reuses each PASSed task's recorded base, so its fingerprint and attestation still match.`
|
|
147
|
+
- Negative: `Apply --base to the parent, or fall back to HEAD when no base is known.`
|
|
148
|
+
|
|
149
|
+
## REVIEW-BATCH-003
|
|
150
|
+
|
|
151
|
+
Verification MUST run one task at a time, each task's review MUST start as soon as its own verification passes, and no more than `--width` reviews (default 2) MUST run at once.
|
|
152
|
+
|
|
153
|
+
- Trigger: Tasks in a batch need verification and review.
|
|
154
|
+
- Action: Skip verification for a task that already holds fresh PASS evidence, using the same check review preflight makes. Verify the rest serially. Start a task's review right after its verification passes. Give a task whose verification fails no review, and continue with the others.
|
|
155
|
+
- Evidence: `data.tasks[].verify` is `skipped`, `PASS` or `FAIL`, and a task that failed verification has no review.
|
|
156
|
+
- Positive: `Task B is verified while task A is being reviewed.`
|
|
157
|
+
- Negative: `Run every task's test suite at once.`
|
|
158
|
+
|
|
159
|
+
## REVIEW-BATCH-004
|
|
160
|
+
|
|
161
|
+
A transient NOT_RUN MUST lower the review width to 1 for the rest of the batch and earn exactly one retry, and every other result MUST be final.
|
|
162
|
+
|
|
163
|
+
- Trigger: A review in a batch returns NOT_RUN.
|
|
164
|
+
- Action: When the reason is `probe-failed`, `stalled`, `timeout`, `quota-exhausted` or `network-unreachable`, set the width to 1 for every review not yet started and retry that task once, alone, after the rest have finished. Never retry another reason, never retry twice, and never raise the width again.
|
|
165
|
+
- Evidence: `review.retried` is true for a retried task, and a second transient NOT_RUN stays NOT_RUN.
|
|
166
|
+
- Positive: `probe-failed on task A drops the width to 1, and A runs once more after the others.`
|
|
167
|
+
- Negative: `Retry host-sandboxed until it passes.`
|
|
168
|
+
|
|
169
|
+
## REVIEW-BATCH-005
|
|
170
|
+
|
|
171
|
+
A batch MUST stop when the source it started on changes or the process is interrupted.
|
|
172
|
+
|
|
173
|
+
- Trigger: HEAD or the working tree differs from the start of the batch, or the process receives SIGINT or SIGTERM.
|
|
174
|
+
- Action: Require a clean working tree before starting and check HEAD and the tree before every step. On a change, abort the reviews in flight, start nothing new, and report each unfinished task as `aborted`. A signal exits 130 or 143.
|
|
175
|
+
- Evidence: `data.tasks[].aborted` is true for every task that did not finish.
|
|
176
|
+
- Positive: `A commit during the batch aborts it and the unfinished tasks say aborted.`
|
|
177
|
+
- Negative: `Let in-flight reviews finish against a HEAD that no longer matches.`
|
|
178
|
+
|
|
179
|
+
## REVIEW-BATCH-006
|
|
180
|
+
|
|
181
|
+
A batch MUST probe each review target at most once while the probe succeeds and MUST NOT cache a failed probe.
|
|
182
|
+
|
|
183
|
+
- Trigger: Reviews in a batch need target preflight.
|
|
184
|
+
- Action: Share the probe in flight between concurrent reviews and keep a successful answer for the rest of the batch. Forget a failed answer, so the retry of a task probes again.
|
|
185
|
+
- Evidence: A batch of several reviews makes one probe per target, and a retry after a failure makes a new one.
|
|
186
|
+
- Positive: `Five reviews share one successful agy probe.`
|
|
187
|
+
- Negative: `Cache probe-failed and refuse every remaining task.`
|
|
188
|
+
|
|
189
|
+
## REVIEW-BATCH-007
|
|
190
|
+
|
|
191
|
+
A batch result MUST report every task's outcome and MUST exit 0 only when every task passed.
|
|
192
|
+
|
|
193
|
+
- Trigger: A batch ends.
|
|
194
|
+
- Action: Return `BATCH_RESULT`, `BATCH_FAILED` or `BATCH_NOT_RUN` with `data.tasks[]` holding each task's id, base, verify status, review status and reason, `reused`, `retried` and `aborted`, without the full reports, which stay under `.agent-ops/reviews/`. Exit 0 when every task passed, 1 when any review or verification failed, and 2 otherwise. `batch` is started like `review`: it needs `--yes` and trust, and REVIEW-HOST-001 applies to it.
|
|
195
|
+
- Evidence: The envelope code and the exit code agree with the per-task outcomes.
|
|
196
|
+
- Positive: `One FAIL and one NOT_RUN exit 1, so the defect is handled first.`
|
|
197
|
+
- Negative: `Exit 0 while a task is still NOT_RUN.`
|