@kylecheng3146/agent-ops 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/packages/cli/src/args.js +23 -3
- package/dist/packages/cli/src/cli.js +2 -0
- package/dist/packages/cli/src/commands/review.js +4 -1
- package/dist/packages/cli/src/commands/task.js +9 -3
- package/dist/packages/cli/src/wizard.js +5 -5
- package/dist/runtime/src/install/doctor.js +18 -4
- package/dist/runtime/src/install/harness.js +3 -0
- package/dist/runtime/src/review/execute.js +329 -118
- package/dist/runtime/src/review/extract.js +10 -2
- package/dist/runtime/src/review/invocation.js +67 -15
- package/dist/runtime/src/review/probe.js +10 -4
- package/dist/runtime/src/review/render.js +18 -2
- package/dist/runtime/src/review/roles.js +13 -3
- package/dist/runtime/src/review/runner.js +134 -32
- package/dist/runtime/src/schema/validate.js +7 -1
- package/dist/runtime/src/task/render.js +3 -0
- package/dist/runtime/src/task/service.js +18 -2
- package/dist/runtime/src/verify/spawn.js +27 -10
- package/docs/en/guides/configuration.md +14 -5
- package/docs/en/spec/review.md +21 -4
- package/docs/zh-TW/guides/configuration.md +12 -5
- package/docs/zh-TW/spec/review.md +15 -4
- package/package.json +1 -1
- package/schemas/task.schema.json +3 -0
|
@@ -1,17 +1,26 @@
|
|
|
1
|
-
import { mkdtemp, realpath, rm } from "node:fs/promises";
|
|
1
|
+
import { chmod, copyFile, lstat, mkdir, mkdtemp, realpath, rm } from "node:fs/promises";
|
|
2
2
|
import { tmpdir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
4
|
import { runVerificationCommand } from "../verify/spawn.js";
|
|
5
|
+
import { redactSecrets } from "../security/redact.js";
|
|
5
6
|
import { extractReviewObject } from "./extract.js";
|
|
6
7
|
import { buildTargetInvocation } from "./invocation.js";
|
|
7
8
|
import { reviewReportResults, reviewReportStatus, validateReviewReport } from "./report.js";
|
|
8
9
|
import { detectHostTarget, orderChain } from "./roles.js";
|
|
9
|
-
import { buildReviewPrompt } from "./runner.js";
|
|
10
|
-
/**
|
|
11
|
-
|
|
10
|
+
import { buildAdversarialPrompt, buildReviewPrompt } from "./runner.js";
|
|
11
|
+
/**
|
|
12
|
+
* Full repository reviews need far more headroom than the lightweight auth
|
|
13
|
+
* probe. Five minutes was not enough: reviewing a real working tree, codex and
|
|
14
|
+
* claude both exceeded it on this repository, and a timeout costs the whole
|
|
15
|
+
* review while looking like an unavailable target.
|
|
16
|
+
*/
|
|
17
|
+
export const DEFAULT_REVIEW_TIMEOUT_MS = 900_000;
|
|
18
|
+
// USER is load-bearing, not cosmetic: a credential store keyed by account name
|
|
19
|
+
// — the macOS keychain claude reads — cannot be opened without it, and its
|
|
20
|
+
// absence surfaces as "Not logged in" on an install that is logged in.
|
|
12
21
|
const EXECUTION_ENV = [
|
|
13
22
|
"PATH", "PATHEXT", "SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC",
|
|
14
|
-
"LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP"
|
|
23
|
+
"LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP", "USER"
|
|
15
24
|
];
|
|
16
25
|
const AUTH_ENV = {
|
|
17
26
|
claude: ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"],
|
|
@@ -26,7 +35,15 @@ export function isolatedReviewEnvironment(target, directory, source) {
|
|
|
26
35
|
env[key] = value;
|
|
27
36
|
}
|
|
28
37
|
}
|
|
29
|
-
|
|
38
|
+
// claude and agy read their credentials out of the invoking user's home, so
|
|
39
|
+
// replacing it does not isolate them — it only makes them report "not logged
|
|
40
|
+
// in" on an install that is logged in. claude is isolated by its own flags
|
|
41
|
+
// instead: `--safe-mode` disables CLAUDE.md, skills, plugins, hooks, MCP
|
|
42
|
+
// servers and custom agents while explicitly keeping auth. codex keeps a
|
|
43
|
+
// replaced home because CODEX_HOME carries its credentials separately, so
|
|
44
|
+
// the isolation costs it nothing. agy uses its native sandbox and plan mode,
|
|
45
|
+
// but keeps its home because that is where its OAuth session lives.
|
|
46
|
+
if (target === "agy" || target === "claude") {
|
|
30
47
|
env.HOME = source.HOME ?? directory;
|
|
31
48
|
env.USERPROFILE = source.USERPROFILE ?? env.HOME;
|
|
32
49
|
env.XDG_CONFIG_HOME = source.XDG_CONFIG_HOME ?? join(env.HOME, ".config");
|
|
@@ -47,17 +64,15 @@ export function isolatedReviewEnvironment(target, directory, source) {
|
|
|
47
64
|
}
|
|
48
65
|
return env;
|
|
49
66
|
}
|
|
67
|
+
/** Only eligible targets appear: an ineligible one never reaches this gate. */
|
|
50
68
|
const REQUIRED_HELP_FLAGS = {
|
|
51
69
|
claude: [
|
|
52
70
|
"--add-dir", "--permission-mode", "--no-session-persistence",
|
|
53
71
|
"--safe-mode", "--disable-slash-commands", "--json-schema"
|
|
54
72
|
],
|
|
73
|
+
agy: ["--add-dir", "--sandbox", "--mode", "--json-schema", "--log-file"],
|
|
55
74
|
codex: [
|
|
56
75
|
"--cd", "--ephemeral", "--ignore-user-config", "--ignore-rules"
|
|
57
|
-
],
|
|
58
|
-
agy: [
|
|
59
|
-
"--add-dir", "--sandbox", "--mode", "--disable-slash-commands",
|
|
60
|
-
"--json-schema"
|
|
61
76
|
]
|
|
62
77
|
};
|
|
63
78
|
/**
|
|
@@ -69,133 +84,329 @@ const ADVANCING = new Set([
|
|
|
69
84
|
"spawn-failed",
|
|
70
85
|
"timeout"
|
|
71
86
|
]);
|
|
87
|
+
const DIAGNOSTIC_MAX_CHARS = 200;
|
|
88
|
+
/**
|
|
89
|
+
* The target's first line of complaint, redacted and clipped. It travels on the
|
|
90
|
+
* attempt record rather than only in a progress line, because progress is
|
|
91
|
+
* suppressed under `--json`: a machine consumer would otherwise be left with
|
|
92
|
+
* the bare authentication guess this exists to qualify. Never evidence.
|
|
93
|
+
*/
|
|
94
|
+
function firstComplaint(...streams) {
|
|
95
|
+
for (const stream of streams) {
|
|
96
|
+
const line = redactSecrets(stream)
|
|
97
|
+
.split(/\r?\n/u)
|
|
98
|
+
.map((value) => value.trim())
|
|
99
|
+
.find((value) => value.length > 0);
|
|
100
|
+
if (line !== undefined) {
|
|
101
|
+
return line.slice(0, DIAGNOSTIC_MAX_CHARS);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return undefined;
|
|
105
|
+
}
|
|
106
|
+
function rejectedCallReason(output) {
|
|
107
|
+
if (/\b(?:quota|rate limit|usage limit|too many requests)\b/iu.test(output)) {
|
|
108
|
+
return "quota-exhausted";
|
|
109
|
+
}
|
|
110
|
+
if (/\b(?:not logged in|login required|log in to|authentication required|unauthenticated|unauthorized)\b/iu.test(output) ||
|
|
111
|
+
/\b(?:invalid|expired)\s+(?:api key|token|credential)/iu.test(output) ||
|
|
112
|
+
/\b401\b/u.test(output)) {
|
|
113
|
+
return "login-required";
|
|
114
|
+
}
|
|
115
|
+
return "capability-unavailable";
|
|
116
|
+
}
|
|
117
|
+
async function snapshotRepository(request, destination, options) {
|
|
118
|
+
const cloned = await runVerificationCommand({
|
|
119
|
+
id: `review-snapshot-${request.label}`,
|
|
120
|
+
command: "git",
|
|
121
|
+
args: ["clone", "--no-hardlinks", "--quiet", "--", request.repositoryRoot, destination],
|
|
122
|
+
cwd: dirname(destination),
|
|
123
|
+
required: true,
|
|
124
|
+
evidence: { kind: "exit-code" },
|
|
125
|
+
timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 60_000)
|
|
126
|
+
}, {
|
|
127
|
+
cwd: dirname(destination),
|
|
128
|
+
...(options.runner === undefined ? {} : { runner: options.runner })
|
|
129
|
+
});
|
|
130
|
+
if (cloned.status !== "PASS") {
|
|
131
|
+
return firstComplaint(cloned.stderr, cloned.stdout) ??
|
|
132
|
+
`git clone failed (${cloned.failureClass})`;
|
|
133
|
+
}
|
|
134
|
+
for (const path of request.changedFiles ?? []) {
|
|
135
|
+
const source = join(request.repositoryRoot, path);
|
|
136
|
+
const target = join(destination, path);
|
|
137
|
+
try {
|
|
138
|
+
const stat = await lstat(source);
|
|
139
|
+
if (!stat.isFile()) {
|
|
140
|
+
return `changed path is not a regular file: ${path}`;
|
|
141
|
+
}
|
|
142
|
+
await mkdir(dirname(target), { recursive: true });
|
|
143
|
+
await copyFile(source, target);
|
|
144
|
+
await chmod(target, stat.mode & 0o777);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
|
|
148
|
+
await rm(target, { recursive: true, force: true });
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
throw error;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* One target's attempt at one prompt, in a throwaway home directory. Returns a
|
|
158
|
+
* validated report or the reason this target produced no usable verdict; the
|
|
159
|
+
* caller decides whether that reason is worth advancing past.
|
|
160
|
+
*/
|
|
161
|
+
async function attemptTarget(request, options) {
|
|
162
|
+
const { target } = request;
|
|
163
|
+
const skip = (reason, diagnostic, verb = "trying next target") => ({
|
|
164
|
+
kind: "skip",
|
|
165
|
+
reason,
|
|
166
|
+
diagnostic,
|
|
167
|
+
message: `${target}: ${reason} → ${verb} (${diagnostic})`
|
|
168
|
+
});
|
|
169
|
+
const attemptDirectory = await mkdtemp(join(tmpdir(), "agent-ops-review-"));
|
|
170
|
+
try {
|
|
171
|
+
const invocationRequest = {
|
|
172
|
+
target,
|
|
173
|
+
prompt: request.prompt,
|
|
174
|
+
repositoryRoot: request.repositoryRoot,
|
|
175
|
+
...(target === "agy"
|
|
176
|
+
? { logFile: join(attemptDirectory, "agy.log") }
|
|
177
|
+
: {}),
|
|
178
|
+
...(options.model === undefined ? {} : { model: options.model }),
|
|
179
|
+
...(options.effort === undefined ? {} : { effort: options.effort })
|
|
180
|
+
};
|
|
181
|
+
let invocation = buildTargetInvocation(invocationRequest);
|
|
182
|
+
let executionDirectory = attemptDirectory;
|
|
183
|
+
if (invocation === undefined) {
|
|
184
|
+
return skip("capability-unavailable", "no read-only mode is available for this target", "skipping");
|
|
185
|
+
}
|
|
186
|
+
const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
|
|
187
|
+
const capability = await runVerificationCommand({
|
|
188
|
+
id: `review-capability-${request.label}`,
|
|
189
|
+
command: invocation.command,
|
|
190
|
+
args: target === "codex" ? ["exec", "--help"] : ["--help"],
|
|
191
|
+
cwd: attemptDirectory,
|
|
192
|
+
required: true,
|
|
193
|
+
evidence: { kind: "exit-code" },
|
|
194
|
+
timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
|
|
195
|
+
}, {
|
|
196
|
+
cwd: attemptDirectory,
|
|
197
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
198
|
+
env: environment,
|
|
199
|
+
replaceEnv: true
|
|
200
|
+
});
|
|
201
|
+
const help = `${capability.stdout}\n${capability.stderr}`;
|
|
202
|
+
const missingFlags = (REQUIRED_HELP_FLAGS[target] ?? []).filter((flag) => !help.includes(flag));
|
|
203
|
+
if (capability.status !== "PASS" ||
|
|
204
|
+
capability.stdoutTruncated ||
|
|
205
|
+
capability.stderrTruncated ||
|
|
206
|
+
missingFlags.length > 0) {
|
|
207
|
+
// This gate is the one a renamed upstream flag trips, so it names the
|
|
208
|
+
// flags it could not find. Without them the skip is indistinguishable
|
|
209
|
+
// from an uninstalled CLI, in the human line and in the attempt record.
|
|
210
|
+
return skip("capability-unavailable", missingFlags.length > 0
|
|
211
|
+
? `help output is missing ${missingFlags.join(", ")}`
|
|
212
|
+
: capability.stdoutTruncated || capability.stderrTruncated
|
|
213
|
+
? "help output exceeded the capture limit"
|
|
214
|
+
: firstComplaint(capability.stderr, capability.stdout) ??
|
|
215
|
+
`help probe failed (${capability.failureClass})`, "skipping");
|
|
216
|
+
}
|
|
217
|
+
if (target === "agy") {
|
|
218
|
+
const snapshotRoot = join(attemptDirectory, "repository");
|
|
219
|
+
const snapshotError = await snapshotRepository(request, snapshotRoot, options);
|
|
220
|
+
if (snapshotError !== undefined) {
|
|
221
|
+
return skip("capability-unavailable", snapshotError, "skipping");
|
|
222
|
+
}
|
|
223
|
+
invocation = buildTargetInvocation({
|
|
224
|
+
...invocationRequest,
|
|
225
|
+
prompt: [
|
|
226
|
+
`Repository root: ${snapshotRoot}`,
|
|
227
|
+
"Run every repository-relative inspection in that directory.",
|
|
228
|
+
"For terminal commands, use only git status, git diff, git log, or git show; " +
|
|
229
|
+
"read specific files with file-reading tools instead of ls, find, cat, or rg.",
|
|
230
|
+
request.prompt
|
|
231
|
+
].join("\n"),
|
|
232
|
+
repositoryRoot: snapshotRoot
|
|
233
|
+
});
|
|
234
|
+
executionDirectory = snapshotRoot;
|
|
235
|
+
}
|
|
236
|
+
if (invocation === undefined) {
|
|
237
|
+
return skip("capability-unavailable", "review invocation disappeared", "skipping");
|
|
238
|
+
}
|
|
239
|
+
const spawned = await runVerificationCommand({
|
|
240
|
+
id: `review-${request.label}`,
|
|
241
|
+
command: invocation.command,
|
|
242
|
+
args: [...invocation.args],
|
|
243
|
+
cwd: executionDirectory,
|
|
244
|
+
required: true,
|
|
245
|
+
evidence: { kind: "exit-code" },
|
|
246
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
|
|
247
|
+
}, {
|
|
248
|
+
cwd: executionDirectory,
|
|
249
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
250
|
+
...(options.outputLimitBytes === undefined
|
|
251
|
+
? {}
|
|
252
|
+
: { outputLimitBytes: options.outputLimitBytes }),
|
|
253
|
+
stdin: invocation.stdin,
|
|
254
|
+
env: environment,
|
|
255
|
+
replaceEnv: true
|
|
256
|
+
});
|
|
257
|
+
if (ADVANCING.has(spawned.failureClass)) {
|
|
258
|
+
return {
|
|
259
|
+
...skip("missing-cli", `the process did not complete: ${spawned.failureClass}`),
|
|
260
|
+
...(spawned.failureClass === undefined
|
|
261
|
+
? {}
|
|
262
|
+
: { attemptReason: spawned.failureClass })
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
if (spawned.stdoutTruncated ||
|
|
266
|
+
(spawned.stderrTruncated && target !== "codex")) {
|
|
267
|
+
return skip("output-too-large", `${spawned.stdoutTruncated ? "stdout" : "stderr"} exceeded the capture limit`);
|
|
268
|
+
}
|
|
269
|
+
if (spawned.failureClass === "nonzero-exit") {
|
|
270
|
+
const output = `${spawned.stderr}\n${spawned.stdout}`;
|
|
271
|
+
return skip(rejectedCallReason(output), firstComplaint(spawned.stderr, spawned.stdout) ??
|
|
272
|
+
`the call was rejected with exit ${spawned.exitCode ?? "unknown"} and no output`);
|
|
273
|
+
}
|
|
274
|
+
const payload = extractReviewObject(target, spawned.stdout);
|
|
275
|
+
const parsed = payload === undefined
|
|
276
|
+
? undefined
|
|
277
|
+
: validateReviewReport(payload, request.expectedCriterionIds, request.changedFiles);
|
|
278
|
+
if (parsed === undefined || !parsed.ok) {
|
|
279
|
+
const reason = parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE") ? "incomplete-scope" : "unparseable-output";
|
|
280
|
+
// Which contract the answer broke, or what the target said instead of
|
|
281
|
+
// answering. Without this the skip names only the classification, and a
|
|
282
|
+
// target that runs but never returns a usable report is undebuggable.
|
|
283
|
+
const errors = parsed === undefined
|
|
284
|
+
? ""
|
|
285
|
+
: parsed.errors
|
|
286
|
+
.slice(0, 3)
|
|
287
|
+
.map((error) => `${error.path}: ${error.code}`)
|
|
288
|
+
.join("; ");
|
|
289
|
+
const fields = parsed?.errors.some((error) => error.code === "INVALID_FIELDS") &&
|
|
290
|
+
payload !== undefined
|
|
291
|
+
? ` (fields: ${Object.keys(payload).sort().join(", ")})`
|
|
292
|
+
: "";
|
|
293
|
+
return skip(reason, errors.length > 0
|
|
294
|
+
? `${errors}${fields}`
|
|
295
|
+
: firstComplaint(spawned.stdout, spawned.stderr) ??
|
|
296
|
+
"the answer carried no review report");
|
|
297
|
+
}
|
|
298
|
+
return { kind: "verdict", report: parsed.value };
|
|
299
|
+
}
|
|
300
|
+
finally {
|
|
301
|
+
await rm(attemptDirectory, { recursive: true, force: true });
|
|
302
|
+
}
|
|
303
|
+
}
|
|
72
304
|
/**
|
|
73
305
|
* Builds the `execute` callback `runIndependentReview` expects: walk the
|
|
74
|
-
* configured targets in order and return the first real verdict.
|
|
306
|
+
* configured targets in order and return the first real verdict. A PASS is then
|
|
307
|
+
* handed to a different target to refute, so a single agreeable reviewer cannot
|
|
308
|
+
* wave a change through on its own.
|
|
75
309
|
*/
|
|
76
310
|
export function createReviewExecutor(options) {
|
|
77
311
|
const report = options.onProgress ?? (() => { });
|
|
78
312
|
const host = detectHostTarget(options.env ?? process.env);
|
|
79
313
|
const chain = orderChain(options.targets, host);
|
|
80
314
|
return async (request) => {
|
|
81
|
-
const
|
|
315
|
+
const expectedCriterionIds = request.invocation.packet.criteria.map((criterion) => criterion.id);
|
|
82
316
|
const repositoryRoot = await realpath(options.cwd);
|
|
317
|
+
const shared = {
|
|
318
|
+
repositoryRoot,
|
|
319
|
+
expectedCriterionIds,
|
|
320
|
+
...(request.invocation.scope?.changedFiles === undefined
|
|
321
|
+
? {}
|
|
322
|
+
: { changedFiles: request.invocation.scope.changedFiles })
|
|
323
|
+
};
|
|
83
324
|
const attempts = [];
|
|
84
325
|
let lastReason = "missing-cli";
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
326
|
+
/**
|
|
327
|
+
* Ask a target other than the one that passed — and other than the host —
|
|
328
|
+
* to refute the verdict. Returns undefined when no such target produced a
|
|
329
|
+
* report, which leaves the primary PASS standing unchallenged. Targets the
|
|
330
|
+
* primary pass already walked past are excluded: a CLI that could not
|
|
331
|
+
* answer the review prompt will not answer this one either.
|
|
332
|
+
*/
|
|
333
|
+
const refute = async (primary, primaryTarget) => {
|
|
334
|
+
const walked = new Set(attempts.map((attempt) => attempt.target));
|
|
335
|
+
const candidates = chain.filter((target) => target !== primaryTarget && target !== host && !walked.has(target));
|
|
336
|
+
for (const [index, target] of candidates.entries()) {
|
|
337
|
+
const outcome = await attemptTarget({
|
|
338
|
+
...shared,
|
|
89
339
|
target,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
attempts.push({ target, status: "NOT_RUN", reason: lastReason });
|
|
98
|
-
report(`${target}: no read-only mode available → skipping`);
|
|
99
|
-
continue;
|
|
100
|
-
}
|
|
101
|
-
if (target === host) {
|
|
102
|
-
report(`${target}: reviewer == host; no independent target configured`);
|
|
103
|
-
}
|
|
104
|
-
const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
|
|
105
|
-
const capability = await runVerificationCommand({
|
|
106
|
-
id: `review-capability-${target}-${index}`,
|
|
107
|
-
command: invocation.command,
|
|
108
|
-
args: target === "codex" ? ["exec", "--help"] : ["--help"],
|
|
109
|
-
cwd: attemptDirectory,
|
|
110
|
-
required: true,
|
|
111
|
-
evidence: { kind: "exit-code" },
|
|
112
|
-
timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
|
|
113
|
-
}, {
|
|
114
|
-
cwd: attemptDirectory,
|
|
115
|
-
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
116
|
-
env: environment,
|
|
117
|
-
replaceEnv: true
|
|
118
|
-
});
|
|
119
|
-
if (capability.status !== "PASS" ||
|
|
120
|
-
capability.stdoutTruncated ||
|
|
121
|
-
capability.stderrTruncated ||
|
|
122
|
-
REQUIRED_HELP_FLAGS[target].some((flag) => !`${capability.stdout}\n${capability.stderr}`.includes(flag))) {
|
|
123
|
-
lastReason = "capability-unavailable";
|
|
124
|
-
attempts.push({ target, status: "NOT_RUN", reason: lastReason });
|
|
125
|
-
report(`${target}: required CLI capabilities unavailable → skipping`);
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
const spawned = await runVerificationCommand({
|
|
129
|
-
id: `review-${target}-${index}`,
|
|
130
|
-
command: invocation.command,
|
|
131
|
-
args: [...invocation.args],
|
|
132
|
-
cwd: attemptDirectory,
|
|
133
|
-
required: true,
|
|
134
|
-
evidence: { kind: "exit-code" },
|
|
135
|
-
timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
|
|
136
|
-
}, {
|
|
137
|
-
cwd: attemptDirectory,
|
|
138
|
-
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
139
|
-
...(options.outputLimitBytes === undefined
|
|
140
|
-
? {}
|
|
141
|
-
: { outputLimitBytes: options.outputLimitBytes }),
|
|
142
|
-
stdin: invocation.stdin,
|
|
143
|
-
env: environment,
|
|
144
|
-
replaceEnv: true
|
|
145
|
-
});
|
|
146
|
-
if (ADVANCING.has(spawned.failureClass)) {
|
|
147
|
-
lastReason = "missing-cli";
|
|
340
|
+
label: `adversarial-${target}-${index}`,
|
|
341
|
+
prompt: buildAdversarialPrompt({ ...request.invocation, harness: target }, primary)
|
|
342
|
+
}, options);
|
|
343
|
+
if (outcome.kind === "skip") {
|
|
344
|
+
// Recorded, not just reported: progress is suppressed under --json,
|
|
345
|
+
// and without this a PASS with no `adversarial` field cannot be told
|
|
346
|
+
// apart from a PASS that had no second target to challenge it.
|
|
148
347
|
attempts.push({
|
|
149
348
|
target,
|
|
150
349
|
status: "NOT_RUN",
|
|
151
|
-
reason:
|
|
350
|
+
reason: outcome.attemptReason ?? outcome.reason,
|
|
351
|
+
diagnostic: outcome.diagnostic
|
|
152
352
|
});
|
|
153
|
-
report(`${target}: ${
|
|
154
|
-
continue;
|
|
155
|
-
}
|
|
156
|
-
if (spawned.stdoutTruncated ||
|
|
157
|
-
(spawned.stderrTruncated && target !== "codex")) {
|
|
158
|
-
lastReason = "output-too-large";
|
|
159
|
-
attempts.push({ target, status: "NOT_RUN", reason: lastReason });
|
|
160
|
-
report(`${target}: ${lastReason} → trying next target`);
|
|
161
|
-
continue;
|
|
162
|
-
}
|
|
163
|
-
if (spawned.failureClass === "nonzero-exit") {
|
|
164
|
-
lastReason = "login-required";
|
|
165
|
-
attempts.push({ target, status: "NOT_RUN", reason: lastReason });
|
|
166
|
-
report(`${target}: ${lastReason} → trying next target`);
|
|
167
|
-
continue;
|
|
168
|
-
}
|
|
169
|
-
const payload = extractReviewObject(target, spawned.stdout);
|
|
170
|
-
const parsed = payload === undefined
|
|
171
|
-
? undefined
|
|
172
|
-
: validateReviewReport(payload, expected, request.invocation.scope?.changedFiles);
|
|
173
|
-
if (parsed === undefined || !parsed.ok) {
|
|
174
|
-
lastReason = parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE") ? "incomplete-scope" : "unparseable-output";
|
|
175
|
-
attempts.push({ target, status: "NOT_RUN", reason: lastReason });
|
|
176
|
-
report(`${target}: ${lastReason} → trying next target`);
|
|
353
|
+
report(`${target}: adversarial re-check unavailable (${outcome.reason})`);
|
|
177
354
|
continue;
|
|
178
355
|
}
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
356
|
+
const refuted = reviewReportStatus(outcome.report) === "FAIL";
|
|
357
|
+
report(`${target}: adversarial re-check ${refuted ? "refuted the PASS" : "upheld the PASS"}`);
|
|
358
|
+
return { target, refuted, report: outcome.report };
|
|
359
|
+
}
|
|
360
|
+
if (candidates.length > 0) {
|
|
361
|
+
report("no independent target completed an adversarial re-check");
|
|
362
|
+
}
|
|
363
|
+
return undefined;
|
|
364
|
+
};
|
|
365
|
+
for (const [index, target] of chain.entries()) {
|
|
366
|
+
if (target === host) {
|
|
367
|
+
report(`${target}: reviewer == host; no independent target configured`);
|
|
368
|
+
}
|
|
369
|
+
const outcome = await attemptTarget({
|
|
370
|
+
...shared,
|
|
371
|
+
target,
|
|
372
|
+
label: `${target}-${index}`,
|
|
373
|
+
prompt: buildReviewPrompt({ ...request.invocation, harness: target })
|
|
374
|
+
}, options);
|
|
375
|
+
if (outcome.kind === "skip") {
|
|
376
|
+
lastReason = outcome.reason;
|
|
377
|
+
attempts.push({
|
|
378
|
+
target,
|
|
379
|
+
status: "NOT_RUN",
|
|
380
|
+
reason: outcome.attemptReason ?? outcome.reason,
|
|
381
|
+
diagnostic: outcome.diagnostic
|
|
382
|
+
});
|
|
383
|
+
report(outcome.message);
|
|
384
|
+
continue;
|
|
195
385
|
}
|
|
196
|
-
|
|
197
|
-
|
|
386
|
+
const reportValue = outcome.report;
|
|
387
|
+
const status = reviewReportStatus(reportValue);
|
|
388
|
+
attempts.push({ target, status });
|
|
389
|
+
const independence = host === undefined
|
|
390
|
+
? "unknown"
|
|
391
|
+
: host === target
|
|
392
|
+
? "same-target"
|
|
393
|
+
: "different-target";
|
|
394
|
+
const verdict = {
|
|
395
|
+
results: reviewReportResults(reportValue),
|
|
396
|
+
report: reportValue,
|
|
397
|
+
harness: target,
|
|
398
|
+
attempts,
|
|
399
|
+
independence
|
|
400
|
+
};
|
|
401
|
+
if (status === "FAIL") {
|
|
402
|
+
return { status, ...verdict };
|
|
198
403
|
}
|
|
404
|
+
const adversarial = await refute(reportValue, target);
|
|
405
|
+
return {
|
|
406
|
+
status: adversarial?.refuted === true ? "FAIL" : "PASS",
|
|
407
|
+
...verdict,
|
|
408
|
+
...(adversarial === undefined ? {} : { adversarial })
|
|
409
|
+
};
|
|
199
410
|
}
|
|
200
411
|
return {
|
|
201
412
|
status: "NOT_RUN",
|
|
@@ -21,6 +21,11 @@ function parseObject(text) {
|
|
|
21
21
|
function isRecord(value) {
|
|
22
22
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
23
23
|
}
|
|
24
|
+
/** Agy appends plan UI metadata even when native JSON Schema is enabled. */
|
|
25
|
+
function withoutAgyPlanMetadata(value) {
|
|
26
|
+
const { toolAction: _toolAction, toolSummary: _toolSummary, ...report } = value;
|
|
27
|
+
return report;
|
|
28
|
+
}
|
|
24
29
|
/**
|
|
25
30
|
* The model's answer as text, before any JSON contract is applied. Returns
|
|
26
31
|
* undefined rather than throwing so the caller can report
|
|
@@ -63,7 +68,10 @@ export function extractReviewObject(target, stdout) {
|
|
|
63
68
|
const key = target === "claude" ? "structured_output" : "response";
|
|
64
69
|
const value = envelope?.[key];
|
|
65
70
|
if (isRecord(value)) {
|
|
66
|
-
return value;
|
|
71
|
+
return target === "agy" ? withoutAgyPlanMetadata(value) : value;
|
|
67
72
|
}
|
|
68
|
-
|
|
73
|
+
const parsed = typeof value === "string" ? extractJsonObject(value) : undefined;
|
|
74
|
+
return parsed === undefined || target !== "agy"
|
|
75
|
+
? parsed
|
|
76
|
+
: withoutAgyPlanMetadata(parsed);
|
|
69
77
|
}
|
|
@@ -11,8 +11,44 @@ function reviewSchemaPath() {
|
|
|
11
11
|
? source
|
|
12
12
|
: resolve(process.cwd(), "schemas", "review-report.schema.json");
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Removes every `pattern`. A target validates the schema with its own regex
|
|
16
|
+
* engine before it will run, and Go's RE2 — agy's — rejects constructs ECMA-262
|
|
17
|
+
* allows: it refused `/$defs/path` for a lookahead, then `/$defs/text` for a
|
|
18
|
+
* `\uXXXX` escape. Enumerating those differences is a losing game, and the
|
|
19
|
+
* schema handed to a target only shapes its answer: `validateReviewReport` is
|
|
20
|
+
* the authority and re-applies every pattern to whatever comes back, so an
|
|
21
|
+
* advisory constraint dropped here weakens nothing.
|
|
22
|
+
*/
|
|
23
|
+
function stripPatterns(value) {
|
|
24
|
+
if (Array.isArray(value)) {
|
|
25
|
+
for (const item of value) {
|
|
26
|
+
stripPatterns(item);
|
|
27
|
+
}
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (typeof value !== "object" || value === null) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const record = value;
|
|
34
|
+
delete record.pattern;
|
|
35
|
+
for (const item of Object.values(record)) {
|
|
36
|
+
stripPatterns(item);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The schema as a reviewer CLI will accept it. The file keeps its `$schema`
|
|
41
|
+
* declaration for this repository's own validation, but a target that resolves
|
|
42
|
+
* meta-schema references offline rejects the whole schema over it — claude
|
|
43
|
+
* answers `--json-schema is not a valid JSON Schema: no schema with key or ref
|
|
44
|
+
* "https://json-schema.org/draft/2020-12/schema"` and never starts. The draft
|
|
45
|
+
* declaration carries no constraint, so dropping it costs nothing.
|
|
46
|
+
*/
|
|
14
47
|
function reviewSchemaText() {
|
|
15
|
-
|
|
48
|
+
const parsed = JSON.parse(readFileSync(reviewSchemaPath(), "utf8"));
|
|
49
|
+
delete parsed.$schema;
|
|
50
|
+
stripPatterns(parsed);
|
|
51
|
+
return JSON.stringify(parsed);
|
|
16
52
|
}
|
|
17
53
|
/**
|
|
18
54
|
* Read-only enforcement per target, verified against each CLI's own help
|
|
@@ -20,12 +56,22 @@ function reviewSchemaText() {
|
|
|
20
56
|
* agent that can edit the code it is reviewing. This is what excludes
|
|
21
57
|
* opencode, whose `--agent plan` is rejected as a subagent and silently falls
|
|
22
58
|
* back to a writable agent.
|
|
59
|
+
*
|
|
60
|
+
* Agy can still mutate its cwd in sandboxed plan mode, so the executor points
|
|
61
|
+
* it at a disposable repository clone. Never combine this with
|
|
62
|
+
* `--dangerously-skip-permissions`, which overrides the permission boundary.
|
|
23
63
|
*/
|
|
24
64
|
export const READ_ONLY_ARGS = {
|
|
25
65
|
agy: ["--sandbox", "--mode", "plan"],
|
|
26
66
|
claude: ["--permission-mode", "plan"],
|
|
27
67
|
codex: ["-s", "read-only"]
|
|
28
68
|
};
|
|
69
|
+
/** Per-target customization suppression. */
|
|
70
|
+
function isolationArgs(target) {
|
|
71
|
+
return target === "claude"
|
|
72
|
+
? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
|
|
73
|
+
: [];
|
|
74
|
+
}
|
|
29
75
|
function modelArgs(target, model) {
|
|
30
76
|
if (model === undefined) {
|
|
31
77
|
return [];
|
|
@@ -73,15 +119,11 @@ export function buildTargetInvocation(request) {
|
|
|
73
119
|
stdin: request.prompt
|
|
74
120
|
};
|
|
75
121
|
}
|
|
76
|
-
const isolation = request.target === "claude"
|
|
77
|
-
? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
|
|
78
|
-
: request.target === "agy"
|
|
79
|
-
? ["--disable-slash-commands"]
|
|
80
|
-
: [];
|
|
81
122
|
return {
|
|
82
123
|
command: request.target,
|
|
83
124
|
args: [
|
|
84
125
|
"-p",
|
|
126
|
+
...(request.target === "agy" ? [request.prompt] : []),
|
|
85
127
|
"--output-format",
|
|
86
128
|
"json",
|
|
87
129
|
"--json-schema",
|
|
@@ -89,10 +131,15 @@ export function buildTargetInvocation(request) {
|
|
|
89
131
|
...(request.repositoryRoot === undefined
|
|
90
132
|
? []
|
|
91
133
|
: ["--add-dir", request.repositoryRoot]),
|
|
92
|
-
...
|
|
134
|
+
...(request.target !== "agy" || request.logFile === undefined
|
|
135
|
+
? []
|
|
136
|
+
: ["--log-file", request.logFile]),
|
|
137
|
+
...isolationArgs(request.target),
|
|
93
138
|
...shared
|
|
94
139
|
],
|
|
95
|
-
|
|
140
|
+
// Agy requires the prompt as the value of --print; a bare -p consumes the
|
|
141
|
+
// following flag. Claude accepts the prompt on stdin, keeping it out of ps.
|
|
142
|
+
stdin: request.target === "agy" ? "" : request.prompt
|
|
96
143
|
};
|
|
97
144
|
}
|
|
98
145
|
/** A deep doctor probe uses stdin but keeps its simple text response contract. */
|
|
@@ -111,14 +158,19 @@ export function buildProbeInvocation(request) {
|
|
|
111
158
|
stdin: request.prompt
|
|
112
159
|
};
|
|
113
160
|
}
|
|
114
|
-
const isolation = request.target === "claude"
|
|
115
|
-
? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
|
|
116
|
-
: request.target === "agy"
|
|
117
|
-
? ["--disable-slash-commands"]
|
|
118
|
-
: [];
|
|
119
161
|
return {
|
|
120
162
|
command: request.target,
|
|
121
|
-
args: [
|
|
122
|
-
|
|
163
|
+
args: [
|
|
164
|
+
"-p",
|
|
165
|
+
...(request.target === "agy" ? [request.prompt] : []),
|
|
166
|
+
"--output-format",
|
|
167
|
+
"json",
|
|
168
|
+
...(request.target !== "agy" || request.logFile === undefined
|
|
169
|
+
? []
|
|
170
|
+
: ["--log-file", request.logFile]),
|
|
171
|
+
...isolationArgs(request.target),
|
|
172
|
+
...readOnly
|
|
173
|
+
],
|
|
174
|
+
stdin: request.target === "agy" ? "" : request.prompt
|
|
123
175
|
};
|
|
124
176
|
}
|