@kylecheng3146/agent-ops 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/packages/cli/src/args.js +47 -0
- package/dist/packages/cli/src/bin.js +74 -25
- package/dist/packages/cli/src/cli.js +13 -1
- package/dist/packages/cli/src/commands/init.js +4 -1
- package/dist/packages/cli/src/commands/review.js +371 -27
- package/dist/packages/cli/src/commands/task.js +4 -1
- package/dist/packages/cli/src/commands/verify.js +13 -1
- package/dist/packages/cli/src/version.js +1 -1
- package/dist/packages/cli/src/wizard.js +62 -3
- package/dist/runtime/src/config/merge.js +17 -2
- package/dist/runtime/src/contracts.js +1 -1
- package/dist/runtime/src/install/doctor.js +42 -1
- package/dist/runtime/src/install/plan.js +11 -5
- package/dist/runtime/src/review/execute.js +180 -0
- package/dist/runtime/src/review/extract.js +69 -0
- package/dist/runtime/src/review/invocation.js +116 -0
- package/dist/runtime/src/review/packet.js +42 -5
- package/dist/runtime/src/review/probe.js +72 -0
- package/dist/runtime/src/review/render.js +62 -0
- package/dist/runtime/src/review/report.js +183 -0
- package/dist/runtime/src/review/result.js +2 -2
- package/dist/runtime/src/review/roles.js +35 -0
- package/dist/runtime/src/review/runner.js +98 -12
- package/dist/runtime/src/review/scope.js +123 -0
- package/dist/runtime/src/schema/validate.js +80 -0
- package/dist/runtime/src/task/service.js +46 -1
- package/dist/runtime/src/task/store.js +16 -4
- package/dist/runtime/src/verify/change-surface.js +38 -2
- package/dist/runtime/src/verify/command-executor.js +4 -1
- package/dist/runtime/src/verify/evidence.js +36 -0
- package/dist/runtime/src/verify/scope.js +1 -2
- package/dist/runtime/src/verify/service.js +66 -9
- package/dist/runtime/src/verify/source-fingerprint.js +49 -0
- package/dist/runtime/src/verify/spawn.js +9 -3
- package/docs/en/guides/configuration.md +68 -0
- package/docs/en/spec/review.md +37 -4
- package/docs/zh-TW/guides/configuration.md +60 -0
- package/docs/zh-TW/spec/review.md +33 -3
- package/package.json +1 -1
- package/schemas/config.schema.json +29 -0
- package/schemas/evidence.schema.json +16 -1
- package/schemas/review-report.schema.json +48 -0
|
@@ -338,6 +338,46 @@ async function checkRegistrationDrift(root, manifest, config) {
|
|
|
338
338
|
return check("registration-drift", "UNKNOWN", "Hook registration drift could not be assessed safely.");
|
|
339
339
|
}
|
|
340
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Guidance lives in `message` rather than a `remediation` field: as of this
|
|
343
|
+
* check, `remediation` does not exist on DoctorCheck. Because target
|
|
344
|
+
* authentication failures surface as one unexplained review failure — the
|
|
345
|
+
* chain deliberately does not sniff stderr for "not logged in" — this text is
|
|
346
|
+
* the operator's only route out, so it names the exact command.
|
|
347
|
+
*/
|
|
348
|
+
async function checkReviewTargets(config, probe, checkAuth) {
|
|
349
|
+
const targets = config?.reviewRoles?.find((role) => role.role === "independent-review")?.targets ?? [];
|
|
350
|
+
if (targets.length === 0) {
|
|
351
|
+
return check("review-targets", "PASS", "External review disabled. Re-run agent-ops init to enable.");
|
|
352
|
+
}
|
|
353
|
+
if (probe === undefined) {
|
|
354
|
+
return check("review-targets", "PASS", `External review targets: ${targets.join(", ")}. ` +
|
|
355
|
+
"Login state unverified; run: agent-ops doctor --check-auth");
|
|
356
|
+
}
|
|
357
|
+
for (const target of targets) {
|
|
358
|
+
const result = await probe(target, checkAuth);
|
|
359
|
+
if (result === "missing-executable") {
|
|
360
|
+
return check("review-targets", "FAIL", `${target} not found. Install it, or remove "${target}" from ` +
|
|
361
|
+
"reviewRoles[].targets.", "UPDATE_REQUIRED");
|
|
362
|
+
}
|
|
363
|
+
if (result === "ineligible") {
|
|
364
|
+
return check("review-targets", "FAIL", `${target} has no read-only mode and cannot review. Remove ` +
|
|
365
|
+
`"${target}" from reviewRoles[].targets.`, "UPDATE_REQUIRED");
|
|
366
|
+
}
|
|
367
|
+
if (result === "timeout") {
|
|
368
|
+
return check("review-targets", "FAIL", `${target} did not answer in time. Re-run: ` +
|
|
369
|
+
"agent-ops doctor --check-auth", "UPDATE_REQUIRED");
|
|
370
|
+
}
|
|
371
|
+
if (checkAuth && result !== "ok") {
|
|
372
|
+
return check("review-targets", "FAIL", `${target} is installed but not authenticated, or it rejected the ` +
|
|
373
|
+
`call. Run: ${target} login`, "UPDATE_REQUIRED");
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
return check("review-targets", "PASS", checkAuth
|
|
377
|
+
? `External review targets authenticated: ${targets.join(", ")}.`
|
|
378
|
+
: `External review targets: ${targets.join(", ")}. ` +
|
|
379
|
+
"Login state unverified; run: agent-ops doctor --check-auth");
|
|
380
|
+
}
|
|
341
381
|
export async function doctorInstallation(options) {
|
|
342
382
|
const manifest = await checkManifest(options.root);
|
|
343
383
|
const config = await checkConfig(options.root);
|
|
@@ -355,7 +395,8 @@ export async function doctorInstallation(options) {
|
|
|
355
395
|
await checkProbe("hook-registration", options.probes?.hookRegistration),
|
|
356
396
|
checkLifecycleSummary(manifest.manifest, config.config),
|
|
357
397
|
await checkProbe("repository-trust", options.probes?.repositoryTrust),
|
|
358
|
-
await checkProbe("smoke-availability", options.probes?.smokeAvailability)
|
|
398
|
+
await checkProbe("smoke-availability", options.probes?.smokeAvailability),
|
|
399
|
+
await checkReviewTargets(config.config, options.probes?.reviewTarget, options.checkReviewTargetAuth === true)
|
|
359
400
|
];
|
|
360
401
|
return {
|
|
361
402
|
checks,
|
|
@@ -40,7 +40,12 @@ async function readCurrentFile(root, path) {
|
|
|
40
40
|
throw error;
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
|
-
function formatConfig(profiles, existing) {
|
|
43
|
+
function formatConfig(profiles, existing, reviewTargets = []) {
|
|
44
|
+
// Absent reviewRoles means external review is disabled; an empty selection
|
|
45
|
+
// must therefore omit the field rather than write an empty array.
|
|
46
|
+
const reviewRoles = reviewTargets.length > 0
|
|
47
|
+
? [{ role: "independent-review", targets: [...reviewTargets] }]
|
|
48
|
+
: existing?.reviewRoles;
|
|
44
49
|
return `${JSON.stringify({
|
|
45
50
|
schemaVersion: CONFIG_SCHEMA_VERSION,
|
|
46
51
|
profiles,
|
|
@@ -51,10 +56,11 @@ function formatConfig(profiles, existing) {
|
|
|
51
56
|
}
|
|
52
57
|
},
|
|
53
58
|
pathMappings: existing?.pathMappings ?? [],
|
|
54
|
-
securityExceptions: existing?.securityExceptions ?? []
|
|
59
|
+
securityExceptions: existing?.securityExceptions ?? [],
|
|
60
|
+
...(reviewRoles === undefined ? {} : { reviewRoles })
|
|
55
61
|
}, null, 2)}\n`;
|
|
56
62
|
}
|
|
57
|
-
async function planConfig(root, profiles, existingManifest, suppliedConfig) {
|
|
63
|
+
async function planConfig(root, profiles, existingManifest, suppliedConfig, reviewTargets = []) {
|
|
58
64
|
const current = await readCurrentFile(root, CONFIG_PATH);
|
|
59
65
|
const owned = findOwnedArtifact(existingManifest, CONFIG_PATH);
|
|
60
66
|
if (current !== null && owned === undefined) {
|
|
@@ -86,7 +92,7 @@ async function planConfig(root, profiles, existingManifest, suppliedConfig) {
|
|
|
86
92
|
}
|
|
87
93
|
existingConfig = result.value;
|
|
88
94
|
}
|
|
89
|
-
const content = formatConfig(profiles, existingConfig);
|
|
95
|
+
const content = formatConfig(profiles, existingConfig, reviewTargets);
|
|
90
96
|
return {
|
|
91
97
|
operation: {
|
|
92
98
|
kind: "write",
|
|
@@ -413,7 +419,7 @@ export async function createInstallPlan(options) {
|
|
|
413
419
|
: [];
|
|
414
420
|
const operations = [];
|
|
415
421
|
const artifacts = [];
|
|
416
|
-
const config = await planConfig(options.root, resolved.profiles, existing?.manifest ?? null, options.existingConfig);
|
|
422
|
+
const config = await planConfig(options.root, resolved.profiles, existing?.manifest ?? null, options.existingConfig, options.reviewTargets ?? []);
|
|
417
423
|
operations.push(config.operation);
|
|
418
424
|
artifacts.push(config.record);
|
|
419
425
|
for (const artifact of contribution.artifacts) {
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { mkdtemp, realpath, rm } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { runVerificationCommand } from "../verify/spawn.js";
|
|
5
|
+
import { extractReviewObject } from "./extract.js";
|
|
6
|
+
import { buildTargetInvocation } from "./invocation.js";
|
|
7
|
+
import { reviewReportResults, reviewReportStatus, validateReviewReport } from "./report.js";
|
|
8
|
+
import { detectHostTarget, orderChain } from "./roles.js";
|
|
9
|
+
import { buildReviewPrompt } from "./runner.js";
|
|
10
|
+
/**
|
|
11
|
+
* Deliberately below the five-minute `spawn.ts` default: a timeout advances the
|
|
12
|
+
* chain, so the worst case is targets x timeout.
|
|
13
|
+
*/
|
|
14
|
+
export const DEFAULT_REVIEW_TIMEOUT_MS = 120_000;
|
|
15
|
+
const EXECUTION_ENV = [
|
|
16
|
+
"PATH", "PATHEXT", "SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC",
|
|
17
|
+
"LANG", "LC_ALL", "TERM", "TMPDIR", "TEMP", "TMP"
|
|
18
|
+
];
|
|
19
|
+
const AUTH_ENV = {
|
|
20
|
+
claude: ["ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN"],
|
|
21
|
+
codex: ["OPENAI_API_KEY"],
|
|
22
|
+
agy: ["AGY_API_KEY"]
|
|
23
|
+
};
|
|
24
|
+
export function isolatedReviewEnvironment(target, directory, source) {
|
|
25
|
+
const env = {};
|
|
26
|
+
for (const key of [...EXECUTION_ENV, ...AUTH_ENV[target]]) {
|
|
27
|
+
const value = source[key];
|
|
28
|
+
if (value !== undefined) {
|
|
29
|
+
env[key] = value;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
env.HOME = directory;
|
|
33
|
+
env.USERPROFILE = directory;
|
|
34
|
+
env.XDG_CONFIG_HOME = join(directory, "config");
|
|
35
|
+
env.XDG_CACHE_HOME = join(directory, "cache");
|
|
36
|
+
return env;
|
|
37
|
+
}
|
|
38
|
+
/** Codex and agy currently lack documented instruction/customization isolation. */
|
|
39
|
+
export function hasRequiredReviewIsolation(target) {
|
|
40
|
+
return target === "claude";
|
|
41
|
+
}
|
|
42
|
+
const REQUIRED_HELP_FLAGS = {
|
|
43
|
+
claude: [
|
|
44
|
+
"--add-dir", "--permission-mode", "--no-session-persistence",
|
|
45
|
+
"--safe-mode", "--disable-slash-commands", "--json-schema"
|
|
46
|
+
],
|
|
47
|
+
codex: [],
|
|
48
|
+
agy: []
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Failure classes that mean no review happened, so trying the next target is
|
|
52
|
+
* not review shopping. Everything else — including FAIL — is terminal.
|
|
53
|
+
*/
|
|
54
|
+
const ADVANCING = new Set([
|
|
55
|
+
"missing-executable",
|
|
56
|
+
"spawn-failed",
|
|
57
|
+
"timeout"
|
|
58
|
+
]);
|
|
59
|
+
/**
|
|
60
|
+
* Builds the `execute` callback `runIndependentReview` expects: walk the
|
|
61
|
+
* configured targets in order and return the first real verdict.
|
|
62
|
+
*/
|
|
63
|
+
export function createReviewExecutor(options) {
|
|
64
|
+
const report = options.onProgress ?? (() => { });
|
|
65
|
+
const host = detectHostTarget(options.env ?? process.env);
|
|
66
|
+
const chain = orderChain(options.targets, host);
|
|
67
|
+
return async (request) => {
|
|
68
|
+
const expected = request.invocation.packet.criteria.map((criterion) => criterion.id);
|
|
69
|
+
const prompt = buildReviewPrompt(request.invocation);
|
|
70
|
+
const repositoryRoot = await realpath(options.cwd);
|
|
71
|
+
let unavailable = false;
|
|
72
|
+
for (const [index, target] of chain.entries()) {
|
|
73
|
+
if (!hasRequiredReviewIsolation(target)) {
|
|
74
|
+
unavailable = true;
|
|
75
|
+
report(`${target}: required context-isolation controls unavailable → skipping`);
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const attemptDirectory = await mkdtemp(join(tmpdir(), "agent-ops-review-"));
|
|
79
|
+
try {
|
|
80
|
+
const invocation = buildTargetInvocation({
|
|
81
|
+
target,
|
|
82
|
+
prompt,
|
|
83
|
+
repositoryRoot,
|
|
84
|
+
...(options.model === undefined ? {} : { model: options.model }),
|
|
85
|
+
...(options.effort === undefined ? {} : { effort: options.effort })
|
|
86
|
+
});
|
|
87
|
+
if (invocation === undefined) {
|
|
88
|
+
unavailable = true;
|
|
89
|
+
report(`${target}: no read-only mode available → skipping`);
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (target === host) {
|
|
93
|
+
report(`${target}: reviewer == host; no independent target configured`);
|
|
94
|
+
}
|
|
95
|
+
const environment = isolatedReviewEnvironment(target, attemptDirectory, options.env ?? process.env);
|
|
96
|
+
const capability = await runVerificationCommand({
|
|
97
|
+
id: `review-capability-${target}-${index}`,
|
|
98
|
+
command: invocation.command,
|
|
99
|
+
args: ["--help"],
|
|
100
|
+
cwd: attemptDirectory,
|
|
101
|
+
required: true,
|
|
102
|
+
evidence: { kind: "exit-code" },
|
|
103
|
+
timeoutMs: Math.min(options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS, 10_000)
|
|
104
|
+
}, {
|
|
105
|
+
cwd: attemptDirectory,
|
|
106
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
107
|
+
env: environment,
|
|
108
|
+
replaceEnv: true
|
|
109
|
+
});
|
|
110
|
+
if (capability.status !== "PASS" ||
|
|
111
|
+
capability.stdoutTruncated ||
|
|
112
|
+
capability.stderrTruncated ||
|
|
113
|
+
REQUIRED_HELP_FLAGS[target].some((flag) => !capability.stdout.includes(flag))) {
|
|
114
|
+
unavailable = true;
|
|
115
|
+
report(`${target}: required CLI capabilities unavailable → skipping`);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const spawned = await runVerificationCommand({
|
|
119
|
+
id: `review-${target}-${index}`,
|
|
120
|
+
command: invocation.command,
|
|
121
|
+
args: [...invocation.args],
|
|
122
|
+
cwd: attemptDirectory,
|
|
123
|
+
required: true,
|
|
124
|
+
evidence: { kind: "exit-code" },
|
|
125
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS
|
|
126
|
+
}, {
|
|
127
|
+
cwd: attemptDirectory,
|
|
128
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
129
|
+
...(options.outputLimitBytes === undefined
|
|
130
|
+
? {}
|
|
131
|
+
: { outputLimitBytes: options.outputLimitBytes }),
|
|
132
|
+
stdin: invocation.stdin,
|
|
133
|
+
env: environment,
|
|
134
|
+
replaceEnv: true
|
|
135
|
+
});
|
|
136
|
+
if (ADVANCING.has(spawned.failureClass)) {
|
|
137
|
+
report(`${target}: ${spawned.failureClass} → trying next target`);
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
if (spawned.stdoutTruncated || spawned.stderrTruncated) {
|
|
141
|
+
return { status: "NOT_RUN", reason: "output-too-large", harness: target };
|
|
142
|
+
}
|
|
143
|
+
const payload = extractReviewObject(target, spawned.stdout);
|
|
144
|
+
const parsed = payload === undefined
|
|
145
|
+
? undefined
|
|
146
|
+
: validateReviewReport(payload, expected, request.invocation.scope?.changedFiles);
|
|
147
|
+
if (parsed === undefined || !parsed.ok) {
|
|
148
|
+
return {
|
|
149
|
+
status: "NOT_RUN",
|
|
150
|
+
reason: parsed?.errors.some((error) => error.code === "INCOMPLETE_SCOPE")
|
|
151
|
+
? "incomplete-scope"
|
|
152
|
+
: "unparseable-output",
|
|
153
|
+
harness: target,
|
|
154
|
+
...(parsed === undefined ? {} : { validationErrors: parsed.errors })
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
const reportValue = parsed.value;
|
|
158
|
+
const results = reviewReportResults(reportValue);
|
|
159
|
+
return {
|
|
160
|
+
status: reviewReportStatus(reportValue),
|
|
161
|
+
results,
|
|
162
|
+
report: reportValue,
|
|
163
|
+
harness: target,
|
|
164
|
+
independence: host === undefined
|
|
165
|
+
? "unknown"
|
|
166
|
+
: host === target
|
|
167
|
+
? "same-target"
|
|
168
|
+
: "different-target"
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
await rm(attemptDirectory, { recursive: true, force: true });
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return {
|
|
176
|
+
status: "NOT_RUN",
|
|
177
|
+
reason: unavailable ? "capability-unavailable" : "missing-cli"
|
|
178
|
+
};
|
|
179
|
+
};
|
|
180
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The key holding the model's answer differs in every envelope, so every
|
|
3
|
+
* target gets its own branch. Verified against tests/fixtures/review/.
|
|
4
|
+
*/
|
|
5
|
+
const ENVELOPE_KEYS = {
|
|
6
|
+
agy: "response",
|
|
7
|
+
claude: "result"
|
|
8
|
+
};
|
|
9
|
+
function parseObject(text) {
|
|
10
|
+
let parsed;
|
|
11
|
+
try {
|
|
12
|
+
parsed = JSON.parse(text);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)
|
|
18
|
+
? parsed
|
|
19
|
+
: undefined;
|
|
20
|
+
}
|
|
21
|
+
function isRecord(value) {
|
|
22
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The model's answer as text, before any JSON contract is applied. Returns
|
|
26
|
+
* undefined rather than throwing so the caller can report
|
|
27
|
+
* `unparseable-output` for every transport failure through one path.
|
|
28
|
+
*/
|
|
29
|
+
export function extractFinalMessage(target, stdout) {
|
|
30
|
+
const key = ENVELOPE_KEYS[target];
|
|
31
|
+
if (key === undefined) {
|
|
32
|
+
// codex: stdout is the final message itself.
|
|
33
|
+
const trimmed = stdout.trim();
|
|
34
|
+
return trimmed.length === 0 ? undefined : trimmed;
|
|
35
|
+
}
|
|
36
|
+
const envelope = parseObject(stdout);
|
|
37
|
+
const value = envelope?.[key];
|
|
38
|
+
if (typeof value !== "string") {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
const trimmed = value.trim();
|
|
42
|
+
return trimmed.length === 0 ? undefined : trimmed;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The last balanced JSON object in a block of model text. Scanning backwards
|
|
46
|
+
* matters: models often restate the schema before answering, and the answer is
|
|
47
|
+
* what comes last. This only ever runs on the extracted final message, never on
|
|
48
|
+
* raw stdout, so it cannot capture a transport envelope.
|
|
49
|
+
*/
|
|
50
|
+
export function extractJsonObject(text) {
|
|
51
|
+
return parseObject(text.trim());
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Review results use a strict native structured-output transport. Unlike the
|
|
55
|
+
* legacy probe parser above, this path neither recovers fenced JSON nor accepts
|
|
56
|
+
* a provider's generic text result field.
|
|
57
|
+
*/
|
|
58
|
+
export function extractReviewObject(target, stdout) {
|
|
59
|
+
if (target === "codex") {
|
|
60
|
+
return extractJsonObject(stdout);
|
|
61
|
+
}
|
|
62
|
+
const envelope = parseObject(stdout);
|
|
63
|
+
const key = target === "claude" ? "structured_output" : "response";
|
|
64
|
+
const value = envelope?.[key];
|
|
65
|
+
if (isRecord(value)) {
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
return typeof value === "string" ? extractJsonObject(value) : undefined;
|
|
69
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
function reviewSchemaPath() {
|
|
5
|
+
const packaged = fileURLToPath(new URL("../../../../schemas/review-report.schema.json", import.meta.url));
|
|
6
|
+
if (existsSync(packaged)) {
|
|
7
|
+
return packaged;
|
|
8
|
+
}
|
|
9
|
+
const source = fileURLToPath(new URL("../../../schemas/review-report.schema.json", import.meta.url));
|
|
10
|
+
return existsSync(source)
|
|
11
|
+
? source
|
|
12
|
+
: resolve(process.cwd(), "schemas", "review-report.schema.json");
|
|
13
|
+
}
|
|
14
|
+
function reviewSchemaText() {
|
|
15
|
+
return readFileSync(reviewSchemaPath(), "utf8");
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Read-only enforcement per target, verified against each CLI's own help
|
|
19
|
+
* output. A target absent from this table is ineligible: review never runs an
|
|
20
|
+
* agent that can edit the code it is reviewing. This is what excludes
|
|
21
|
+
* opencode, whose `--agent plan` is rejected as a subagent and silently falls
|
|
22
|
+
* back to a writable agent.
|
|
23
|
+
*/
|
|
24
|
+
export const READ_ONLY_ARGS = {
|
|
25
|
+
agy: ["--sandbox", "--mode", "plan"],
|
|
26
|
+
claude: ["--permission-mode", "plan"],
|
|
27
|
+
codex: ["-s", "read-only"]
|
|
28
|
+
};
|
|
29
|
+
function modelArgs(target, model) {
|
|
30
|
+
if (model === undefined) {
|
|
31
|
+
return [];
|
|
32
|
+
}
|
|
33
|
+
return target === "codex" ? ["-m", model] : ["--model", model];
|
|
34
|
+
}
|
|
35
|
+
function effortArgs(target, effort) {
|
|
36
|
+
if (effort === undefined) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
// codex has no --effort flag; reasoning effort is a config override.
|
|
40
|
+
return target === "codex"
|
|
41
|
+
? ["-c", `model_reasoning_effort=${effort}`]
|
|
42
|
+
: ["--effort", effort];
|
|
43
|
+
}
|
|
44
|
+
export function buildTargetInvocation(request) {
|
|
45
|
+
const readOnly = READ_ONLY_ARGS[request.target];
|
|
46
|
+
if (readOnly === undefined || readOnly.length === 0) {
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
const shared = [
|
|
50
|
+
...readOnly,
|
|
51
|
+
...modelArgs(request.target, request.model),
|
|
52
|
+
...effortArgs(request.target, request.effort)
|
|
53
|
+
];
|
|
54
|
+
if (request.target === "codex") {
|
|
55
|
+
// codex writes progress to stderr and leaves stdout as the bare final
|
|
56
|
+
// message, so it needs no output-format flag and no scratch file.
|
|
57
|
+
// Without --skip-git-repo-check it refuses to run outside a trusted git
|
|
58
|
+
// directory, which a caller would otherwise read as "not authenticated".
|
|
59
|
+
return {
|
|
60
|
+
command: "codex",
|
|
61
|
+
args: [
|
|
62
|
+
"exec",
|
|
63
|
+
"-",
|
|
64
|
+
"--skip-git-repo-check",
|
|
65
|
+
"--output-schema",
|
|
66
|
+
reviewSchemaPath(),
|
|
67
|
+
...(request.repositoryRoot === undefined
|
|
68
|
+
? []
|
|
69
|
+
: ["--add-dir", request.repositoryRoot, "--ephemeral", "--ignore-rules"]),
|
|
70
|
+
...shared
|
|
71
|
+
],
|
|
72
|
+
stdin: request.prompt
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const isolation = request.target === "claude"
|
|
76
|
+
? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
|
|
77
|
+
: [];
|
|
78
|
+
return {
|
|
79
|
+
command: request.target,
|
|
80
|
+
args: [
|
|
81
|
+
"-p",
|
|
82
|
+
"--output-format",
|
|
83
|
+
"json",
|
|
84
|
+
"--json-schema",
|
|
85
|
+
reviewSchemaText(),
|
|
86
|
+
...(request.repositoryRoot === undefined
|
|
87
|
+
? []
|
|
88
|
+
: ["--add-dir", request.repositoryRoot]),
|
|
89
|
+
...isolation,
|
|
90
|
+
...shared
|
|
91
|
+
],
|
|
92
|
+
stdin: request.prompt
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** A deep doctor probe uses stdin but keeps its simple text response contract. */
|
|
96
|
+
export function buildProbeInvocation(request) {
|
|
97
|
+
const readOnly = READ_ONLY_ARGS[request.target];
|
|
98
|
+
if (readOnly === undefined || readOnly.length === 0) {
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
101
|
+
if (request.target === "codex") {
|
|
102
|
+
return {
|
|
103
|
+
command: "codex",
|
|
104
|
+
args: ["exec", "-", "--skip-git-repo-check", ...readOnly],
|
|
105
|
+
stdin: request.prompt
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const isolation = request.target === "claude"
|
|
109
|
+
? ["--no-session-persistence", "--safe-mode", "--disable-slash-commands"]
|
|
110
|
+
: [];
|
|
111
|
+
return {
|
|
112
|
+
command: request.target,
|
|
113
|
+
args: ["-p", "--output-format", "json", ...isolation, ...readOnly],
|
|
114
|
+
stdin: request.prompt
|
|
115
|
+
};
|
|
116
|
+
}
|
|
@@ -1,10 +1,47 @@
|
|
|
1
|
+
const MAX_PACKET_BYTES = 64 * 1024;
|
|
2
|
+
function safe(value) {
|
|
3
|
+
return safeTaskText(redactSecrets(value));
|
|
4
|
+
}
|
|
5
|
+
function checkSensitive(value) {
|
|
6
|
+
const decision = evaluateGuardrail({
|
|
7
|
+
kind: "content",
|
|
8
|
+
content: value,
|
|
9
|
+
scope: "review-packet"
|
|
10
|
+
});
|
|
11
|
+
if (decision.action === "block") {
|
|
12
|
+
throw new AgentOpsError("REVIEW_SENSITIVE_INPUT", "Review input contains credential-shaped content.");
|
|
13
|
+
}
|
|
14
|
+
}
|
|
1
15
|
export function buildReviewPacket(input) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
16
|
+
for (const value of [
|
|
17
|
+
input.request,
|
|
18
|
+
...input.criteria.flatMap((criterion) => [criterion.id, criterion.description, ...(criterion.verifierIds ?? [])]),
|
|
19
|
+
...input.artifactRefs,
|
|
20
|
+
...input.evidenceRequirements.flatMap((requirement) => [requirement.criterionId, requirement.requirement])
|
|
21
|
+
]) {
|
|
22
|
+
checkSensitive(value);
|
|
23
|
+
}
|
|
24
|
+
const packet = {
|
|
25
|
+
request: safe(input.request),
|
|
26
|
+
criteria: input.criteria.map((criterion) => ({
|
|
27
|
+
id: safe(criterion.id),
|
|
28
|
+
description: safe(criterion.description),
|
|
29
|
+
...(criterion.verifierIds === undefined
|
|
30
|
+
? {}
|
|
31
|
+
: { verifierIds: criterion.verifierIds.map(safe) })
|
|
32
|
+
})),
|
|
33
|
+
artifactRefs: input.artifactRefs.map(safe),
|
|
6
34
|
evidenceRequirements: input.evidenceRequirements.map((requirement) => ({
|
|
7
|
-
|
|
35
|
+
criterionId: safe(requirement.criterionId),
|
|
36
|
+
requirement: safe(requirement.requirement)
|
|
8
37
|
}))
|
|
9
38
|
};
|
|
39
|
+
if (Buffer.byteLength(JSON.stringify(packet), "utf8") > MAX_PACKET_BYTES) {
|
|
40
|
+
throw new AgentOpsError("REVIEW_SCOPE_TOO_LARGE", "Review packet exceeds the 64 KiB limit.");
|
|
41
|
+
}
|
|
42
|
+
return packet;
|
|
10
43
|
}
|
|
44
|
+
import { evaluateGuardrail } from "../guardrails/evaluate.js";
|
|
45
|
+
import { AgentOpsError } from "../fs/paths.js";
|
|
46
|
+
import { redactSecrets } from "../security/redact.js";
|
|
47
|
+
import { safeTaskText } from "../task/render.js";
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { runVerificationCommand } from "../verify/spawn.js";
|
|
2
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { extractFinalMessage } from "./extract.js";
|
|
6
|
+
import { hasRequiredReviewIsolation, isolatedReviewEnvironment } from "./execute.js";
|
|
7
|
+
import { buildProbeInvocation } from "./invocation.js";
|
|
8
|
+
const PROBE_PROMPT = "Reply with the single word OK and nothing else.";
|
|
9
|
+
/**
|
|
10
|
+
* Matches the review timeout rather than being "quick": codex at high
|
|
11
|
+
* reasoning effort answers a trivial prompt in ~20s, and a probe that times out
|
|
12
|
+
* would otherwise be reported as an authentication failure.
|
|
13
|
+
*/
|
|
14
|
+
const PROBE_TIMEOUT_MS = 120_000;
|
|
15
|
+
/**
|
|
16
|
+
* The only check that actually proves a target is usable: ask it something
|
|
17
|
+
* trivial and see whether an answer comes back. A credential-file check can
|
|
18
|
+
* pass while the token is expired, and self-declaration ("already logged in?")
|
|
19
|
+
* is not evidence at all.
|
|
20
|
+
*/
|
|
21
|
+
export async function probeReviewTarget(target, options) {
|
|
22
|
+
const deep = options.deep === true;
|
|
23
|
+
if (deep && !hasRequiredReviewIsolation(target)) {
|
|
24
|
+
return "ineligible";
|
|
25
|
+
}
|
|
26
|
+
const invocation = buildProbeInvocation({ target, prompt: PROBE_PROMPT });
|
|
27
|
+
if (invocation === undefined) {
|
|
28
|
+
return "ineligible";
|
|
29
|
+
}
|
|
30
|
+
const directory = deep
|
|
31
|
+
? await mkdtemp(join(tmpdir(), "agent-ops-review-probe-"))
|
|
32
|
+
: options.cwd;
|
|
33
|
+
try {
|
|
34
|
+
const spawned = await runVerificationCommand({
|
|
35
|
+
id: `review-probe-${target}`,
|
|
36
|
+
command: invocation.command,
|
|
37
|
+
args: deep ? [...invocation.args] : ["--version"],
|
|
38
|
+
cwd: directory,
|
|
39
|
+
required: true,
|
|
40
|
+
evidence: { kind: "exit-code" },
|
|
41
|
+
timeoutMs: options.timeoutMs ?? PROBE_TIMEOUT_MS
|
|
42
|
+
}, {
|
|
43
|
+
cwd: directory,
|
|
44
|
+
...(options.runner === undefined ? {} : { runner: options.runner }),
|
|
45
|
+
...(deep
|
|
46
|
+
? {
|
|
47
|
+
stdin: invocation.stdin,
|
|
48
|
+
env: isolatedReviewEnvironment(target, directory, process.env),
|
|
49
|
+
replaceEnv: true
|
|
50
|
+
}
|
|
51
|
+
: {})
|
|
52
|
+
});
|
|
53
|
+
if (spawned.failureClass === "missing-executable") {
|
|
54
|
+
return "missing-executable";
|
|
55
|
+
}
|
|
56
|
+
if (spawned.timedOut) {
|
|
57
|
+
return "timeout";
|
|
58
|
+
}
|
|
59
|
+
if (!deep) {
|
|
60
|
+
return spawned.status === "PASS" ? "ok" : "unauthenticated";
|
|
61
|
+
}
|
|
62
|
+
return spawned.status === "PASS" &&
|
|
63
|
+
extractFinalMessage(target, spawned.stdout) !== undefined
|
|
64
|
+
? "ok"
|
|
65
|
+
: "unauthenticated";
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
if (deep) {
|
|
69
|
+
await rm(directory, { recursive: true, force: true });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { redactSecrets } from "../security/redact.js";
|
|
2
|
+
import { safeTaskText } from "../task/render.js";
|
|
3
|
+
function safe(value) {
|
|
4
|
+
return safeTaskText(redactSecrets(value));
|
|
5
|
+
}
|
|
6
|
+
function lineList(values) {
|
|
7
|
+
return values.length === 0 ? ["- none"] : values.map((value) => `- ${safe(value)}`);
|
|
8
|
+
}
|
|
9
|
+
export function renderReviewResult(result) {
|
|
10
|
+
const lines = [
|
|
11
|
+
`Independent review: ${result.status}`,
|
|
12
|
+
`Reviewer: ${result.harness}; model: ${safe(result.model)}; effort: ${safe(result.effort)}.`
|
|
13
|
+
];
|
|
14
|
+
if (result.scope !== undefined) {
|
|
15
|
+
lines.push(result.scope.mode === "base"
|
|
16
|
+
? `Scope: ${result.scope.mode} ${safe(result.scope.baseRef)} (${safe(result.scope.resolvedBase)}).`
|
|
17
|
+
: "Scope: worktree.");
|
|
18
|
+
}
|
|
19
|
+
if (result.independence !== undefined) {
|
|
20
|
+
lines.push(`Independence: ${result.independence}.`);
|
|
21
|
+
}
|
|
22
|
+
if (result.verification !== undefined) {
|
|
23
|
+
lines.push("Machine verification:");
|
|
24
|
+
for (const command of result.verification.commands) {
|
|
25
|
+
lines.push(`- ${safe(command.criterionId)}/${safe(command.commandId)}: ${command.status}` +
|
|
26
|
+
`${command.required ? " (required)" : " (optional)"}` +
|
|
27
|
+
`${command.evidenceReference === undefined ? "" : ` — ${safe(command.evidenceReference)}`}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (result.report === undefined) {
|
|
31
|
+
lines.push(`Reason: ${result.reason ?? "unknown"}.`);
|
|
32
|
+
for (const error of result.validationErrors ?? []) {
|
|
33
|
+
lines.push(`- ${safe(error.path)}: ${safe(error.code)} — ${safe(error.message)}`);
|
|
34
|
+
}
|
|
35
|
+
lines.push("Run: agent-ops doctor --check-auth to verify target authentication.");
|
|
36
|
+
return `${lines.join("\n")}\n`;
|
|
37
|
+
}
|
|
38
|
+
const report = result.report;
|
|
39
|
+
const nonBlocking = report.findings.filter((finding) => !finding.blocking).length;
|
|
40
|
+
lines.push(`Non-blocking findings: ${nonBlocking}.`, "", "Summary:", safe(report.summary), "", "Criteria:");
|
|
41
|
+
for (const item of report.results) {
|
|
42
|
+
lines.push(`- ${safe(item.criterionId)}: ${item.status} — ${safe(item.summary)}`);
|
|
43
|
+
lines.push(...item.evidence.map((evidence) => ` - ${safe(evidence)}`));
|
|
44
|
+
}
|
|
45
|
+
lines.push("", "Findings:");
|
|
46
|
+
if (report.findings.length === 0) {
|
|
47
|
+
lines.push("- none");
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
for (const finding of report.findings) {
|
|
51
|
+
lines.push(`- [${finding.severity}] ${finding.blocking ? "blocking" : "non-blocking"}: ${safe(finding.title)}`);
|
|
52
|
+
lines.push(` ${safe(finding.details)}`);
|
|
53
|
+
lines.push(` Recommendation: ${safe(finding.recommendation)}`);
|
|
54
|
+
lines.push(...finding.locations.map((location) => ` Location: ${safe(location.path)}${location.line === undefined ? "" : `:${location.line}`}`));
|
|
55
|
+
lines.push(...finding.evidence.map((evidence) => ` Evidence: ${safe(evidence)}`));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
lines.push("", "Residual risks:", ...lineList(report.residualRisks));
|
|
59
|
+
lines.push("", "Changed files inspected:", ...lineList(report.changedFilesInspected));
|
|
60
|
+
lines.push("", "Supporting files inspected:", ...lineList(report.supportingFilesInspected));
|
|
61
|
+
return `${lines.join("\n")}\n`;
|
|
62
|
+
}
|