@navels/neal 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -209
- package/SECURITY.md +16 -19
- package/dist/neal/adjudicator/{blocked-adjudicator.js → consultant.js} +21 -21
- package/dist/neal/adjudicator/execute.js +33 -33
- package/dist/neal/adjudicator/final-completion.js +2 -31
- package/dist/neal/adjudicator/planning.js +230 -38
- package/dist/neal/agents/prompts.js +6 -17
- package/dist/neal/agents/rounds.js +43 -73
- package/dist/neal/agents/schemas.js +47 -13
- package/dist/neal/blocked-guidance.js +16 -9
- package/dist/neal/cli.js +2 -2
- package/dist/neal/commands/check.js +2 -2
- package/dist/neal/commands/compat.js +100 -36
- package/dist/neal/commands/interactive-activity.js +2 -2
- package/dist/neal/commands/recovery-guidance.js +20 -8
- package/dist/neal/commands/runtime.js +4 -2
- package/dist/neal/config.js +16 -7
- package/dist/neal/context/inline-review-context.js +13 -28
- package/dist/neal/context/reviewer-context.js +19 -14
- package/dist/neal/eval/reviewer-eval-manifest.js +86 -0
- package/dist/neal/eval/reviewer-eval.js +155 -0
- package/dist/neal/interactive-controls.js +5 -0
- package/dist/neal/orchestrator/completion.js +49 -2
- package/dist/neal/orchestrator/failures.js +7 -0
- package/dist/neal/orchestrator/notifications.js +9 -9
- package/dist/neal/orchestrator/phases/planning.js +117 -80
- package/dist/neal/orchestrator/phases/recovery.js +151 -115
- package/dist/neal/orchestrator/phases/review.js +28 -1
- package/dist/neal/orchestrator/transitions.js +7 -1
- package/dist/neal/orchestrator.js +3 -0
- package/dist/neal/plan-queue.js +29 -1
- package/dist/neal/prompts/execute.js +7 -19
- package/dist/neal/prompts/planning.js +64 -55
- package/dist/neal/prompts/review-doctrine.js +13 -50
- package/dist/neal/prompts/specialized.js +7 -17
- package/dist/neal/prompts/specs.js +180 -32
- package/dist/neal/providers/agent-settings-isolation.js +35 -0
- package/dist/neal/providers/anthropic-claude.js +18 -1
- package/dist/neal/providers/detection.js +3 -7
- package/dist/neal/providers/git-config-isolation.js +83 -0
- package/dist/neal/providers/openai-codex.js +25 -3
- package/dist/neal/providers/{generic-agentic-tools.js → openai-compatible-tools.js} +11 -5
- package/dist/neal/providers/openai-compatible.js +1078 -430
- package/dist/neal/providers/openrouter-routing.js +68 -0
- package/dist/neal/providers/pricing.js +1 -1
- package/dist/neal/providers/rate-card.js +6 -32
- package/dist/neal/providers/registry.js +6 -17
- package/dist/neal/providers/types.js +12 -0
- package/dist/neal/resume-decision.js +7 -6
- package/dist/neal/resume-planner.js +9 -0
- package/dist/neal/retrospective.js +3 -3
- package/dist/neal/review-debt.js +30 -0
- package/dist/neal/review-findings/provider.js +17 -29
- package/dist/neal/review.js +17 -1
- package/dist/neal/state-invariants.js +11 -3
- package/dist/neal/state-views.js +27 -5
- package/dist/neal/state.js +59 -15
- package/dist/neal/status.js +28 -1
- package/dist/neal/support.js +3 -3
- package/docs/README.md +22 -18
- package/docs/{ADJUDICATOR_INVENTORY.md → adjudicator-inventory.md} +58 -57
- package/docs/architecture.md +79 -83
- package/docs/automation.md +11 -11
- package/docs/compat.md +91 -61
- package/docs/compatible-models.md +145 -121
- package/docs/demo.md +15 -5
- package/docs/maintenance.md +65 -34
- package/docs/plan-format.md +61 -33
- package/docs/prompt-evals.md +96 -0
- package/docs/{PROMPT_SPECS.md → prompt-specs.md} +52 -48
- package/docs/providers.md +279 -349
- package/docs/release.md +63 -81
- package/docs/review-convergence.md +246 -0
- package/docs/state-machine.md +61 -59
- package/docs/storage.md +40 -31
- package/docs/troubleshooting.md +32 -29
- package/examples/compat/add-edit-verify/PLAN.md +2 -0
- package/examples/compat/add-edit-verify/broken.diff +2 -2
- package/examples/compat/add-edit-verify/good.diff +2 -2
- package/examples/compat/add-edit-verify/src/add.js +1 -1
- package/examples/compat/manifest.json +25 -29
- package/examples/compat/sum-grep-edit/PLAN.md +33 -0
- package/examples/compat/sum-grep-edit/broken.diff +12 -0
- package/examples/compat/sum-grep-edit/good.diff +12 -0
- package/examples/compat/sum-grep-edit/package.json +5 -0
- package/examples/compat/sum-grep-edit/src/numbers.js +9 -0
- package/examples/compat/sum-grep-edit/test/numbers.test.js +15 -0
- package/examples/issue-triage-js/README.md +0 -15
- package/examples/reviewer-eval/clean-doc/base/src/clamp.ts +3 -0
- package/examples/reviewer-eval/clean-doc/change.diff +9 -0
- package/examples/reviewer-eval/clean-extract/base/src/greet.ts +3 -0
- package/examples/reviewer-eval/clean-extract/change.diff +13 -0
- package/examples/reviewer-eval/dropped-error/base/src/load.ts +4 -0
- package/examples/reviewer-eval/dropped-error/change.diff +15 -0
- package/examples/reviewer-eval/manifest.json +45 -0
- package/examples/reviewer-eval/off-by-one/base/src/paginate.ts +5 -0
- package/examples/reviewer-eval/off-by-one/change.diff +11 -0
- package/{config.yml → neal.yml} +1 -1
- package/package.json +15 -14
- package/dist/neal/providers/generic-agentic.js +0 -1256
- package/docs/comparison.md +0 -105
- package/docs/issue-pipeline.md +0 -124
- package/examples/compat/is-even-add-test/PLAN.md +0 -30
- package/examples/compat/is-even-add-test/broken.diff +0 -11
- package/examples/compat/is-even-add-test/good.diff +0 -11
- package/examples/compat/is-even-add-test/package.json +0 -5
- package/examples/compat/is-even-add-test/src/is-even.js +0 -3
- package/examples/compat/is-even-add-test/test/is-even.test.js +0 -9
- package/examples/compat/reverse-grep-edit/PLAN.md +0 -32
- package/examples/compat/reverse-grep-edit/broken.diff +0 -12
- package/examples/compat/reverse-grep-edit/good.diff +0 -12
- package/examples/compat/reverse-grep-edit/package.json +0 -5
- package/examples/compat/reverse-grep-edit/src/strings.js +0 -10
- package/examples/compat/reverse-grep-edit/test/strings.test.js +0 -15
package/dist/neal/config.js
CHANGED
|
@@ -13,13 +13,14 @@ const DEFAULT_CONFIG = {
|
|
|
13
13
|
phase_heartbeat_ms: 60_000,
|
|
14
14
|
max_review_rounds: 20,
|
|
15
15
|
review_stuck_window: 5,
|
|
16
|
+
plan_review_debt_round_threshold: 3,
|
|
16
17
|
inactivity_timeout_ms: 600_000,
|
|
17
18
|
api_retry_limit: 10,
|
|
18
19
|
agent_turn_startup_timeout_ms: 300_000,
|
|
19
20
|
agent_turn_retry_limit: 1,
|
|
20
21
|
interactive_blocked_recovery_max_turns: 3,
|
|
21
22
|
final_completion_continue_execution_max: 2,
|
|
22
|
-
|
|
23
|
+
consultant_max_attempts: 1,
|
|
23
24
|
notify_bin: null,
|
|
24
25
|
},
|
|
25
26
|
agent: {
|
|
@@ -51,7 +52,7 @@ function buildWriterProviderSetupLines(guidance, missingProviderKeys) {
|
|
|
51
52
|
}
|
|
52
53
|
return [
|
|
53
54
|
'Neal is not set up yet.',
|
|
54
|
-
`Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in
|
|
55
|
+
`Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in neal.yml.`,
|
|
55
56
|
];
|
|
56
57
|
}
|
|
57
58
|
function formatInlineList(values) {
|
|
@@ -173,7 +174,7 @@ function mergeConfig(base, override) {
|
|
|
173
174
|
};
|
|
174
175
|
}
|
|
175
176
|
export function getConfigSourceInfo(cwd = process.cwd()) {
|
|
176
|
-
const repoPath = resolve(cwd, '
|
|
177
|
+
const repoPath = resolve(cwd, 'neal.yml');
|
|
177
178
|
const userPath = join(homedir(), '.neal', 'config.yml');
|
|
178
179
|
return {
|
|
179
180
|
repo: {
|
|
@@ -293,11 +294,19 @@ export function getReviewStuckWindow(cwd = process.cwd()) {
|
|
|
293
294
|
return (parseNumberValue(config.neal?.review_stuck_window) ??
|
|
294
295
|
DEFAULT_CONFIG.neal.review_stuck_window);
|
|
295
296
|
}
|
|
296
|
-
|
|
297
|
+
// The reviewer round at or past which a novel verification-hardening plan-review
|
|
298
|
+
// finding converts to recorded plan-review debt instead of forcing another
|
|
299
|
+
// revision round. Read at the decision site by cwd, mirroring getMaxReviewRounds.
|
|
300
|
+
export function getPlanReviewDebtRoundThreshold(cwd = process.cwd()) {
|
|
297
301
|
const config = loadConfigFile(cwd);
|
|
298
|
-
|
|
299
|
-
DEFAULT_CONFIG.neal.
|
|
300
|
-
|
|
302
|
+
return (parseNumberValue(config.neal?.plan_review_debt_round_threshold) ??
|
|
303
|
+
DEFAULT_CONFIG.neal.plan_review_debt_round_threshold);
|
|
304
|
+
}
|
|
305
|
+
export function getConsultantMaxAttempts(cwd = process.cwd()) {
|
|
306
|
+
const config = loadConfigFile(cwd);
|
|
307
|
+
const raw = parseNumberValue(config.neal?.consultant_max_attempts) ??
|
|
308
|
+
DEFAULT_CONFIG.neal.consultant_max_attempts;
|
|
309
|
+
// Clamp to a non-negative integer (0 disables the consultant entirely).
|
|
301
310
|
if (!Number.isFinite(raw) || raw <= 0) {
|
|
302
311
|
return 0;
|
|
303
312
|
}
|
|
@@ -4,8 +4,9 @@ import { getProviderDefinition, isRegisteredProviderId } from '../providers/regi
|
|
|
4
4
|
// truncation posture of truncateForPrompt in src/neal/agents/structured-json.ts:
|
|
5
5
|
// truncate with an explicit marker instead of silently dropping content.
|
|
6
6
|
export const INLINE_SECTION_MAX_CHARS = 200_000;
|
|
7
|
-
// Canonical forbidden-phrase list for no-read
|
|
8
|
-
//
|
|
7
|
+
// Canonical forbidden-phrase list for no-read prompts (today: the blocked-run
|
|
8
|
+
// consultant, which always judges from Neal-inlined in-memory context). A
|
|
9
|
+
// no-read prompt must contain no instruction that requires repository, file,
|
|
9
10
|
// tool, or shell access of any kind. Both the prompt-builder implementations
|
|
10
11
|
// and the runtime capture tests assert against this shared list, so a newly
|
|
11
12
|
// added repo-access phrase fails the shared assertion rather than silently
|
|
@@ -22,31 +23,18 @@ export const NO_READ_PROMPT_FORBIDDEN_MARKERS = [
|
|
|
22
23
|
'is available at',
|
|
23
24
|
'Inspect cited artifacts',
|
|
24
25
|
];
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
return getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess.read === false;
|
|
33
|
-
}
|
|
34
|
-
// Three-way review-doctrine access mode for the configured reviewer provider's
|
|
35
|
-
// structured-advisor role. Unregistered provider ids default to 'tool-access',
|
|
36
|
-
// mirroring reviewerNeedsInlineContext's false. No repository read access
|
|
37
|
-
// selects 'no-read' (Neal inlines the review context); read access without
|
|
38
|
-
// shell access selects 'read-only' (the reviewer inspects the repository with
|
|
39
|
-
// read tools but must not be instructed to run commands); read plus shell
|
|
40
|
-
// access selects 'tool-access'. Write access does not affect review doctrine:
|
|
41
|
-
// reviews never instruct repository mutation.
|
|
26
|
+
// Two-way review-doctrine access mode for the configured reviewer provider's
|
|
27
|
+
// structured-advisor role. Unregistered provider ids default to 'tool-access'.
|
|
28
|
+
// Every registered reviewer provider has repository read access; read access
|
|
29
|
+
// without shell access selects 'read-only' (the reviewer inspects the
|
|
30
|
+
// repository with read tools but must not be instructed to run commands);
|
|
31
|
+
// read plus shell access selects 'tool-access'. Write access does not affect
|
|
32
|
+
// review doctrine: reviews never instruct repository mutation.
|
|
42
33
|
export function getReviewerDoctrineAccessMode(reviewer) {
|
|
43
34
|
if (!isRegisteredProviderId(reviewer.provider)) {
|
|
44
35
|
return 'tool-access';
|
|
45
36
|
}
|
|
46
37
|
const toolAccess = getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess;
|
|
47
|
-
if (toolAccess.read === false) {
|
|
48
|
-
return 'no-read';
|
|
49
|
-
}
|
|
50
38
|
if (toolAccess.shell === false) {
|
|
51
39
|
return 'read-only';
|
|
52
40
|
}
|
|
@@ -55,12 +43,9 @@ export function getReviewerDoctrineAccessMode(reviewer) {
|
|
|
55
43
|
// True when the configured reviewer is a read-only reviewer (read tools, no
|
|
56
44
|
// shell) that has no commit-range diff tool of its own, so Neal must inline the
|
|
57
45
|
// commit-range diff into its reviewer prompt. Read-only reviewers that expose a
|
|
58
|
-
// commit-range diff tool (providesRangeDiffTool === true, e.g.
|
|
59
|
-
// inspect the range with that tool and do not receive an inlined diff
|
|
60
|
-
//
|
|
61
|
-
// shell git themselves. The inlined commit-range diff rides a dedicated prompt
|
|
62
|
-
// channel separate from the no-read inlineContext channel, so it never trips
|
|
63
|
-
// resolveReviewerPromptAccessMode's no-read-only inlineContext guard.
|
|
46
|
+
// commit-range diff tool (providesRangeDiffTool === true, e.g. openai-compatible)
|
|
47
|
+
// inspect the range with that tool and do not receive an inlined diff, and
|
|
48
|
+
// tool-access reviewers run shell git themselves.
|
|
64
49
|
export function readOnlyReviewerNeedsInlinedDiff(reviewer) {
|
|
65
50
|
if (!isRegisteredProviderId(reviewer.provider)) {
|
|
66
51
|
return false;
|
|
@@ -37,6 +37,13 @@ export function buildReviewerContextPacket(args) {
|
|
|
37
37
|
claim: finding.claim,
|
|
38
38
|
requiredAction: finding.requiredAction,
|
|
39
39
|
}));
|
|
40
|
+
const inheritedPlanReviewDebt = state.inheritedPlanReviewDebt.map((item) => ({
|
|
41
|
+
canonicalId: item.canonicalId,
|
|
42
|
+
findingClass: item.findingClass ?? null,
|
|
43
|
+
originRound: item.originRound ?? null,
|
|
44
|
+
claim: item.claim,
|
|
45
|
+
requiredAction: item.requiredAction,
|
|
46
|
+
}));
|
|
40
47
|
const finalCompletion = summarizeFinalCompletion(state);
|
|
41
48
|
const packetWithoutMarkdown = {
|
|
42
49
|
version: 1,
|
|
@@ -55,6 +62,7 @@ export function buildReviewerContextPacket(args) {
|
|
|
55
62
|
},
|
|
56
63
|
completedScopes,
|
|
57
64
|
findings,
|
|
65
|
+
inheritedPlanReviewDebt,
|
|
58
66
|
finalCompletion,
|
|
59
67
|
citations: buildReviewerContextCitations(state),
|
|
60
68
|
limits: {
|
|
@@ -100,21 +108,21 @@ function buildReviewerContextCitations(state) {
|
|
|
100
108
|
{ label: REVIEWER_CONTEXT_MARKDOWN, path: toDisplayPath(state.cwd, join(state.runDir, REVIEWER_CONTEXT_MARKDOWN)) },
|
|
101
109
|
];
|
|
102
110
|
}
|
|
103
|
-
export function renderReviewerContextMarkdown(packet
|
|
104
|
-
const mode = options.mode ?? 'tool-access';
|
|
111
|
+
export function renderReviewerContextMarkdown(packet) {
|
|
105
112
|
const scopeLines = packet.completedScopes.length
|
|
106
113
|
? packet.completedScopes.map((scope) => `- Scope ${scope.number}: ${scope.result}; marker=${scope.marker}; finalCommit=${scope.finalCommit ?? 'none'}; reviewRounds=${scope.reviewRounds}; findings=${scope.findings}; residualDebt=${scope.residualReviewDebt}; summary=${scope.summary ?? 'none'}`)
|
|
107
114
|
: ['- none'];
|
|
108
115
|
const findingLines = packet.findings.length
|
|
109
116
|
? packet.findings.map((finding) => `- ${finding.canonicalId} (${finding.id}): ${finding.severity}/${finding.status}; source=${finding.source}; files=${finding.files.join(', ') || 'none'}; claim=${finding.claim}; requiredAction=${finding.requiredAction}`)
|
|
110
117
|
: ['- none'];
|
|
118
|
+
const inheritedDebtLines = packet.inheritedPlanReviewDebt.length
|
|
119
|
+
? packet.inheritedPlanReviewDebt.map((item) => `- ${item.canonicalId}: findingClass=${item.findingClass ?? 'n/a'}; originRound=${item.originRound ?? 'n/a'}; claim=${item.claim}; requiredAction=${item.requiredAction}`)
|
|
120
|
+
: ['- none'];
|
|
111
121
|
const finalCompletion = packet.finalCompletion;
|
|
112
122
|
return [
|
|
113
123
|
'# Reviewer Continuity Context',
|
|
114
124
|
'',
|
|
115
|
-
|
|
116
|
-
? 'Use this bounded Neal-owned context for continuity only. All required review context is inlined into this prompt; you have no access to the cited run artifacts or the checkout.'
|
|
117
|
-
: 'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
|
|
125
|
+
'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
|
|
118
126
|
'',
|
|
119
127
|
'## Run',
|
|
120
128
|
`- id: ${packet.run.id}`,
|
|
@@ -137,19 +145,16 @@ export function renderReviewerContextMarkdown(packet, options = {}) {
|
|
|
137
145
|
? `- truncated: showing latest ${packet.findings.length} of ${packet.limits.findingCount}`
|
|
138
146
|
: '- truncated: no',
|
|
139
147
|
'',
|
|
148
|
+
'## Inherited Plan-Review Debt',
|
|
149
|
+
...inheritedDebtLines,
|
|
150
|
+
'',
|
|
140
151
|
'## Final Completion',
|
|
141
152
|
finalCompletion
|
|
142
153
|
? `- state=${finalCompletion.state}; effectiveAction=${finalCompletion.effectiveAction ?? 'none'}; hasSummary=${finalCompletion.hasSummary}; hasReviewVerdict=${finalCompletion.hasReviewVerdict}; continueExecutionCount=${finalCompletion.continueExecutionCount}; capReached=${finalCompletion.continueExecutionCapReached}`
|
|
143
154
|
: '- not applicable',
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
...(
|
|
147
|
-
? []
|
|
148
|
-
: [
|
|
149
|
-
'',
|
|
150
|
-
'## Citations',
|
|
151
|
-
...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
|
|
152
|
-
]),
|
|
155
|
+
'',
|
|
156
|
+
'## Citations',
|
|
157
|
+
...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
|
|
153
158
|
].join('\n');
|
|
154
159
|
}
|
|
155
160
|
function runIdFromDir(runDir) {
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
function fail(context, message) {
|
|
4
|
+
throw new Error(`reviewer-eval manifest: ${context} ${message}`);
|
|
5
|
+
}
|
|
6
|
+
function asString(value, context, field) {
|
|
7
|
+
const candidate = value[field];
|
|
8
|
+
if (typeof candidate !== 'string' || candidate.trim() === '') {
|
|
9
|
+
fail(context, `must define a non-empty string ${field}`);
|
|
10
|
+
}
|
|
11
|
+
return candidate;
|
|
12
|
+
}
|
|
13
|
+
function parseExpectedFinding(raw, context) {
|
|
14
|
+
if (typeof raw !== 'object' || raw === null) {
|
|
15
|
+
fail(context, 'each expectedFindings entry must be an object');
|
|
16
|
+
}
|
|
17
|
+
return {
|
|
18
|
+
file: asString(raw, context, 'file'),
|
|
19
|
+
defectClass: asString(raw, context, 'defectClass'),
|
|
20
|
+
description: asString(raw, context, 'description'),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
// Every path the diff adds or modifies, read from its `+++ b/<path>` headers.
|
|
24
|
+
function diffTargetFiles(diffText) {
|
|
25
|
+
const files = new Set();
|
|
26
|
+
for (const line of diffText.split('\n')) {
|
|
27
|
+
const match = /^\+\+\+ b\/(.+)$/.exec(line);
|
|
28
|
+
if (match) {
|
|
29
|
+
files.add(match[1].trim());
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return files;
|
|
33
|
+
}
|
|
34
|
+
export function loadReviewerEvalManifest(manifestDir) {
|
|
35
|
+
const dir = isAbsolute(manifestDir) ? manifestDir : resolve(manifestDir);
|
|
36
|
+
const manifestPath = join(dir, 'manifest.json');
|
|
37
|
+
const parsed = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
|
38
|
+
const rawFixtures = parsed.fixtures;
|
|
39
|
+
if (!Array.isArray(rawFixtures) || rawFixtures.length === 0) {
|
|
40
|
+
fail('manifest.json', 'must define a non-empty fixtures array');
|
|
41
|
+
}
|
|
42
|
+
const seen = new Set();
|
|
43
|
+
return rawFixtures.map((raw, index) => {
|
|
44
|
+
const context = `fixtures[${index}]`;
|
|
45
|
+
if (typeof raw !== 'object' || raw === null) {
|
|
46
|
+
fail(context, 'must be an object');
|
|
47
|
+
}
|
|
48
|
+
const id = asString(raw, context, 'id');
|
|
49
|
+
if (seen.has(id)) {
|
|
50
|
+
fail(context, `duplicate fixture id ${id}`);
|
|
51
|
+
}
|
|
52
|
+
seen.add(id);
|
|
53
|
+
const kind = raw.kind;
|
|
54
|
+
if (kind !== 'defective' && kind !== 'clean') {
|
|
55
|
+
fail(id, "kind must be 'defective' or 'clean'");
|
|
56
|
+
}
|
|
57
|
+
const diff = asString(raw, id, 'diff');
|
|
58
|
+
const baseDirRelative = asString(raw, id, 'baseDir');
|
|
59
|
+
const rawExpected = raw.expectedFindings;
|
|
60
|
+
if (!Array.isArray(rawExpected)) {
|
|
61
|
+
fail(id, 'expectedFindings must be an array');
|
|
62
|
+
}
|
|
63
|
+
const expectedFindings = rawExpected.map((entry) => parseExpectedFinding(entry, id));
|
|
64
|
+
if (kind === 'defective' && expectedFindings.length === 0) {
|
|
65
|
+
fail(id, 'a defective fixture must label at least one expected finding');
|
|
66
|
+
}
|
|
67
|
+
if (kind === 'clean' && expectedFindings.length > 0) {
|
|
68
|
+
fail(id, 'a clean fixture must not label any expected findings');
|
|
69
|
+
}
|
|
70
|
+
const diffPath = join(dir, diff);
|
|
71
|
+
const diffText = readFileSync(diffPath, 'utf8');
|
|
72
|
+
const targetFiles = diffTargetFiles(diffText);
|
|
73
|
+
for (const expected of expectedFindings) {
|
|
74
|
+
if (!targetFiles.has(expected.file)) {
|
|
75
|
+
fail(id, `expected finding file ${expected.file} is not modified by ${diff}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
id,
|
|
80
|
+
kind,
|
|
81
|
+
expectedFindings,
|
|
82
|
+
diffPath,
|
|
83
|
+
baseDir: join(dir, baseDirRelative),
|
|
84
|
+
};
|
|
85
|
+
});
|
|
86
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// Reviewer-recall eval scoring (issue #22 Phase B).
|
|
2
|
+
//
|
|
3
|
+
// Given a labeled fixture set — diffs with known defects, plus clean diffs —
|
|
4
|
+
// and the findings a reviewer produced for each, this computes recall,
|
|
5
|
+
// precision, and blocking-finding rate. It is the measurement instrument for
|
|
6
|
+
// the eval-gated reviewer-doctrine rewrite (Phase C): run it against a
|
|
7
|
+
// reviewer prompt before and after a change and compare the numbers.
|
|
8
|
+
//
|
|
9
|
+
// The scoring here is pure and deterministic; the live reviewer invocation
|
|
10
|
+
// lives in scripts/eval-reviewer.mjs (operator-run, subscription-billed, like
|
|
11
|
+
// scripts/qualify-sdk.sh). Matching is intentionally coarse — a blocking
|
|
12
|
+
// finding matches an expected label when they name the same file — because a
|
|
13
|
+
// finer match (defect-class, line range) would encode judgment the labels
|
|
14
|
+
// cannot reliably carry. The coarseness is documented so a reader does not
|
|
15
|
+
// over-trust the numbers: recall counts "did the reviewer flag the right
|
|
16
|
+
// file", not "did it describe the exact defect".
|
|
17
|
+
/**
|
|
18
|
+
* Projects a reviewer round's raw findings (as produced by the review-findings
|
|
19
|
+
* loop: objects carrying at least a severity and a files list) into a scored
|
|
20
|
+
* observation. Unknown severities are treated as non_blocking so a malformed
|
|
21
|
+
* finding cannot inflate the blocking count, and a missing files list becomes
|
|
22
|
+
* empty. Keeping this here lets the operator runner stay a thin shell over the
|
|
23
|
+
* tested scoring core.
|
|
24
|
+
*/
|
|
25
|
+
export function toReviewerEvalObservation(fixtureId, rawFindings) {
|
|
26
|
+
const findings = rawFindings.map((finding) => ({
|
|
27
|
+
severity: finding.severity === 'blocking' ? 'blocking' : 'non_blocking',
|
|
28
|
+
files: Array.isArray(finding.files) ? finding.files.filter((file) => typeof file === 'string') : [],
|
|
29
|
+
}));
|
|
30
|
+
return { fixtureId, findings };
|
|
31
|
+
}
|
|
32
|
+
function normalizeFilePath(path) {
|
|
33
|
+
return path.trim().replace(/\\/g, '/').replace(/^\.\//, '');
|
|
34
|
+
}
|
|
35
|
+
function findingFileSet(observation) {
|
|
36
|
+
const files = new Set();
|
|
37
|
+
for (const finding of observation.findings) {
|
|
38
|
+
if (finding.severity !== 'blocking') {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
for (const file of finding.files) {
|
|
42
|
+
files.add(normalizeFilePath(file));
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return files;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Scores reviewer observations against labeled fixtures. Every fixture must
|
|
49
|
+
* have exactly one observation (missing observations throw — a fixture the
|
|
50
|
+
* runner failed to evaluate must not silently count as perfect recall).
|
|
51
|
+
*/
|
|
52
|
+
export function scoreReviewerEval(fixtures, observations) {
|
|
53
|
+
const observationById = new Map(observations.map((observation) => [observation.fixtureId, observation]));
|
|
54
|
+
const fixtureResults = [];
|
|
55
|
+
let totalExpectedLabels = 0;
|
|
56
|
+
let matchedLabels = 0;
|
|
57
|
+
let totalBlockingFindings = 0;
|
|
58
|
+
let truePositiveFindings = 0;
|
|
59
|
+
let falsePositiveFindings = 0;
|
|
60
|
+
let cleanFixturesFlagged = 0;
|
|
61
|
+
for (const fixture of fixtures) {
|
|
62
|
+
const observation = observationById.get(fixture.id);
|
|
63
|
+
if (!observation) {
|
|
64
|
+
throw new Error(`scoreReviewerEval: no observation for fixture ${fixture.id}`);
|
|
65
|
+
}
|
|
66
|
+
const blockingFindings = observation.findings.filter((finding) => finding.severity === 'blocking');
|
|
67
|
+
const blockingFindingCount = blockingFindings.length;
|
|
68
|
+
const findingFiles = findingFileSet(observation);
|
|
69
|
+
const matched = [];
|
|
70
|
+
const missed = [];
|
|
71
|
+
const labeledFiles = new Set();
|
|
72
|
+
for (const expected of fixture.expectedFindings) {
|
|
73
|
+
const file = normalizeFilePath(expected.file);
|
|
74
|
+
labeledFiles.add(file);
|
|
75
|
+
if (findingFiles.has(file)) {
|
|
76
|
+
matched.push(expected);
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
missed.push(expected);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
// A blocking finding is a true positive when it touches at least one
|
|
83
|
+
// labeled file; every other blocking finding (all of them on a clean
|
|
84
|
+
// fixture) is a false positive.
|
|
85
|
+
let truePositiveFindingCount = 0;
|
|
86
|
+
let falsePositiveFindingCount = 0;
|
|
87
|
+
for (const finding of blockingFindings) {
|
|
88
|
+
const touchesLabel = finding.files.some((file) => labeledFiles.has(normalizeFilePath(file)));
|
|
89
|
+
if (touchesLabel) {
|
|
90
|
+
truePositiveFindingCount += 1;
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
falsePositiveFindingCount += 1;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
totalExpectedLabels += fixture.expectedFindings.length;
|
|
97
|
+
matchedLabels += matched.length;
|
|
98
|
+
totalBlockingFindings += blockingFindingCount;
|
|
99
|
+
truePositiveFindings += truePositiveFindingCount;
|
|
100
|
+
falsePositiveFindings += falsePositiveFindingCount;
|
|
101
|
+
if (fixture.kind === 'clean' && blockingFindingCount > 0) {
|
|
102
|
+
cleanFixturesFlagged += 1;
|
|
103
|
+
}
|
|
104
|
+
fixtureResults.push({
|
|
105
|
+
fixtureId: fixture.id,
|
|
106
|
+
kind: fixture.kind,
|
|
107
|
+
matchedLabels: matched,
|
|
108
|
+
missedLabels: missed,
|
|
109
|
+
truePositiveFindingCount,
|
|
110
|
+
falsePositiveFindingCount,
|
|
111
|
+
blockingFindingCount,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
const defectiveFixtures = fixtures.filter((fixture) => fixture.kind === 'defective').length;
|
|
115
|
+
const cleanFixtures = fixtures.filter((fixture) => fixture.kind === 'clean').length;
|
|
116
|
+
return {
|
|
117
|
+
fixtures: fixtureResults,
|
|
118
|
+
totalFixtures: fixtures.length,
|
|
119
|
+
defectiveFixtures,
|
|
120
|
+
cleanFixtures,
|
|
121
|
+
recall: totalExpectedLabels === 0 ? null : matchedLabels / totalExpectedLabels,
|
|
122
|
+
precision: totalBlockingFindings === 0 ? null : truePositiveFindings / totalBlockingFindings,
|
|
123
|
+
totalExpectedLabels,
|
|
124
|
+
matchedLabels,
|
|
125
|
+
totalBlockingFindings,
|
|
126
|
+
truePositiveFindings,
|
|
127
|
+
falsePositiveFindings,
|
|
128
|
+
cleanFalsePositiveRate: cleanFixtures === 0 ? null : cleanFixturesFlagged / cleanFixtures,
|
|
129
|
+
cleanFixturesFlagged,
|
|
130
|
+
blockingFindingRate: fixtures.length === 0 ? 0 : totalBlockingFindings / fixtures.length,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Renders a report as a compact human-readable table plus a summary line.
|
|
135
|
+
* The machine-readable form is the ReviewerEvalReport itself (write it as
|
|
136
|
+
* JSON); this is for terminal output.
|
|
137
|
+
*/
|
|
138
|
+
export function renderReviewerEvalReport(report) {
|
|
139
|
+
const pct = (value) => (value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`);
|
|
140
|
+
const lines = [];
|
|
141
|
+
lines.push('| Fixture | Kind | Labels hit | Blocking | False+ |');
|
|
142
|
+
lines.push('|---|---|---:|---:|---:|');
|
|
143
|
+
for (const fixture of report.fixtures) {
|
|
144
|
+
const labels = fixture.kind === 'defective'
|
|
145
|
+
? `${fixture.matchedLabels.length}/${fixture.matchedLabels.length + fixture.missedLabels.length}`
|
|
146
|
+
: '—';
|
|
147
|
+
lines.push(`| ${fixture.fixtureId} | ${fixture.kind} | ${labels} | ${fixture.blockingFindingCount} | ${fixture.falsePositiveFindingCount} |`);
|
|
148
|
+
}
|
|
149
|
+
lines.push('');
|
|
150
|
+
lines.push(`Recall ${pct(report.recall)} (${report.matchedLabels}/${report.totalExpectedLabels} labels) · ` +
|
|
151
|
+
`Precision ${pct(report.precision)} (${report.truePositiveFindings}/${report.totalBlockingFindings} findings) · ` +
|
|
152
|
+
`Clean false-positive ${pct(report.cleanFalsePositiveRate)} (${report.cleanFixturesFlagged}/${report.cleanFixtures}) · ` +
|
|
153
|
+
`Blocking rate ${report.blockingFindingRate.toFixed(2)}/fixture`);
|
|
154
|
+
return lines.join('\n');
|
|
155
|
+
}
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import readline from 'node:readline';
|
|
3
3
|
import { clearDiagnosticFooter, getBufferedDetailSnapshot, isDiagnosticDetailVisible, showDiagnosticDetailView, showDiagnosticNarrativeView, writeNarrative, } from './diagnostic.js';
|
|
4
|
+
export function renderInteractiveKeyHint(allowStopRequest) {
|
|
5
|
+
return allowStopRequest
|
|
6
|
+
? '[neal] keys: q stop after current scope, v show/hide details\n'
|
|
7
|
+
: '[neal] keys: v show/hide details\n';
|
|
8
|
+
}
|
|
4
9
|
export function createInteractiveKeyController(options = {}) {
|
|
5
10
|
let stopRequested = false;
|
|
6
11
|
let cleanedUp = false;
|
|
@@ -21,7 +21,7 @@ import { saveState } from '../state.js';
|
|
|
21
21
|
import { getFinalCompletionReviewerScratchDir, getScopeReviewerScratchDir } from '../storage-paths.js';
|
|
22
22
|
import { requireFinalCompletionView, resolveFinalCompletionReviewAction, } from '../state-views.js';
|
|
23
23
|
import { filterAllowedDirtyPathStatus, filterWrapperOwnedWorktreeStatus, formatDirtyWorktreeDiagnostic, } from '../worktree-status.js';
|
|
24
|
-
import { shouldNotifyFailure } from './failures.js';
|
|
24
|
+
import { REVIEWER_CONTENT_REFUSED_BLOCK_REASON, shouldNotifyFailure } from './failures.js';
|
|
25
25
|
import { persistUnattendedBlockUnresolvedFailure } from './phases/shared.js';
|
|
26
26
|
async function writeFailedFinalCompletionReviewArtifact(args) {
|
|
27
27
|
const unstructuredOutput = await readFinalCompletionUnstructuredOutput({
|
|
@@ -52,6 +52,38 @@ async function persistFailedFinalCompletionReview(args) {
|
|
|
52
52
|
});
|
|
53
53
|
return failedState;
|
|
54
54
|
}
|
|
55
|
+
// Sibling of persistFailedFinalCompletionReview for the terminal blocked
|
|
56
|
+
// landing of a reviewer content-safety refusal (`content_refused`). Only a
|
|
57
|
+
// ReviewerRoundError can carry that kind; FinalCompletionReviewerVerdictError
|
|
58
|
+
// never does, so persistFailedFinalCompletionReview keeps serving the
|
|
59
|
+
// verdict-invalid path unchanged. Persists a terminal
|
|
60
|
+
// phase:'blocked'/status:'blocked' state so the run ends blocked (exit 2)
|
|
61
|
+
// rather than as a generic terminal failure, still writing the failed
|
|
62
|
+
// final-completion review artifact and execution artifacts.
|
|
63
|
+
async function persistBlockedFinalCompletionReview(args) {
|
|
64
|
+
const blockedState = await saveState(args.statePath, {
|
|
65
|
+
...args.state,
|
|
66
|
+
phase: 'blocked',
|
|
67
|
+
reviewerSessionHandle: null,
|
|
68
|
+
status: 'blocked',
|
|
69
|
+
blockedFromPhase: null,
|
|
70
|
+
blockerReason: REVIEWER_CONTENT_REFUSED_BLOCK_REASON,
|
|
71
|
+
});
|
|
72
|
+
await args.runtime.writeExecutionArtifacts(blockedState);
|
|
73
|
+
await writeFailedFinalCompletionReviewArtifact({
|
|
74
|
+
state: blockedState,
|
|
75
|
+
source: 'reviewer_verdict',
|
|
76
|
+
sessionHandle: args.error.sessionHandle,
|
|
77
|
+
});
|
|
78
|
+
await args.logger?.event('phase.error', {
|
|
79
|
+
phase: 'final_completion_review',
|
|
80
|
+
sessionHandle: args.error.sessionHandle,
|
|
81
|
+
subtype: args.error.subtype,
|
|
82
|
+
errorKind: args.error.kind,
|
|
83
|
+
message: args.error.message,
|
|
84
|
+
});
|
|
85
|
+
return blockedState;
|
|
86
|
+
}
|
|
55
87
|
function normalizeFinalCommitMessage(message) {
|
|
56
88
|
const normalizedNewlines = message.replace(/\r\n/g, '\n');
|
|
57
89
|
const convertedEscapes = normalizedNewlines.replace(/\\n(?=- )/g, '\n');
|
|
@@ -480,6 +512,21 @@ export async function runFinalCompletionReviewPhase(state, statePath, logger, ru
|
|
|
480
512
|
}));
|
|
481
513
|
}
|
|
482
514
|
catch (error) {
|
|
515
|
+
// A reviewer content-safety refusal ends the run blocked (exit 2) with a
|
|
516
|
+
// durable actionable reason, by RETURNING the terminal blocked state rather
|
|
517
|
+
// than throwing. Only a ReviewerRoundError carries `content_refused`; the
|
|
518
|
+
// verdict-invalid error never does, so it stays on the failed path below.
|
|
519
|
+
if (error instanceof ReviewerRoundError && error.kind === 'content_refused') {
|
|
520
|
+
const blockedState = await persistBlockedFinalCompletionReview({
|
|
521
|
+
state,
|
|
522
|
+
statePath,
|
|
523
|
+
error,
|
|
524
|
+
logger,
|
|
525
|
+
runtime,
|
|
526
|
+
});
|
|
527
|
+
await notifyBlocked(blockedState, REVIEWER_CONTENT_REFUSED_BLOCK_REASON, logger);
|
|
528
|
+
return blockedState;
|
|
529
|
+
}
|
|
483
530
|
if (error instanceof ReviewerRoundError || error instanceof FinalCompletionReviewerVerdictError) {
|
|
484
531
|
const failedState = await persistFailedFinalCompletionReview({
|
|
485
532
|
state,
|
|
@@ -539,7 +586,7 @@ export async function runFinalCompletionReviewPhase(state, statePath, logger, ru
|
|
|
539
586
|
// budgets reset here exactly as they do at every scope-advance
|
|
540
587
|
// transition, so an earlier scope's adjudication or split-plan
|
|
541
588
|
// consumption never exhausts the reopened scope's budget.
|
|
542
|
-
// (`
|
|
589
|
+
// (`consultantAttemptCount` resets via the shared next-scope
|
|
543
590
|
// reset spread above.)
|
|
544
591
|
splitPlanCountForCurrentScope: 0,
|
|
545
592
|
blockedFromPhase: null,
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import { CoderRoundError } from '../agents.js';
|
|
2
|
+
// Shared, durable blocked reason for a reviewer content-safety refusal
|
|
3
|
+
// (`content_refused`). Defined once so the wording cannot drift across the
|
|
4
|
+
// three reviewer-phase catch sites that route it to a terminal blocked landing.
|
|
5
|
+
export const REVIEWER_CONTENT_REFUSED_BLOCK_REASON = 'The reviewer refused this content on content-safety grounds. The change ' +
|
|
6
|
+
'under review may read as security-adjacent. Switch the reviewer to a ' +
|
|
7
|
+
'different provider, rephrase the scope, or seek provider authorization, ' +
|
|
8
|
+
'then re-run.';
|
|
2
9
|
export function isCoderTimeoutError(error) {
|
|
3
10
|
return (error.kind === 'timeout' ||
|
|
4
11
|
error.kind === 'no_progress_timeout' ||
|
|
@@ -3,40 +3,40 @@ import { notify } from '../../notifier.js';
|
|
|
3
3
|
import { getCurrentScopeLabel, getExecutionPlanPath, getExecutionPlanScopeCount, getParentScopeLabel, renderScopeProgressSegments, } from '../scopes.js';
|
|
4
4
|
import { saveState } from '../state.js';
|
|
5
5
|
import { getDerivedPlanView } from '../state-views.js';
|
|
6
|
-
// Concise representation of the read-only
|
|
6
|
+
// Concise representation of the read-only consultant advice for the
|
|
7
7
|
// operator notification surface. Present only on an attended run whose active
|
|
8
|
-
// interactive-blocked-recovery record carries
|
|
8
|
+
// interactive-blocked-recovery record carries consultant advice (knob > 0,
|
|
9
9
|
// budget available, eligible source phase). Returns '' otherwise so notification
|
|
10
10
|
// behavior for recovery states without advice — and terminal blocked
|
|
11
11
|
// notifications, where the active record is already finalized to null — is
|
|
12
12
|
// unchanged.
|
|
13
|
-
function
|
|
14
|
-
const advice = state.interactiveBlockedRecovery?.
|
|
13
|
+
function consultantAdviceNotificationSuffix(state) {
|
|
14
|
+
const advice = state.interactiveBlockedRecovery?.consultantAdvice;
|
|
15
15
|
if (!advice) {
|
|
16
16
|
return '';
|
|
17
17
|
}
|
|
18
18
|
const directive = advice.resolutionDirective.trim() || 'n/a';
|
|
19
|
-
return ` |
|
|
19
|
+
return ` | consultant advice (read-only): triage ${advice.triageCategory}; suggested directive: ${directive}`;
|
|
20
20
|
}
|
|
21
21
|
async function notifyBlocked(state, reason, logger) {
|
|
22
22
|
const planName = basename(state.planDoc);
|
|
23
|
-
const adviceSuffix =
|
|
23
|
+
const adviceSuffix = consultantAdviceNotificationSuffix(state);
|
|
24
24
|
await logger?.event('notify.blocked', {
|
|
25
25
|
reason,
|
|
26
26
|
planName,
|
|
27
|
-
|
|
27
|
+
consultantAdvice: state.interactiveBlockedRecovery?.consultantAdvice ?? null,
|
|
28
28
|
});
|
|
29
29
|
await notify('blocked', `[neal] ${planName}: ${reason}${adviceSuffix}`, state.cwd);
|
|
30
30
|
}
|
|
31
31
|
async function notifyInteractiveBlockedRecovery(state, reason, logger) {
|
|
32
32
|
const planName = basename(state.planDoc);
|
|
33
33
|
const scopeLabel = getCurrentScopeLabel(state);
|
|
34
|
-
const adviceSuffix =
|
|
34
|
+
const adviceSuffix = consultantAdviceNotificationSuffix(state);
|
|
35
35
|
await logger?.event('notify.interactive_blocked_recovery', {
|
|
36
36
|
reason,
|
|
37
37
|
planName,
|
|
38
38
|
scopeNumber: scopeLabel,
|
|
39
|
-
|
|
39
|
+
consultantAdvice: state.interactiveBlockedRecovery?.consultantAdvice ?? null,
|
|
40
40
|
});
|
|
41
41
|
await notify('retry', `[neal] ${planName}: interactive blocked recovery for scope ${scopeLabel}: ${reason}${adviceSuffix}`, state.cwd);
|
|
42
42
|
}
|