@navels/neal 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +206 -209
  2. package/SECURITY.md +16 -19
  3. package/dist/neal/adjudicator/{blocked-adjudicator.js → consultant.js} +21 -21
  4. package/dist/neal/adjudicator/execute.js +33 -33
  5. package/dist/neal/adjudicator/final-completion.js +2 -31
  6. package/dist/neal/adjudicator/planning.js +230 -38
  7. package/dist/neal/agents/prompts.js +6 -17
  8. package/dist/neal/agents/rounds.js +43 -73
  9. package/dist/neal/agents/schemas.js +47 -13
  10. package/dist/neal/blocked-guidance.js +16 -9
  11. package/dist/neal/cli.js +2 -2
  12. package/dist/neal/commands/check.js +2 -2
  13. package/dist/neal/commands/compat.js +100 -36
  14. package/dist/neal/commands/interactive-activity.js +2 -2
  15. package/dist/neal/commands/recovery-guidance.js +20 -8
  16. package/dist/neal/commands/runtime.js +4 -2
  17. package/dist/neal/config.js +16 -7
  18. package/dist/neal/context/inline-review-context.js +13 -28
  19. package/dist/neal/context/reviewer-context.js +19 -14
  20. package/dist/neal/eval/reviewer-eval-manifest.js +86 -0
  21. package/dist/neal/eval/reviewer-eval.js +155 -0
  22. package/dist/neal/interactive-controls.js +5 -0
  23. package/dist/neal/orchestrator/completion.js +49 -2
  24. package/dist/neal/orchestrator/failures.js +7 -0
  25. package/dist/neal/orchestrator/notifications.js +9 -9
  26. package/dist/neal/orchestrator/phases/planning.js +117 -80
  27. package/dist/neal/orchestrator/phases/recovery.js +151 -115
  28. package/dist/neal/orchestrator/phases/review.js +28 -1
  29. package/dist/neal/orchestrator/transitions.js +7 -1
  30. package/dist/neal/orchestrator.js +3 -0
  31. package/dist/neal/plan-queue.js +29 -1
  32. package/dist/neal/prompts/execute.js +7 -19
  33. package/dist/neal/prompts/planning.js +64 -55
  34. package/dist/neal/prompts/review-doctrine.js +13 -50
  35. package/dist/neal/prompts/specialized.js +7 -17
  36. package/dist/neal/prompts/specs.js +180 -32
  37. package/dist/neal/providers/agent-settings-isolation.js +35 -0
  38. package/dist/neal/providers/anthropic-claude.js +18 -1
  39. package/dist/neal/providers/detection.js +3 -7
  40. package/dist/neal/providers/git-config-isolation.js +83 -0
  41. package/dist/neal/providers/openai-codex.js +25 -3
  42. package/dist/neal/providers/{generic-agentic-tools.js → openai-compatible-tools.js} +11 -5
  43. package/dist/neal/providers/openai-compatible.js +1078 -430
  44. package/dist/neal/providers/openrouter-routing.js +68 -0
  45. package/dist/neal/providers/pricing.js +1 -1
  46. package/dist/neal/providers/rate-card.js +6 -32
  47. package/dist/neal/providers/registry.js +6 -17
  48. package/dist/neal/providers/types.js +12 -0
  49. package/dist/neal/resume-decision.js +7 -6
  50. package/dist/neal/resume-planner.js +9 -0
  51. package/dist/neal/retrospective.js +3 -3
  52. package/dist/neal/review-debt.js +30 -0
  53. package/dist/neal/review-findings/provider.js +17 -29
  54. package/dist/neal/review.js +17 -1
  55. package/dist/neal/state-invariants.js +11 -3
  56. package/dist/neal/state-views.js +27 -5
  57. package/dist/neal/state.js +59 -15
  58. package/dist/neal/status.js +28 -1
  59. package/dist/neal/support.js +3 -3
  60. package/docs/README.md +22 -18
  61. package/docs/{ADJUDICATOR_INVENTORY.md → adjudicator-inventory.md} +58 -57
  62. package/docs/architecture.md +79 -83
  63. package/docs/automation.md +11 -11
  64. package/docs/compat.md +91 -61
  65. package/docs/compatible-models.md +145 -121
  66. package/docs/demo.md +15 -5
  67. package/docs/maintenance.md +65 -34
  68. package/docs/plan-format.md +61 -33
  69. package/docs/prompt-evals.md +96 -0
  70. package/docs/{PROMPT_SPECS.md → prompt-specs.md} +52 -48
  71. package/docs/providers.md +279 -349
  72. package/docs/release.md +63 -81
  73. package/docs/review-convergence.md +246 -0
  74. package/docs/state-machine.md +61 -59
  75. package/docs/storage.md +40 -31
  76. package/docs/troubleshooting.md +32 -29
  77. package/examples/compat/add-edit-verify/PLAN.md +2 -0
  78. package/examples/compat/add-edit-verify/broken.diff +2 -2
  79. package/examples/compat/add-edit-verify/good.diff +2 -2
  80. package/examples/compat/add-edit-verify/src/add.js +1 -1
  81. package/examples/compat/manifest.json +25 -29
  82. package/examples/compat/sum-grep-edit/PLAN.md +33 -0
  83. package/examples/compat/sum-grep-edit/broken.diff +12 -0
  84. package/examples/compat/sum-grep-edit/good.diff +12 -0
  85. package/examples/compat/sum-grep-edit/package.json +5 -0
  86. package/examples/compat/sum-grep-edit/src/numbers.js +9 -0
  87. package/examples/compat/sum-grep-edit/test/numbers.test.js +15 -0
  88. package/examples/issue-triage-js/README.md +0 -15
  89. package/examples/reviewer-eval/clean-doc/base/src/clamp.ts +3 -0
  90. package/examples/reviewer-eval/clean-doc/change.diff +9 -0
  91. package/examples/reviewer-eval/clean-extract/base/src/greet.ts +3 -0
  92. package/examples/reviewer-eval/clean-extract/change.diff +13 -0
  93. package/examples/reviewer-eval/dropped-error/base/src/load.ts +4 -0
  94. package/examples/reviewer-eval/dropped-error/change.diff +15 -0
  95. package/examples/reviewer-eval/manifest.json +45 -0
  96. package/examples/reviewer-eval/off-by-one/base/src/paginate.ts +5 -0
  97. package/examples/reviewer-eval/off-by-one/change.diff +11 -0
  98. package/{config.yml → neal.yml} +1 -1
  99. package/package.json +15 -14
  100. package/dist/neal/providers/generic-agentic.js +0 -1256
  101. package/docs/comparison.md +0 -105
  102. package/docs/issue-pipeline.md +0 -124
  103. package/examples/compat/is-even-add-test/PLAN.md +0 -30
  104. package/examples/compat/is-even-add-test/broken.diff +0 -11
  105. package/examples/compat/is-even-add-test/good.diff +0 -11
  106. package/examples/compat/is-even-add-test/package.json +0 -5
  107. package/examples/compat/is-even-add-test/src/is-even.js +0 -3
  108. package/examples/compat/is-even-add-test/test/is-even.test.js +0 -9
  109. package/examples/compat/reverse-grep-edit/PLAN.md +0 -32
  110. package/examples/compat/reverse-grep-edit/broken.diff +0 -12
  111. package/examples/compat/reverse-grep-edit/good.diff +0 -12
  112. package/examples/compat/reverse-grep-edit/package.json +0 -5
  113. package/examples/compat/reverse-grep-edit/src/strings.js +0 -10
  114. package/examples/compat/reverse-grep-edit/test/strings.test.js +0 -15
@@ -13,13 +13,14 @@ const DEFAULT_CONFIG = {
13
13
  phase_heartbeat_ms: 60_000,
14
14
  max_review_rounds: 20,
15
15
  review_stuck_window: 5,
16
+ plan_review_debt_round_threshold: 3,
16
17
  inactivity_timeout_ms: 600_000,
17
18
  api_retry_limit: 10,
18
19
  agent_turn_startup_timeout_ms: 300_000,
19
20
  agent_turn_retry_limit: 1,
20
21
  interactive_blocked_recovery_max_turns: 3,
21
22
  final_completion_continue_execution_max: 2,
22
- review_stuck_arbiter_max_attempts: 1,
23
+ consultant_max_attempts: 1,
23
24
  notify_bin: null,
24
25
  },
25
26
  agent: {
@@ -51,7 +52,7 @@ function buildWriterProviderSetupLines(guidance, missingProviderKeys) {
51
52
  }
52
53
  return [
53
54
  'Neal is not set up yet.',
54
- `Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in config.yml.`,
55
+ `Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in neal.yml.`,
55
56
  ];
56
57
  }
57
58
  function formatInlineList(values) {
@@ -173,7 +174,7 @@ function mergeConfig(base, override) {
173
174
  };
174
175
  }
175
176
  export function getConfigSourceInfo(cwd = process.cwd()) {
176
- const repoPath = resolve(cwd, 'config.yml');
177
+ const repoPath = resolve(cwd, 'neal.yml');
177
178
  const userPath = join(homedir(), '.neal', 'config.yml');
178
179
  return {
179
180
  repo: {
@@ -293,11 +294,19 @@ export function getReviewStuckWindow(cwd = process.cwd()) {
293
294
  return (parseNumberValue(config.neal?.review_stuck_window) ??
294
295
  DEFAULT_CONFIG.neal.review_stuck_window);
295
296
  }
296
- export function getReviewStuckArbiterMaxAttempts(cwd = process.cwd()) {
297
+ // The reviewer round at or past which a novel verification-hardening plan-review
298
+ // finding converts to recorded plan-review debt instead of forcing another
299
+ // revision round. Read at the decision site by cwd, mirroring getMaxReviewRounds.
300
+ export function getPlanReviewDebtRoundThreshold(cwd = process.cwd()) {
297
301
  const config = loadConfigFile(cwd);
298
- const raw = parseNumberValue(config.neal?.review_stuck_arbiter_max_attempts) ??
299
- DEFAULT_CONFIG.neal.review_stuck_arbiter_max_attempts;
300
- // Clamp to a non-negative integer (0 disables the arbiter entirely).
302
+ return (parseNumberValue(config.neal?.plan_review_debt_round_threshold) ??
303
+ DEFAULT_CONFIG.neal.plan_review_debt_round_threshold);
304
+ }
305
+ export function getConsultantMaxAttempts(cwd = process.cwd()) {
306
+ const config = loadConfigFile(cwd);
307
+ const raw = parseNumberValue(config.neal?.consultant_max_attempts) ??
308
+ DEFAULT_CONFIG.neal.consultant_max_attempts;
309
+ // Clamp to a non-negative integer (0 disables the consultant entirely).
301
310
  if (!Number.isFinite(raw) || raw <= 0) {
302
311
  return 0;
303
312
  }
@@ -4,8 +4,9 @@ import { getProviderDefinition, isRegisteredProviderId } from '../providers/regi
4
4
  // truncation posture of truncateForPrompt in src/neal/agents/structured-json.ts:
5
5
  // truncate with an explicit marker instead of silently dropping content.
6
6
  export const INLINE_SECTION_MAX_CHARS = 200_000;
7
- // Canonical forbidden-phrase list for no-read reviewer prompts. A no-read
8
- // reviewer prompt must contain no instruction that requires repository, file,
7
+ // Canonical forbidden-phrase list for no-read prompts (today: the blocked-run
8
+ // consultant, which always judges from Neal-inlined in-memory context). A
9
+ // no-read prompt must contain no instruction that requires repository, file,
9
10
  // tool, or shell access of any kind. Both the prompt-builder implementations
10
11
  // and the runtime capture tests assert against this shared list, so a newly
11
12
  // added repo-access phrase fails the shared assertion rather than silently
@@ -22,31 +23,18 @@ export const NO_READ_PROMPT_FORBIDDEN_MARKERS = [
22
23
  'is available at',
23
24
  'Inspect cited artifacts',
24
25
  ];
25
- // True when the configured reviewer provider's structured-advisor role has no
26
- // repository read access, so Neal must inline the review context (diff, plan,
27
- // artifacts) into reviewer prompts.
28
- export function reviewerNeedsInlineContext(reviewer) {
29
- if (!isRegisteredProviderId(reviewer.provider)) {
30
- return false;
31
- }
32
- return getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess.read === false;
33
- }
34
- // Three-way review-doctrine access mode for the configured reviewer provider's
35
- // structured-advisor role. Unregistered provider ids default to 'tool-access',
36
- // mirroring reviewerNeedsInlineContext's false. No repository read access
37
- // selects 'no-read' (Neal inlines the review context); read access without
38
- // shell access selects 'read-only' (the reviewer inspects the repository with
39
- // read tools but must not be instructed to run commands); read plus shell
40
- // access selects 'tool-access'. Write access does not affect review doctrine:
41
- // reviews never instruct repository mutation.
26
+ // Two-way review-doctrine access mode for the configured reviewer provider's
27
+ // structured-advisor role. Unregistered provider ids default to 'tool-access'.
28
+ // Every registered reviewer provider has repository read access; read access
29
+ // without shell access selects 'read-only' (the reviewer inspects the
30
+ // repository with read tools but must not be instructed to run commands);
31
+ // read plus shell access selects 'tool-access'. Write access does not affect
32
+ // review doctrine: reviews never instruct repository mutation.
42
33
  export function getReviewerDoctrineAccessMode(reviewer) {
43
34
  if (!isRegisteredProviderId(reviewer.provider)) {
44
35
  return 'tool-access';
45
36
  }
46
37
  const toolAccess = getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess;
47
- if (toolAccess.read === false) {
48
- return 'no-read';
49
- }
50
38
  if (toolAccess.shell === false) {
51
39
  return 'read-only';
52
40
  }
@@ -55,12 +43,9 @@ export function getReviewerDoctrineAccessMode(reviewer) {
55
43
  // True when the configured reviewer is a read-only reviewer (read tools, no
56
44
  // shell) that has no commit-range diff tool of its own, so Neal must inline the
57
45
  // commit-range diff into its reviewer prompt. Read-only reviewers that expose a
58
- // commit-range diff tool (providesRangeDiffTool === true, e.g. generic-agentic)
59
- // inspect the range with that tool and do not receive an inlined diff; no-read
60
- // reviewers get the full inlineContext instead, and tool-access reviewers run
61
- // shell git themselves. The inlined commit-range diff rides a dedicated prompt
62
- // channel separate from the no-read inlineContext channel, so it never trips
63
- // resolveReviewerPromptAccessMode's no-read-only inlineContext guard.
46
+ // commit-range diff tool (providesRangeDiffTool === true, e.g. openai-compatible)
47
+ // inspect the range with that tool and do not receive an inlined diff, and
48
+ // tool-access reviewers run shell git themselves.
64
49
  export function readOnlyReviewerNeedsInlinedDiff(reviewer) {
65
50
  if (!isRegisteredProviderId(reviewer.provider)) {
66
51
  return false;
@@ -37,6 +37,13 @@ export function buildReviewerContextPacket(args) {
37
37
  claim: finding.claim,
38
38
  requiredAction: finding.requiredAction,
39
39
  }));
40
+ const inheritedPlanReviewDebt = state.inheritedPlanReviewDebt.map((item) => ({
41
+ canonicalId: item.canonicalId,
42
+ findingClass: item.findingClass ?? null,
43
+ originRound: item.originRound ?? null,
44
+ claim: item.claim,
45
+ requiredAction: item.requiredAction,
46
+ }));
40
47
  const finalCompletion = summarizeFinalCompletion(state);
41
48
  const packetWithoutMarkdown = {
42
49
  version: 1,
@@ -55,6 +62,7 @@ export function buildReviewerContextPacket(args) {
55
62
  },
56
63
  completedScopes,
57
64
  findings,
65
+ inheritedPlanReviewDebt,
58
66
  finalCompletion,
59
67
  citations: buildReviewerContextCitations(state),
60
68
  limits: {
@@ -100,21 +108,21 @@ function buildReviewerContextCitations(state) {
100
108
  { label: REVIEWER_CONTEXT_MARKDOWN, path: toDisplayPath(state.cwd, join(state.runDir, REVIEWER_CONTEXT_MARKDOWN)) },
101
109
  ];
102
110
  }
103
- export function renderReviewerContextMarkdown(packet, options = {}) {
104
- const mode = options.mode ?? 'tool-access';
111
+ export function renderReviewerContextMarkdown(packet) {
105
112
  const scopeLines = packet.completedScopes.length
106
113
  ? packet.completedScopes.map((scope) => `- Scope ${scope.number}: ${scope.result}; marker=${scope.marker}; finalCommit=${scope.finalCommit ?? 'none'}; reviewRounds=${scope.reviewRounds}; findings=${scope.findings}; residualDebt=${scope.residualReviewDebt}; summary=${scope.summary ?? 'none'}`)
107
114
  : ['- none'];
108
115
  const findingLines = packet.findings.length
109
116
  ? packet.findings.map((finding) => `- ${finding.canonicalId} (${finding.id}): ${finding.severity}/${finding.status}; source=${finding.source}; files=${finding.files.join(', ') || 'none'}; claim=${finding.claim}; requiredAction=${finding.requiredAction}`)
110
117
  : ['- none'];
118
+ const inheritedDebtLines = packet.inheritedPlanReviewDebt.length
119
+ ? packet.inheritedPlanReviewDebt.map((item) => `- ${item.canonicalId}: findingClass=${item.findingClass ?? 'n/a'}; originRound=${item.originRound ?? 'n/a'}; claim=${item.claim}; requiredAction=${item.requiredAction}`)
120
+ : ['- none'];
111
121
  const finalCompletion = packet.finalCompletion;
112
122
  return [
113
123
  '# Reviewer Continuity Context',
114
124
  '',
115
- mode === 'inline'
116
- ? 'Use this bounded Neal-owned context for continuity only. All required review context is inlined into this prompt; you have no access to the cited run artifacts or the checkout.'
117
- : 'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
125
+ 'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
118
126
  '',
119
127
  '## Run',
120
128
  `- id: ${packet.run.id}`,
@@ -137,19 +145,16 @@ export function renderReviewerContextMarkdown(packet, options = {}) {
137
145
  ? `- truncated: showing latest ${packet.findings.length} of ${packet.limits.findingCount}`
138
146
  : '- truncated: no',
139
147
  '',
148
+ '## Inherited Plan-Review Debt',
149
+ ...inheritedDebtLines,
150
+ '',
140
151
  '## Final Completion',
141
152
  finalCompletion
142
153
  ? `- state=${finalCompletion.state}; effectiveAction=${finalCompletion.effectiveAction ?? 'none'}; hasSummary=${finalCompletion.hasSummary}; hasReviewVerdict=${finalCompletion.hasReviewVerdict}; continueExecutionCount=${finalCompletion.continueExecutionCount}; capReached=${finalCompletion.continueExecutionCapReached}`
143
154
  : '- not applicable',
144
- // Citation paths are reading instructions; a no-read reviewer cannot
145
- // follow them, so the inline rendering omits the section entirely.
146
- ...(mode === 'inline'
147
- ? []
148
- : [
149
- '',
150
- '## Citations',
151
- ...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
152
- ]),
155
+ '',
156
+ '## Citations',
157
+ ...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
153
158
  ].join('\n');
154
159
  }
155
160
  function runIdFromDir(runDir) {
@@ -0,0 +1,86 @@
1
+ import { readFileSync } from 'node:fs';
2
+ import { isAbsolute, join, resolve } from 'node:path';
3
+ function fail(context, message) {
4
+ throw new Error(`reviewer-eval manifest: ${context} ${message}`);
5
+ }
6
+ function asString(value, context, field) {
7
+ const candidate = value[field];
8
+ if (typeof candidate !== 'string' || candidate.trim() === '') {
9
+ fail(context, `must define a non-empty string ${field}`);
10
+ }
11
+ return candidate;
12
+ }
13
+ function parseExpectedFinding(raw, context) {
14
+ if (typeof raw !== 'object' || raw === null) {
15
+ fail(context, 'each expectedFindings entry must be an object');
16
+ }
17
+ return {
18
+ file: asString(raw, context, 'file'),
19
+ defectClass: asString(raw, context, 'defectClass'),
20
+ description: asString(raw, context, 'description'),
21
+ };
22
+ }
23
+ // Every path the diff adds or modifies, read from its `+++ b/<path>` headers.
24
+ function diffTargetFiles(diffText) {
25
+ const files = new Set();
26
+ for (const line of diffText.split('\n')) {
27
+ const match = /^\+\+\+ b\/(.+)$/.exec(line);
28
+ if (match) {
29
+ files.add(match[1].trim());
30
+ }
31
+ }
32
+ return files;
33
+ }
34
+ export function loadReviewerEvalManifest(manifestDir) {
35
+ const dir = isAbsolute(manifestDir) ? manifestDir : resolve(manifestDir);
36
+ const manifestPath = join(dir, 'manifest.json');
37
+ const parsed = JSON.parse(readFileSync(manifestPath, 'utf8'));
38
+ const rawFixtures = parsed.fixtures;
39
+ if (!Array.isArray(rawFixtures) || rawFixtures.length === 0) {
40
+ fail('manifest.json', 'must define a non-empty fixtures array');
41
+ }
42
+ const seen = new Set();
43
+ return rawFixtures.map((raw, index) => {
44
+ const context = `fixtures[${index}]`;
45
+ if (typeof raw !== 'object' || raw === null) {
46
+ fail(context, 'must be an object');
47
+ }
48
+ const id = asString(raw, context, 'id');
49
+ if (seen.has(id)) {
50
+ fail(context, `duplicate fixture id ${id}`);
51
+ }
52
+ seen.add(id);
53
+ const kind = raw.kind;
54
+ if (kind !== 'defective' && kind !== 'clean') {
55
+ fail(id, "kind must be 'defective' or 'clean'");
56
+ }
57
+ const diff = asString(raw, id, 'diff');
58
+ const baseDirRelative = asString(raw, id, 'baseDir');
59
+ const rawExpected = raw.expectedFindings;
60
+ if (!Array.isArray(rawExpected)) {
61
+ fail(id, 'expectedFindings must be an array');
62
+ }
63
+ const expectedFindings = rawExpected.map((entry) => parseExpectedFinding(entry, id));
64
+ if (kind === 'defective' && expectedFindings.length === 0) {
65
+ fail(id, 'a defective fixture must label at least one expected finding');
66
+ }
67
+ if (kind === 'clean' && expectedFindings.length > 0) {
68
+ fail(id, 'a clean fixture must not label any expected findings');
69
+ }
70
+ const diffPath = join(dir, diff);
71
+ const diffText = readFileSync(diffPath, 'utf8');
72
+ const targetFiles = diffTargetFiles(diffText);
73
+ for (const expected of expectedFindings) {
74
+ if (!targetFiles.has(expected.file)) {
75
+ fail(id, `expected finding file ${expected.file} is not modified by ${diff}`);
76
+ }
77
+ }
78
+ return {
79
+ id,
80
+ kind,
81
+ expectedFindings,
82
+ diffPath,
83
+ baseDir: join(dir, baseDirRelative),
84
+ };
85
+ });
86
+ }
@@ -0,0 +1,155 @@
1
+ // Reviewer-recall eval scoring (issue #22 Phase B).
2
+ //
3
+ // Given a labeled fixture set — diffs with known defects, plus clean diffs —
4
+ // and the findings a reviewer produced for each, this computes recall,
5
+ // precision, and blocking-finding rate. It is the measurement instrument for
6
+ // the eval-gated reviewer-doctrine rewrite (Phase C): run it against a
7
+ // reviewer prompt before and after a change and compare the numbers.
8
+ //
9
+ // The scoring here is pure and deterministic; the live reviewer invocation
10
+ // lives in scripts/eval-reviewer.mjs (operator-run, subscription-billed, like
11
+ // scripts/qualify-sdk.sh). Matching is intentionally coarse — a blocking
12
+ // finding matches an expected label when they name the same file — because a
13
+ // finer match (defect-class, line range) would encode judgment the labels
14
+ // cannot reliably carry. The coarseness is documented so a reader does not
15
+ // over-trust the numbers: recall counts "did the reviewer flag the right
16
+ // file", not "did it describe the exact defect".
17
+ /**
18
+ * Projects a reviewer round's raw findings (as produced by the review-findings
19
+ * loop: objects carrying at least a severity and a files list) into a scored
20
+ * observation. Unknown severities are treated as non_blocking so a malformed
21
+ * finding cannot inflate the blocking count, and a missing files list becomes
22
+ * empty. Keeping this here lets the operator runner stay a thin shell over the
23
+ * tested scoring core.
24
+ */
25
+ export function toReviewerEvalObservation(fixtureId, rawFindings) {
26
+ const findings = rawFindings.map((finding) => ({
27
+ severity: finding.severity === 'blocking' ? 'blocking' : 'non_blocking',
28
+ files: Array.isArray(finding.files) ? finding.files.filter((file) => typeof file === 'string') : [],
29
+ }));
30
+ return { fixtureId, findings };
31
+ }
32
+ function normalizeFilePath(path) {
33
+ return path.trim().replace(/\\/g, '/').replace(/^\.\//, '');
34
+ }
35
+ function findingFileSet(observation) {
36
+ const files = new Set();
37
+ for (const finding of observation.findings) {
38
+ if (finding.severity !== 'blocking') {
39
+ continue;
40
+ }
41
+ for (const file of finding.files) {
42
+ files.add(normalizeFilePath(file));
43
+ }
44
+ }
45
+ return files;
46
+ }
47
+ /**
48
+ * Scores reviewer observations against labeled fixtures. Every fixture must
49
+ * have exactly one observation (missing observations throw — a fixture the
50
+ * runner failed to evaluate must not silently count as perfect recall).
51
+ */
52
+ export function scoreReviewerEval(fixtures, observations) {
53
+ const observationById = new Map(observations.map((observation) => [observation.fixtureId, observation]));
54
+ const fixtureResults = [];
55
+ let totalExpectedLabels = 0;
56
+ let matchedLabels = 0;
57
+ let totalBlockingFindings = 0;
58
+ let truePositiveFindings = 0;
59
+ let falsePositiveFindings = 0;
60
+ let cleanFixturesFlagged = 0;
61
+ for (const fixture of fixtures) {
62
+ const observation = observationById.get(fixture.id);
63
+ if (!observation) {
64
+ throw new Error(`scoreReviewerEval: no observation for fixture ${fixture.id}`);
65
+ }
66
+ const blockingFindings = observation.findings.filter((finding) => finding.severity === 'blocking');
67
+ const blockingFindingCount = blockingFindings.length;
68
+ const findingFiles = findingFileSet(observation);
69
+ const matched = [];
70
+ const missed = [];
71
+ const labeledFiles = new Set();
72
+ for (const expected of fixture.expectedFindings) {
73
+ const file = normalizeFilePath(expected.file);
74
+ labeledFiles.add(file);
75
+ if (findingFiles.has(file)) {
76
+ matched.push(expected);
77
+ }
78
+ else {
79
+ missed.push(expected);
80
+ }
81
+ }
82
+ // A blocking finding is a true positive when it touches at least one
83
+ // labeled file; every other blocking finding (all of them on a clean
84
+ // fixture) is a false positive.
85
+ let truePositiveFindingCount = 0;
86
+ let falsePositiveFindingCount = 0;
87
+ for (const finding of blockingFindings) {
88
+ const touchesLabel = finding.files.some((file) => labeledFiles.has(normalizeFilePath(file)));
89
+ if (touchesLabel) {
90
+ truePositiveFindingCount += 1;
91
+ }
92
+ else {
93
+ falsePositiveFindingCount += 1;
94
+ }
95
+ }
96
+ totalExpectedLabels += fixture.expectedFindings.length;
97
+ matchedLabels += matched.length;
98
+ totalBlockingFindings += blockingFindingCount;
99
+ truePositiveFindings += truePositiveFindingCount;
100
+ falsePositiveFindings += falsePositiveFindingCount;
101
+ if (fixture.kind === 'clean' && blockingFindingCount > 0) {
102
+ cleanFixturesFlagged += 1;
103
+ }
104
+ fixtureResults.push({
105
+ fixtureId: fixture.id,
106
+ kind: fixture.kind,
107
+ matchedLabels: matched,
108
+ missedLabels: missed,
109
+ truePositiveFindingCount,
110
+ falsePositiveFindingCount,
111
+ blockingFindingCount,
112
+ });
113
+ }
114
+ const defectiveFixtures = fixtures.filter((fixture) => fixture.kind === 'defective').length;
115
+ const cleanFixtures = fixtures.filter((fixture) => fixture.kind === 'clean').length;
116
+ return {
117
+ fixtures: fixtureResults,
118
+ totalFixtures: fixtures.length,
119
+ defectiveFixtures,
120
+ cleanFixtures,
121
+ recall: totalExpectedLabels === 0 ? null : matchedLabels / totalExpectedLabels,
122
+ precision: totalBlockingFindings === 0 ? null : truePositiveFindings / totalBlockingFindings,
123
+ totalExpectedLabels,
124
+ matchedLabels,
125
+ totalBlockingFindings,
126
+ truePositiveFindings,
127
+ falsePositiveFindings,
128
+ cleanFalsePositiveRate: cleanFixtures === 0 ? null : cleanFixturesFlagged / cleanFixtures,
129
+ cleanFixturesFlagged,
130
+ blockingFindingRate: fixtures.length === 0 ? 0 : totalBlockingFindings / fixtures.length,
131
+ };
132
+ }
133
+ /**
134
+ * Renders a report as a compact human-readable table plus a summary line.
135
+ * The machine-readable form is the ReviewerEvalReport itself (write it as
136
+ * JSON); this is for terminal output.
137
+ */
138
+ export function renderReviewerEvalReport(report) {
139
+ const pct = (value) => (value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`);
140
+ const lines = [];
141
+ lines.push('| Fixture | Kind | Labels hit | Blocking | False+ |');
142
+ lines.push('|---|---|---:|---:|---:|');
143
+ for (const fixture of report.fixtures) {
144
+ const labels = fixture.kind === 'defective'
145
+ ? `${fixture.matchedLabels.length}/${fixture.matchedLabels.length + fixture.missedLabels.length}`
146
+ : '—';
147
+ lines.push(`| ${fixture.fixtureId} | ${fixture.kind} | ${labels} | ${fixture.blockingFindingCount} | ${fixture.falsePositiveFindingCount} |`);
148
+ }
149
+ lines.push('');
150
+ lines.push(`Recall ${pct(report.recall)} (${report.matchedLabels}/${report.totalExpectedLabels} labels) · ` +
151
+ `Precision ${pct(report.precision)} (${report.truePositiveFindings}/${report.totalBlockingFindings} findings) · ` +
152
+ `Clean false-positive ${pct(report.cleanFalsePositiveRate)} (${report.cleanFixturesFlagged}/${report.cleanFixtures}) · ` +
153
+ `Blocking rate ${report.blockingFindingRate.toFixed(2)}/fixture`);
154
+ return lines.join('\n');
155
+ }
@@ -1,6 +1,11 @@
1
1
  import { existsSync } from 'node:fs';
2
2
  import readline from 'node:readline';
3
3
  import { clearDiagnosticFooter, getBufferedDetailSnapshot, isDiagnosticDetailVisible, showDiagnosticDetailView, showDiagnosticNarrativeView, writeNarrative, } from './diagnostic.js';
4
+ export function renderInteractiveKeyHint(allowStopRequest) {
5
+ return allowStopRequest
6
+ ? '[neal] keys: q stop after current scope, v show/hide details\n'
7
+ : '[neal] keys: v show/hide details\n';
8
+ }
4
9
  export function createInteractiveKeyController(options = {}) {
5
10
  let stopRequested = false;
6
11
  let cleanedUp = false;
@@ -21,7 +21,7 @@ import { saveState } from '../state.js';
21
21
  import { getFinalCompletionReviewerScratchDir, getScopeReviewerScratchDir } from '../storage-paths.js';
22
22
  import { requireFinalCompletionView, resolveFinalCompletionReviewAction, } from '../state-views.js';
23
23
  import { filterAllowedDirtyPathStatus, filterWrapperOwnedWorktreeStatus, formatDirtyWorktreeDiagnostic, } from '../worktree-status.js';
24
- import { shouldNotifyFailure } from './failures.js';
24
+ import { REVIEWER_CONTENT_REFUSED_BLOCK_REASON, shouldNotifyFailure } from './failures.js';
25
25
  import { persistUnattendedBlockUnresolvedFailure } from './phases/shared.js';
26
26
  async function writeFailedFinalCompletionReviewArtifact(args) {
27
27
  const unstructuredOutput = await readFinalCompletionUnstructuredOutput({
@@ -52,6 +52,38 @@ async function persistFailedFinalCompletionReview(args) {
52
52
  });
53
53
  return failedState;
54
54
  }
55
+ // Sibling of persistFailedFinalCompletionReview for the terminal blocked
56
+ // landing of a reviewer content-safety refusal (`content_refused`). Only a
57
+ // ReviewerRoundError can carry that kind; FinalCompletionReviewerVerdictError
58
+ // never does, so persistFailedFinalCompletionReview keeps serving the
59
+ // verdict-invalid path unchanged. Persists a terminal
60
+ // phase:'blocked'/status:'blocked' state so the run ends blocked (exit 2)
61
+ // rather than as a generic terminal failure, still writing the failed
62
+ // final-completion review artifact and execution artifacts.
63
+ async function persistBlockedFinalCompletionReview(args) {
64
+ const blockedState = await saveState(args.statePath, {
65
+ ...args.state,
66
+ phase: 'blocked',
67
+ reviewerSessionHandle: null,
68
+ status: 'blocked',
69
+ blockedFromPhase: null,
70
+ blockerReason: REVIEWER_CONTENT_REFUSED_BLOCK_REASON,
71
+ });
72
+ await args.runtime.writeExecutionArtifacts(blockedState);
73
+ await writeFailedFinalCompletionReviewArtifact({
74
+ state: blockedState,
75
+ source: 'reviewer_verdict',
76
+ sessionHandle: args.error.sessionHandle,
77
+ });
78
+ await args.logger?.event('phase.error', {
79
+ phase: 'final_completion_review',
80
+ sessionHandle: args.error.sessionHandle,
81
+ subtype: args.error.subtype,
82
+ errorKind: args.error.kind,
83
+ message: args.error.message,
84
+ });
85
+ return blockedState;
86
+ }
55
87
  function normalizeFinalCommitMessage(message) {
56
88
  const normalizedNewlines = message.replace(/\r\n/g, '\n');
57
89
  const convertedEscapes = normalizedNewlines.replace(/\\n(?=- )/g, '\n');
@@ -480,6 +512,21 @@ export async function runFinalCompletionReviewPhase(state, statePath, logger, ru
480
512
  }));
481
513
  }
482
514
  catch (error) {
515
+ // A reviewer content-safety refusal ends the run blocked (exit 2) with a
516
+ // durable actionable reason, by RETURNING the terminal blocked state rather
517
+ // than throwing. Only a ReviewerRoundError carries `content_refused`; the
518
+ // verdict-invalid error never does, so it stays on the failed path below.
519
+ if (error instanceof ReviewerRoundError && error.kind === 'content_refused') {
520
+ const blockedState = await persistBlockedFinalCompletionReview({
521
+ state,
522
+ statePath,
523
+ error,
524
+ logger,
525
+ runtime,
526
+ });
527
+ await notifyBlocked(blockedState, REVIEWER_CONTENT_REFUSED_BLOCK_REASON, logger);
528
+ return blockedState;
529
+ }
483
530
  if (error instanceof ReviewerRoundError || error instanceof FinalCompletionReviewerVerdictError) {
484
531
  const failedState = await persistFailedFinalCompletionReview({
485
532
  state,
@@ -539,7 +586,7 @@ export async function runFinalCompletionReviewPhase(state, statePath, logger, ru
539
586
  // budgets reset here exactly as they do at every scope-advance
540
587
  // transition, so an earlier scope's adjudication or split-plan
541
588
  // consumption never exhausts the reopened scope's budget.
542
- // (`reviewStuckArbiterCount` resets via the shared next-scope
589
+ // (`consultantAttemptCount` resets via the shared next-scope
543
590
  // reset spread above.)
544
591
  splitPlanCountForCurrentScope: 0,
545
592
  blockedFromPhase: null,
@@ -1,4 +1,11 @@
1
1
  import { CoderRoundError } from '../agents.js';
2
+ // Shared, durable blocked reason for a reviewer content-safety refusal
3
+ // (`content_refused`). Defined once so the wording cannot drift across the
4
+ // three reviewer-phase catch sites that route it to a terminal blocked landing.
5
+ export const REVIEWER_CONTENT_REFUSED_BLOCK_REASON = 'The reviewer refused this content on content-safety grounds. The change ' +
6
+ 'under review may read as security-adjacent. Switch the reviewer to a ' +
7
+ 'different provider, rephrase the scope, or seek provider authorization, ' +
8
+ 'then re-run.';
2
9
  export function isCoderTimeoutError(error) {
3
10
  return (error.kind === 'timeout' ||
4
11
  error.kind === 'no_progress_timeout' ||
@@ -3,40 +3,40 @@ import { notify } from '../../notifier.js';
3
3
  import { getCurrentScopeLabel, getExecutionPlanPath, getExecutionPlanScopeCount, getParentScopeLabel, renderScopeProgressSegments, } from '../scopes.js';
4
4
  import { saveState } from '../state.js';
5
5
  import { getDerivedPlanView } from '../state-views.js';
6
- // Concise representation of the read-only blocked-adjudicator advice for the
6
+ // Concise representation of the read-only consultant advice for the
7
7
  // operator notification surface. Present only on an attended run whose active
8
- // interactive-blocked-recovery record carries adjudicator advice (knob > 0,
8
+ // interactive-blocked-recovery record carries consultant advice (knob > 0,
9
9
  // budget available, eligible source phase). Returns '' otherwise so notification
10
10
  // behavior for recovery states without advice — and terminal blocked
11
11
  // notifications, where the active record is already finalized to null — is
12
12
  // unchanged.
13
- function adjudicatorAdviceNotificationSuffix(state) {
14
- const advice = state.interactiveBlockedRecovery?.adjudicatorAdvice;
13
+ function consultantAdviceNotificationSuffix(state) {
14
+ const advice = state.interactiveBlockedRecovery?.consultantAdvice;
15
15
  if (!advice) {
16
16
  return '';
17
17
  }
18
18
  const directive = advice.resolutionDirective.trim() || 'n/a';
19
- return ` | adjudicator advice (read-only): triage ${advice.triageCategory}; suggested directive: ${directive}`;
19
+ return ` | consultant advice (read-only): triage ${advice.triageCategory}; suggested directive: ${directive}`;
20
20
  }
21
21
  async function notifyBlocked(state, reason, logger) {
22
22
  const planName = basename(state.planDoc);
23
- const adviceSuffix = adjudicatorAdviceNotificationSuffix(state);
23
+ const adviceSuffix = consultantAdviceNotificationSuffix(state);
24
24
  await logger?.event('notify.blocked', {
25
25
  reason,
26
26
  planName,
27
- adjudicatorAdvice: state.interactiveBlockedRecovery?.adjudicatorAdvice ?? null,
27
+ consultantAdvice: state.interactiveBlockedRecovery?.consultantAdvice ?? null,
28
28
  });
29
29
  await notify('blocked', `[neal] ${planName}: ${reason}${adviceSuffix}`, state.cwd);
30
30
  }
31
31
  async function notifyInteractiveBlockedRecovery(state, reason, logger) {
32
32
  const planName = basename(state.planDoc);
33
33
  const scopeLabel = getCurrentScopeLabel(state);
34
- const adviceSuffix = adjudicatorAdviceNotificationSuffix(state);
34
+ const adviceSuffix = consultantAdviceNotificationSuffix(state);
35
35
  await logger?.event('notify.interactive_blocked_recovery', {
36
36
  reason,
37
37
  planName,
38
38
  scopeNumber: scopeLabel,
39
- adjudicatorAdvice: state.interactiveBlockedRecovery?.adjudicatorAdvice ?? null,
39
+ consultantAdvice: state.interactiveBlockedRecovery?.consultantAdvice ?? null,
40
40
  });
41
41
  await notify('retry', `[neal] ${planName}: interactive blocked recovery for scope ${scopeLabel}: ${reason}${adviceSuffix}`, state.cwd);
42
42
  }