@navels/neal 0.1.0 → 0.3.0

package/dist/neal/context/inline-review-context.js

@@ -4,8 +4,9 @@ import { getProviderDefinition, isRegisteredProviderId } from '../providers/regi
 // truncation posture of truncateForPrompt in src/neal/agents/structured-json.ts:
 // truncate with an explicit marker instead of silently dropping content.
 export const INLINE_SECTION_MAX_CHARS = 200_000;
-// Canonical forbidden-phrase list for no-read reviewer prompts. A no-read
-// reviewer prompt must contain no instruction that requires repository, file,
+// Canonical forbidden-phrase list for no-read prompts (today: the blocked-run
+// consultant, which always judges from Neal-inlined in-memory context). A
+// no-read prompt must contain no instruction that requires repository, file,
 // tool, or shell access of any kind. Both the prompt-builder implementations
 // and the runtime capture tests assert against this shared list, so a newly
 // added repo-access phrase fails the shared assertion rather than silently
@@ -22,31 +23,18 @@ export const NO_READ_PROMPT_FORBIDDEN_MARKERS = [
     'is available at',
     'Inspect cited artifacts',
 ];
-// True when the configured reviewer provider's structured-advisor role has no
-// repository read access, so Neal must inline the review context (diff, plan,
-// artifacts) into reviewer prompts.
-export function reviewerNeedsInlineContext(reviewer) {
-    if (!isRegisteredProviderId(reviewer.provider)) {
-        return false;
-    }
-    return getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess.read === false;
-}
-// Three-way review-doctrine access mode for the configured reviewer provider's
-// structured-advisor role. Unregistered provider ids default to 'tool-access',
-// mirroring reviewerNeedsInlineContext's false. No repository read access
-// selects 'no-read' (Neal inlines the review context); read access without
-// shell access selects 'read-only' (the reviewer inspects the repository with
-// read tools but must not be instructed to run commands); read plus shell
-// access selects 'tool-access'. Write access does not affect review doctrine:
-// reviews never instruct repository mutation.
+// Two-way review-doctrine access mode for the configured reviewer provider's
+// structured-advisor role. Unregistered provider ids default to 'tool-access'.
+// Every registered reviewer provider has repository read access; read access
+// without shell access selects 'read-only' (the reviewer inspects the
+// repository with read tools but must not be instructed to run commands);
+// read plus shell access selects 'tool-access'. Write access does not affect
+// review doctrine: reviews never instruct repository mutation.
 export function getReviewerDoctrineAccessMode(reviewer) {
     if (!isRegisteredProviderId(reviewer.provider)) {
         return 'tool-access';
     }
     const toolAccess = getProviderDefinition(reviewer.provider).capabilities['structured-advisor'].toolAccess;
-    if (toolAccess.read === false) {
-        return 'no-read';
-    }
     if (toolAccess.shell === false) {
         return 'read-only';
     }
@@ -55,12 +43,9 @@ export function getReviewerDoctrineAccessMode(reviewer) {
 // True when the configured reviewer is a read-only reviewer (read tools, no
 // shell) that has no commit-range diff tool of its own, so Neal must inline the
 // commit-range diff into its reviewer prompt. Read-only reviewers that expose a
-// commit-range diff tool (providesRangeDiffTool === true, e.g. generic-agentic)
-// inspect the range with that tool and do not receive an inlined diff; no-read
-// reviewers get the full inlineContext instead, and tool-access reviewers run
-// shell git themselves. The inlined commit-range diff rides a dedicated prompt
-// channel separate from the no-read inlineContext channel, so it never trips
-// resolveReviewerPromptAccessMode's no-read-only inlineContext guard.
+// commit-range diff tool (providesRangeDiffTool === true, e.g. openai-compatible)
+// inspect the range with that tool and do not receive an inlined diff, and
+// tool-access reviewers run shell git themselves.
 export function readOnlyReviewerNeedsInlinedDiff(reviewer) {
     if (!isRegisteredProviderId(reviewer.provider)) {
         return false;

package/dist/neal/context/reviewer-context.js

@@ -37,6 +37,13 @@ export function buildReviewerContextPacket(args) {
         claim: finding.claim,
         requiredAction: finding.requiredAction,
     }));
+    const inheritedPlanReviewDebt = state.inheritedPlanReviewDebt.map((item) => ({
+        canonicalId: item.canonicalId,
+        findingClass: item.findingClass ?? null,
+        originRound: item.originRound ?? null,
+        claim: item.claim,
+        requiredAction: item.requiredAction,
+    }));
     const finalCompletion = summarizeFinalCompletion(state);
     const packetWithoutMarkdown = {
         version: 1,
@@ -55,6 +62,7 @@ export function buildReviewerContextPacket(args) {
         },
         completedScopes,
         findings,
+        inheritedPlanReviewDebt,
         finalCompletion,
         citations: buildReviewerContextCitations(state),
         limits: {
@@ -100,21 +108,21 @@ function buildReviewerContextCitations(state) {
         { label: REVIEWER_CONTEXT_MARKDOWN, path: toDisplayPath(state.cwd, join(state.runDir, REVIEWER_CONTEXT_MARKDOWN)) },
     ];
 }
-export function renderReviewerContextMarkdown(packet, options = {}) {
-    const mode = options.mode ?? 'tool-access';
+export function renderReviewerContextMarkdown(packet) {
     const scopeLines = packet.completedScopes.length
         ? packet.completedScopes.map((scope) => `- Scope ${scope.number}: ${scope.result}; marker=${scope.marker}; finalCommit=${scope.finalCommit ?? 'none'}; reviewRounds=${scope.reviewRounds}; findings=${scope.findings}; residualDebt=${scope.residualReviewDebt}; summary=${scope.summary ?? 'none'}`)
         : ['- none'];
     const findingLines = packet.findings.length
         ? packet.findings.map((finding) => `- ${finding.canonicalId} (${finding.id}): ${finding.severity}/${finding.status}; source=${finding.source}; files=${finding.files.join(', ') || 'none'}; claim=${finding.claim}; requiredAction=${finding.requiredAction}`)
         : ['- none'];
+    const inheritedDebtLines = packet.inheritedPlanReviewDebt.length
+        ? packet.inheritedPlanReviewDebt.map((item) => `- ${item.canonicalId}: findingClass=${item.findingClass ?? 'n/a'}; originRound=${item.originRound ?? 'n/a'}; claim=${item.claim}; requiredAction=${item.requiredAction}`)
+        : ['- none'];
     const finalCompletion = packet.finalCompletion;
     return [
         '# Reviewer Continuity Context',
         '',
-        mode === 'inline'
-            ? 'Use this bounded Neal-owned context for continuity only. All required review context is inlined into this prompt; you have no access to the cited run artifacts or the checkout.'
-            : 'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
+        'Use this bounded Neal-owned context for continuity only. Inspect cited artifacts or repository state directly before making findings.',
         '',
         '## Run',
         `- id: ${packet.run.id}`,
@@ -137,19 +145,16 @@ export function renderReviewerContextMarkdown(packet, options = {}) {
             ? `- truncated: showing latest ${packet.findings.length} of ${packet.limits.findingCount}`
             : '- truncated: no',
         '',
+        '## Inherited Plan-Review Debt',
+        ...inheritedDebtLines,
+        '',
         '## Final Completion',
         finalCompletion
             ? `- state=${finalCompletion.state}; effectiveAction=${finalCompletion.effectiveAction ?? 'none'}; hasSummary=${finalCompletion.hasSummary}; hasReviewVerdict=${finalCompletion.hasReviewVerdict}; continueExecutionCount=${finalCompletion.continueExecutionCount}; capReached=${finalCompletion.continueExecutionCapReached}`
             : '- not applicable',
-        // Citation paths are reading instructions; a no-read reviewer cannot
-        // follow them, so the inline rendering omits the section entirely.
-        ...(mode === 'inline'
-            ? []
-            : [
-                '',
-                '## Citations',
-                ...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
-            ]),
+        '',
+        '## Citations',
+        ...packet.citations.map((citation) => `- ${citation.label}: ${citation.path}`),
     ].join('\n');
 }
 function runIdFromDir(runDir) {

package/dist/neal/eval/reviewer-eval-manifest.js

@@ -0,0 +1,86 @@
+import { readFileSync } from 'node:fs';
+import { isAbsolute, join, resolve } from 'node:path';
+function fail(context, message) {
+    throw new Error(`reviewer-eval manifest: ${context} ${message}`);
+}
+function asString(value, context, field) {
+    const candidate = value[field];
+    if (typeof candidate !== 'string' || candidate.trim() === '') {
+        fail(context, `must define a non-empty string ${field}`);
+    }
+    return candidate;
+}
+function parseExpectedFinding(raw, context) {
+    if (typeof raw !== 'object' || raw === null) {
+        fail(context, 'each expectedFindings entry must be an object');
+    }
+    return {
+        file: asString(raw, context, 'file'),
+        defectClass: asString(raw, context, 'defectClass'),
+        description: asString(raw, context, 'description'),
+    };
+}
+// Every path the diff adds or modifies, read from its `+++ b/<path>` headers.
+function diffTargetFiles(diffText) {
+    const files = new Set();
+    for (const line of diffText.split('\n')) {
+        const match = /^\+\+\+ b\/(.+)$/.exec(line);
+        if (match) {
+            files.add(match[1].trim());
+        }
+    }
+    return files;
+}
+export function loadReviewerEvalManifest(manifestDir) {
+    const dir = isAbsolute(manifestDir) ? manifestDir : resolve(manifestDir);
+    const manifestPath = join(dir, 'manifest.json');
+    const parsed = JSON.parse(readFileSync(manifestPath, 'utf8'));
+    const rawFixtures = parsed.fixtures;
+    if (!Array.isArray(rawFixtures) || rawFixtures.length === 0) {
+        fail('manifest.json', 'must define a non-empty fixtures array');
+    }
+    const seen = new Set();
+    return rawFixtures.map((raw, index) => {
+        const context = `fixtures[${index}]`;
+        if (typeof raw !== 'object' || raw === null) {
+            fail(context, 'must be an object');
+        }
+        const id = asString(raw, context, 'id');
+        if (seen.has(id)) {
+            fail(context, `duplicate fixture id ${id}`);
+        }
+        seen.add(id);
+        const kind = raw.kind;
+        if (kind !== 'defective' && kind !== 'clean') {
+            fail(id, "kind must be 'defective' or 'clean'");
+        }
+        const diff = asString(raw, id, 'diff');
+        const baseDirRelative = asString(raw, id, 'baseDir');
+        const rawExpected = raw.expectedFindings;
+        if (!Array.isArray(rawExpected)) {
+            fail(id, 'expectedFindings must be an array');
+        }
+        const expectedFindings = rawExpected.map((entry) => parseExpectedFinding(entry, id));
+        if (kind === 'defective' && expectedFindings.length === 0) {
+            fail(id, 'a defective fixture must label at least one expected finding');
+        }
+        if (kind === 'clean' && expectedFindings.length > 0) {
+            fail(id, 'a clean fixture must not label any expected findings');
+        }
+        const diffPath = join(dir, diff);
+        const diffText = readFileSync(diffPath, 'utf8');
+        const targetFiles = diffTargetFiles(diffText);
+        for (const expected of expectedFindings) {
+            if (!targetFiles.has(expected.file)) {
+                fail(id, `expected finding file ${expected.file} is not modified by ${diff}`);
+            }
+        }
+        return {
+            id,
+            kind,
+            expectedFindings,
+            diffPath,
+            baseDir: join(dir, baseDirRelative),
+        };
+    });
+}

package/dist/neal/eval/reviewer-eval.js

@@ -0,0 +1,155 @@
+// Reviewer-recall eval scoring (issue #22 Phase B).
+//
+// Given a labeled fixture set — diffs with known defects, plus clean diffs —
+// and the findings a reviewer produced for each, this computes recall,
+// precision, and blocking-finding rate. It is the measurement instrument for
+// the eval-gated reviewer-doctrine rewrite (Phase C): run it against a
+// reviewer prompt before and after a change and compare the numbers.
+//
+// The scoring here is pure and deterministic; the live reviewer invocation
+// lives in scripts/eval-reviewer.mjs (operator-run, subscription-billed, like
+// scripts/qualify-sdk.sh). Matching is intentionally coarse — a blocking
+// finding matches an expected label when they name the same file — because a
+// finer match (defect-class, line range) would encode judgment the labels
+// cannot reliably carry. The coarseness is documented so a reader does not
+// over-trust the numbers: recall counts "did the reviewer flag the right
+// file", not "did it describe the exact defect".
+/**
+ * Projects a reviewer round's raw findings (as produced by the review-findings
+ * loop: objects carrying at least a severity and a files list) into a scored
+ * observation. Unknown severities are treated as non_blocking so a malformed
+ * finding cannot inflate the blocking count, and a missing files list becomes
+ * empty. Keeping this here lets the operator runner stay a thin shell over the
+ * tested scoring core.
+ */
+export function toReviewerEvalObservation(fixtureId, rawFindings) {
+    const findings = rawFindings.map((finding) => ({
+        severity: finding.severity === 'blocking' ? 'blocking' : 'non_blocking',
+        files: Array.isArray(finding.files) ? finding.files.filter((file) => typeof file === 'string') : [],
+    }));
+    return { fixtureId, findings };
+}
+function normalizeFilePath(path) {
+    return path.trim().replace(/\\/g, '/').replace(/^\.\//, '');
+}
+function findingFileSet(observation) {
+    const files = new Set();
+    for (const finding of observation.findings) {
+        if (finding.severity !== 'blocking') {
+            continue;
+        }
+        for (const file of finding.files) {
+            files.add(normalizeFilePath(file));
+        }
+    }
+    return files;
+}
+/**
+ * Scores reviewer observations against labeled fixtures. Every fixture must
+ * have exactly one observation (missing observations throw — a fixture the
+ * runner failed to evaluate must not silently count as perfect recall).
+ */
+export function scoreReviewerEval(fixtures, observations) {
+    const observationById = new Map(observations.map((observation) => [observation.fixtureId, observation]));
+    const fixtureResults = [];
+    let totalExpectedLabels = 0;
+    let matchedLabels = 0;
+    let totalBlockingFindings = 0;
+    let truePositiveFindings = 0;
+    let falsePositiveFindings = 0;
+    let cleanFixturesFlagged = 0;
+    for (const fixture of fixtures) {
+        const observation = observationById.get(fixture.id);
+        if (!observation) {
+            throw new Error(`scoreReviewerEval: no observation for fixture ${fixture.id}`);
+        }
+        const blockingFindings = observation.findings.filter((finding) => finding.severity === 'blocking');
+        const blockingFindingCount = blockingFindings.length;
+        const findingFiles = findingFileSet(observation);
+        const matched = [];
+        const missed = [];
+        const labeledFiles = new Set();
+        for (const expected of fixture.expectedFindings) {
+            const file = normalizeFilePath(expected.file);
+            labeledFiles.add(file);
+            if (findingFiles.has(file)) {
+                matched.push(expected);
+            }
+            else {
+                missed.push(expected);
+            }
+        }
+        // A blocking finding is a true positive when it touches at least one
+        // labeled file; every other blocking finding (all of them on a clean
+        // fixture) is a false positive.
+        let truePositiveFindingCount = 0;
+        let falsePositiveFindingCount = 0;
+        for (const finding of blockingFindings) {
+            const touchesLabel = finding.files.some((file) => labeledFiles.has(normalizeFilePath(file)));
+            if (touchesLabel) {
+                truePositiveFindingCount += 1;
+            }
+            else {
+                falsePositiveFindingCount += 1;
+            }
+        }
+        totalExpectedLabels += fixture.expectedFindings.length;
+        matchedLabels += matched.length;
+        totalBlockingFindings += blockingFindingCount;
+        truePositiveFindings += truePositiveFindingCount;
+        falsePositiveFindings += falsePositiveFindingCount;
+        if (fixture.kind === 'clean' && blockingFindingCount > 0) {
+            cleanFixturesFlagged += 1;
+        }
+        fixtureResults.push({
+            fixtureId: fixture.id,
+            kind: fixture.kind,
+            matchedLabels: matched,
+            missedLabels: missed,
+            truePositiveFindingCount,
+            falsePositiveFindingCount,
+            blockingFindingCount,
+        });
+    }
+    const defectiveFixtures = fixtures.filter((fixture) => fixture.kind === 'defective').length;
+    const cleanFixtures = fixtures.filter((fixture) => fixture.kind === 'clean').length;
+    return {
+        fixtures: fixtureResults,
+        totalFixtures: fixtures.length,
+        defectiveFixtures,
+        cleanFixtures,
+        recall: totalExpectedLabels === 0 ? null : matchedLabels / totalExpectedLabels,
+        precision: totalBlockingFindings === 0 ? null : truePositiveFindings / totalBlockingFindings,
+        totalExpectedLabels,
+        matchedLabels,
+        totalBlockingFindings,
+        truePositiveFindings,
+        falsePositiveFindings,
+        cleanFalsePositiveRate: cleanFixtures === 0 ? null : cleanFixturesFlagged / cleanFixtures,
+        cleanFixturesFlagged,
+        blockingFindingRate: fixtures.length === 0 ? 0 : totalBlockingFindings / fixtures.length,
+    };
+}
+/**
+ * Renders a report as a compact human-readable table plus a summary line.
+ * The machine-readable form is the ReviewerEvalReport itself (write it as
+ * JSON); this is for terminal output.
+ */
+export function renderReviewerEvalReport(report) {
+    const pct = (value) => (value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`);
+    const lines = [];
+    lines.push('| Fixture | Kind | Labels hit | Blocking | False+ |');
+    lines.push('|---|---|---:|---:|---:|');
+    for (const fixture of report.fixtures) {
+        const labels = fixture.kind === 'defective'
+            ? `${fixture.matchedLabels.length}/${fixture.matchedLabels.length + fixture.missedLabels.length}`
+            : '—';
+        lines.push(`| ${fixture.fixtureId} | ${fixture.kind} | ${labels} | ${fixture.blockingFindingCount} | ${fixture.falsePositiveFindingCount} |`);
+    }
+    lines.push('');
+    lines.push(`Recall ${pct(report.recall)} (${report.matchedLabels}/${report.totalExpectedLabels} labels) · ` +
+        `Precision ${pct(report.precision)} (${report.truePositiveFindings}/${report.totalBlockingFindings} findings) · ` +
+        `Clean false-positive ${pct(report.cleanFalsePositiveRate)} (${report.cleanFixturesFlagged}/${report.cleanFixtures}) · ` +
+        `Blocking rate ${report.blockingFindingRate.toFixed(2)}/fixture`);
+    return lines.join('\n');
+}

package/dist/neal/git.js

@@ -1,8 +1,14 @@
 import { execFile, spawn } from 'node:child_process';
 import { isAbsolute, resolve } from 'node:path';
+// Git output (diffs, status) scales with scope artifacts — a scope that
+// vendors or generates a large file produces a multi-megabyte diff, and
+// Node's default execFile maxBuffer (1 MiB) kills the run with
+// "stdout maxBuffer length exceeded". 64 MiB clears any realistic scope
+// diff while still bounding a runaway.
+const GIT_MAX_BUFFER = 64 * 1024 * 1024;
 function runGit(args, cwd) {
     return new Promise((resolvePromise, rejectPromise) => {
-        execFile('git', args, { cwd }, (error, stdout, stderr) => {
+        execFile('git', args, { cwd, maxBuffer: GIT_MAX_BUFFER }, (error, stdout, stderr) => {
             if (error) {
                 rejectPromise(new Error(stderr.trim() || error.message));
                 return;
@@ -13,7 +19,7 @@ function runGit(args, cwd) {
 }
 function runGitOptionalConfig(args, cwd) {
     return new Promise((resolvePromise, rejectPromise) => {
-        execFile('git', args, { cwd }, (error, stdout, stderr) => {
+        execFile('git', args, { cwd, maxBuffer: GIT_MAX_BUFFER }, (error, stdout, stderr) => {
             if (!error) {
                 resolvePromise(stdout.trim());
                 return;
@@ -158,7 +164,7 @@ export async function resolveCommitRef(cwd, ref) {
 }
 export async function isAncestorCommit(cwd, ancestor, descendant) {
     return new Promise((resolvePromise, rejectPromise) => {
-        execFile('git', ['merge-base', '--is-ancestor', ancestor, descendant], { cwd }, (error, _stdout, stderr) => {
+        execFile('git', ['merge-base', '--is-ancestor', ancestor, descendant], { cwd, maxBuffer: GIT_MAX_BUFFER }, (error, _stdout, stderr) => {
             if (!error) {
                 resolvePromise(true);
                 return;

package/dist/neal/interactive-controls.js

@@ -1,6 +1,11 @@
 import { existsSync } from 'node:fs';
 import readline from 'node:readline';
 import { clearDiagnosticFooter, getBufferedDetailSnapshot, isDiagnosticDetailVisible, showDiagnosticDetailView, showDiagnosticNarrativeView, writeNarrative, } from './diagnostic.js';
+export function renderInteractiveKeyHint(allowStopRequest) {
+    return allowStopRequest
+        ? '[neal] keys: q stop after current scope, v show/hide details\n'
+        : '[neal] keys: v show/hide details\n';
+}
 export function createInteractiveKeyController(options = {}) {
     let stopRequested = false;
     let cleanedUp = false;