@navels/neal 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -209
- package/SECURITY.md +16 -19
- package/dist/neal/adjudicator/{blocked-adjudicator.js → consultant.js} +21 -21
- package/dist/neal/adjudicator/execute.js +33 -33
- package/dist/neal/adjudicator/final-completion.js +2 -31
- package/dist/neal/adjudicator/planning.js +264 -24
- package/dist/neal/agents/prompts.js +7 -15
- package/dist/neal/agents/rounds.js +51 -73
- package/dist/neal/agents/schemas.js +615 -502
- package/dist/neal/agents/structured-json.js +36 -0
- package/dist/neal/blocked-guidance.js +16 -9
- package/dist/neal/cli.js +2 -2
- package/dist/neal/commands/check.js +2 -2
- package/dist/neal/commands/compat.js +100 -36
- package/dist/neal/commands/interactive-activity.js +2 -2
- package/dist/neal/commands/recovery-guidance.js +20 -8
- package/dist/neal/commands/runtime.js +4 -2
- package/dist/neal/config.js +40 -7
- package/dist/neal/context/inline-review-context.js +13 -28
- package/dist/neal/context/reviewer-context.js +19 -14
- package/dist/neal/eval/reviewer-eval-manifest.js +86 -0
- package/dist/neal/eval/reviewer-eval.js +155 -0
- package/dist/neal/git.js +9 -3
- package/dist/neal/interactive-controls.js +5 -0
- package/dist/neal/orchestrator/completion.js +215 -113
- package/dist/neal/orchestrator/failures.js +7 -0
- package/dist/neal/orchestrator/notifications.js +9 -9
- package/dist/neal/orchestrator/phases/planning.js +124 -112
- package/dist/neal/orchestrator/phases/recovery.js +151 -115
- package/dist/neal/orchestrator/phases/review.js +28 -1
- package/dist/neal/orchestrator/split-plan.js +12 -11
- package/dist/neal/orchestrator/transitions.js +36 -71
- package/dist/neal/orchestrator.js +3 -0
- package/dist/neal/plan-doc.js +24 -1
- package/dist/neal/plan-queue.js +29 -1
- package/dist/neal/prompts/assert-builder.js +8 -1
- package/dist/neal/prompts/execute.js +11 -19
- package/dist/neal/prompts/planning.js +64 -55
- package/dist/neal/prompts/review-doctrine.js +13 -50
- package/dist/neal/prompts/specialized.js +28 -22
- package/dist/neal/prompts/specs.js +223 -17
- package/dist/neal/providers/agent-settings-isolation.js +35 -0
- package/dist/neal/providers/anthropic-claude.js +309 -248
- package/dist/neal/providers/detection.js +3 -7
- package/dist/neal/providers/git-config-isolation.js +83 -0
- package/dist/neal/providers/openai-codex.js +102 -204
- package/dist/neal/providers/{generic-agentic-tools.js → openai-compatible-tools.js} +11 -5
- package/dist/neal/providers/openai-compatible.js +1079 -403
- package/dist/neal/providers/openrouter-routing.js +68 -0
- package/dist/neal/providers/pricing.js +124 -0
- package/dist/neal/providers/rate-card.js +2275 -0
- package/dist/neal/providers/registry.js +6 -17
- package/dist/neal/providers/telemetry.js +4 -0
- package/dist/neal/providers/types.js +12 -0
- package/dist/neal/resume-decision.js +7 -6
- package/dist/neal/resume-planner.js +9 -0
- package/dist/neal/retrospective.js +36 -7
- package/dist/neal/review-debt.js +30 -0
- package/dist/neal/review-findings/provider.js +17 -29
- package/dist/neal/review.js +17 -1
- package/dist/neal/run-metrics.js +74 -9
- package/dist/neal/state-invariants.js +11 -3
- package/dist/neal/state-views.js +27 -5
- package/dist/neal/state.js +59 -15
- package/dist/neal/status.js +28 -1
- package/dist/neal/support.js +3 -3
- package/docs/README.md +22 -18
- package/docs/{ADJUDICATOR_INVENTORY.md → adjudicator-inventory.md} +58 -57
- package/docs/architecture.md +79 -83
- package/docs/automation.md +11 -11
- package/docs/compat.md +91 -61
- package/docs/compatible-models.md +145 -110
- package/docs/demo.md +15 -5
- package/docs/maintenance.md +74 -32
- package/docs/plan-format.md +61 -33
- package/docs/prompt-evals.md +96 -0
- package/docs/{PROMPT_SPECS.md → prompt-specs.md} +52 -48
- package/docs/providers.md +330 -283
- package/docs/release.md +43 -40
- package/docs/review-convergence.md +246 -0
- package/docs/state-machine.md +61 -59
- package/docs/storage.md +40 -31
- package/docs/troubleshooting.md +32 -29
- package/examples/compat/add-edit-verify/PLAN.md +2 -0
- package/examples/compat/add-edit-verify/broken.diff +2 -2
- package/examples/compat/add-edit-verify/good.diff +2 -2
- package/examples/compat/add-edit-verify/src/add.js +1 -1
- package/examples/compat/manifest.json +25 -29
- package/examples/compat/sum-grep-edit/PLAN.md +33 -0
- package/examples/compat/sum-grep-edit/broken.diff +12 -0
- package/examples/compat/sum-grep-edit/good.diff +12 -0
- package/examples/compat/sum-grep-edit/package.json +5 -0
- package/examples/compat/sum-grep-edit/src/numbers.js +9 -0
- package/examples/compat/sum-grep-edit/test/numbers.test.js +15 -0
- package/examples/issue-triage-js/README.md +0 -15
- package/examples/reviewer-eval/clean-doc/base/src/clamp.ts +3 -0
- package/examples/reviewer-eval/clean-doc/change.diff +9 -0
- package/examples/reviewer-eval/clean-extract/base/src/greet.ts +3 -0
- package/examples/reviewer-eval/clean-extract/change.diff +13 -0
- package/examples/reviewer-eval/dropped-error/base/src/load.ts +4 -0
- package/examples/reviewer-eval/dropped-error/change.diff +15 -0
- package/examples/reviewer-eval/manifest.json +45 -0
- package/examples/reviewer-eval/off-by-one/base/src/paginate.ts +5 -0
- package/examples/reviewer-eval/off-by-one/change.diff +11 -0
- package/{config.yml → neal.yml} +1 -1
- package/package.json +20 -15
- package/dist/neal/providers/generic-agentic.js +0 -1238
- package/docs/comparison.md +0 -105
- package/examples/compat/is-even-add-test/PLAN.md +0 -30
- package/examples/compat/is-even-add-test/broken.diff +0 -11
- package/examples/compat/is-even-add-test/good.diff +0 -11
- package/examples/compat/is-even-add-test/package.json +0 -5
- package/examples/compat/is-even-add-test/src/is-even.js +0 -3
- package/examples/compat/is-even-add-test/test/is-even.test.js +0 -9
- package/examples/compat/reverse-grep-edit/PLAN.md +0 -32
- package/examples/compat/reverse-grep-edit/broken.diff +0 -12
- package/examples/compat/reverse-grep-edit/good.diff +0 -12
- package/examples/compat/reverse-grep-edit/package.json +0 -5
- package/examples/compat/reverse-grep-edit/src/strings.js +0 -10
- package/examples/compat/reverse-grep-edit/test/strings.test.js +0 -15
|
@@ -28,6 +28,42 @@ export function buildStructuredJsonPrompt(basePrompt, protocol) {
|
|
|
28
28
|
}
|
|
29
29
|
return lines.join('\n');
|
|
30
30
|
}
|
|
31
|
+
// Base-prompt output-format phrasings that contradict the neal-json transport
|
|
32
|
+
// contract appended by buildStructuredJsonPrompt (which requires a final fenced
|
|
33
|
+
// ```neal-json block, optionally preceded by prose). A structured-JSON base
|
|
34
|
+
// prompt must not carry its own "no fences / no prose outside the JSON object"
|
|
35
|
+
// instruction, because that flatly contradicts the transport for whichever
|
|
36
|
+
// provider renders it. This mirrors the assertNoReadPromptInstructionText /
|
|
37
|
+
// NO_READ_PROMPT_FORBIDDEN_MARKERS precedent in
|
|
38
|
+
// src/neal/context/inline-review-context.ts: a shared marker list plus a
|
|
39
|
+
// render-time guard, so a reintroduced contradiction throws at build time
|
|
40
|
+
// instead of silently shipping. The markers are chosen so they never match the
|
|
41
|
+
// transport wrapper's own lines above (no bare 'fence'/'fenced' marker).
|
|
42
|
+
export const CONFLICTING_OUTPUT_FORMAT_MARKERS = [
|
|
43
|
+
'do not include markdown fences',
|
|
44
|
+
'markdown fences or prose outside',
|
|
45
|
+
'outside the JSON object',
|
|
46
|
+
];
|
|
47
|
+
// Implementation-side guard for builder-owned static output-format instruction
|
|
48
|
+
// text in structured-JSON base prompts. Call it only on Neal-authored
|
|
49
|
+
// instruction lines (never on dynamic content such as JSON.stringify segments
|
|
50
|
+
// or coder-authored summaries, which may legitimately mention these phrases).
|
|
51
|
+
export function assertNoConflictingOutputFormatInstruction(text, label) {
|
|
52
|
+
const lowered = text.toLowerCase();
|
|
53
|
+
const violations = CONFLICTING_OUTPUT_FORMAT_MARKERS.filter((marker) => lowered.includes(marker.toLowerCase()));
|
|
54
|
+
if (violations.length > 0) {
|
|
55
|
+
throw new Error(`${label} produced a structured-JSON base prompt with output-format instructions that conflict with the neal-json transport contract: ${violations.join(', ')}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
// Single guarded-assembly seam for structured-JSON base-prompt output-format
|
|
59
|
+
// lines: asserts the assembled lines carry no transport-conflicting phrasing,
|
|
60
|
+
// then returns a defensive copy. Base-prompt builders funnel their
|
|
61
|
+
// output-format instructions through this so a reintroduced contradiction
|
|
62
|
+
// throws at render, and tests can inject a marker through this one unit.
|
|
63
|
+
export function guardStructuredJsonOutputFormatLines(lines, label) {
|
|
64
|
+
assertNoConflictingOutputFormatInstruction(lines.join('\n'), label);
|
|
65
|
+
return [...lines];
|
|
66
|
+
}
|
|
31
67
|
export function extractStructuredJsonPayload(assistantText) {
|
|
32
68
|
const blocks = findNealJsonBlocks(assistantText);
|
|
33
69
|
if (blocks.length > 1) {
|
|
@@ -4,6 +4,7 @@ import { formatPublicPhase } from './phase-display.js';
|
|
|
4
4
|
import { getRunDisplayStatus } from './run-status.js';
|
|
5
5
|
import { getCurrentScopeLabel } from './scopes.js';
|
|
6
6
|
import { sanitizeSensitiveText } from './sensitive-text.js';
|
|
7
|
+
import { getPlanReviewGuidanceOriginPhase } from './state-views.js';
|
|
7
8
|
import { getRunStatePath } from './state.js';
|
|
8
9
|
const FALLBACK_REASON = 'Neal is waiting for operator guidance before it can continue.';
|
|
9
10
|
const PLAN_REVIEW_REASON = 'Plan review requires operator guidance before Neal can continue.';
|
|
@@ -134,11 +135,11 @@ export function renderBlockedGuidanceSections(guidance) {
|
|
|
134
135
|
return lines;
|
|
135
136
|
}
|
|
136
137
|
function isPlanReviewGuidanceWaiting(state) {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
138
|
+
// Shared discriminator: reviewer_plan plus coder-authored *response* blocks
|
|
139
|
+
// (blockerReason non-null). A dirty-worktree safety block at the same response
|
|
140
|
+
// phase has blockerReason null and is not a guidance wait, so it renders as a
|
|
141
|
+
// normal blocked state rather than a plan-review-guidance panel.
|
|
142
|
+
return getPlanReviewGuidanceOriginPhase(state) !== null;
|
|
142
143
|
}
|
|
143
144
|
function getReasonAndPhase(state, planReviewWaiting) {
|
|
144
145
|
if (state.interactiveBlockedRecovery) {
|
|
@@ -148,9 +149,15 @@ function getReasonAndPhase(state, planReviewWaiting) {
|
|
|
148
149
|
};
|
|
149
150
|
}
|
|
150
151
|
if (planReviewWaiting) {
|
|
152
|
+
// A coder-authored *response* block carries a durable, operator-facing reason;
|
|
153
|
+
// surface it (and its origin phase) so the guidance render shows the coder's
|
|
154
|
+
// concrete question rather than the generic reviewer-plan message. The
|
|
155
|
+
// reviewer-plan message-resume path leaves blockerReason null and keeps the
|
|
156
|
+
// generic reason + `reviewer_plan` source phase unchanged.
|
|
157
|
+
const blockerReason = state.blockerReason?.trim();
|
|
151
158
|
return {
|
|
152
|
-
reason: PLAN_REVIEW_REASON,
|
|
153
|
-
sourcePhase: 'reviewer_plan',
|
|
159
|
+
reason: blockerReason ? state.blockerReason : PLAN_REVIEW_REASON,
|
|
160
|
+
sourcePhase: state.blockedFromPhase ?? 'reviewer_plan',
|
|
154
161
|
};
|
|
155
162
|
}
|
|
156
163
|
return {
|
|
@@ -167,8 +174,8 @@ function buildCandidateText(state, reason) {
|
|
|
167
174
|
state.currentScopeMeaningfulProgressVerdict?.rationale,
|
|
168
175
|
recovery?.blockedReason,
|
|
169
176
|
recovery?.pendingDirective?.operatorGuidance,
|
|
170
|
-
recovery?.
|
|
171
|
-
recovery?.
|
|
177
|
+
recovery?.consultantAdvice?.resolutionDirective,
|
|
178
|
+
recovery?.consultantAdvice?.rationale,
|
|
172
179
|
].filter(Boolean).join(' '));
|
|
173
180
|
}
|
|
174
181
|
function classifyBlockedGuidance(state, reason, candidateText, planReviewWaiting) {
|
package/dist/neal/cli.js
CHANGED
|
@@ -12,7 +12,7 @@ export function buildUsageLines(version) {
|
|
|
12
12
|
' or: neal review [message] (--last <n> | --since <base>)',
|
|
13
13
|
' or: neal squash [plan.md]',
|
|
14
14
|
' or: neal check',
|
|
15
|
-
' or: neal compat [--model <slug>] [--role coder|reviewer|planner|all] [--reference openai-codex|anthropic-claude|
|
|
15
|
+
' or: neal compat [--model <slug>] [--role coder|reviewer|planner|all] [--reference openai-codex|anthropic-claude|openai-compatible:<model>] [--json]',
|
|
16
16
|
' or: neal status [--json] [--run <run-id>]',
|
|
17
17
|
' or: neal status [--json] --all',
|
|
18
18
|
' or: neal version',
|
|
@@ -38,7 +38,7 @@ export function buildUsageLines(version) {
|
|
|
38
38
|
' neal status',
|
|
39
39
|
' neal status --all',
|
|
40
40
|
' neal check',
|
|
41
|
-
' neal compat --model deepseek/deepseek-chat --role all --reference openai-codex --json # --model runs the slug on
|
|
41
|
+
' neal compat --model deepseek/deepseek-chat --role all --reference openai-codex --json # --model runs the slug on openai-compatible; reference roles run on the native adapter',
|
|
42
42
|
' neal setup',
|
|
43
43
|
' neal setup --provider anthropic-claude --all-roles',
|
|
44
44
|
' neal version',
|
|
@@ -345,11 +345,11 @@ export async function runNealCheckCli(options = {}) {
|
|
|
345
345
|
writeLine(stdout, ` ${describeNotificationScript(notifyBin)}`);
|
|
346
346
|
writeLine(stdout, '');
|
|
347
347
|
// Native adapters drive their providers directly; any other writer provider is
|
|
348
|
-
//
|
|
348
|
+
// an openai-compatible model that should be qualified end-to-end with `neal compat`.
|
|
349
349
|
const NATIVE_WRITER_PROVIDERS = new Set(['openai-codex', 'anthropic-claude']);
|
|
350
350
|
if (!NATIVE_WRITER_PROVIDERS.has(agentConfig.coder.provider) ||
|
|
351
351
|
!NATIVE_WRITER_PROVIDERS.has(agentConfig.reviewer.provider)) {
|
|
352
|
-
writeLine(stdout, 'This is
|
|
352
|
+
writeLine(stdout, 'This is an openai-compatible model - run `neal compat` to confirm it can drive the full loop.');
|
|
353
353
|
writeLine(stdout, '');
|
|
354
354
|
}
|
|
355
355
|
const ignoreStatus = await getNealDirGitIgnoreStatus(cwd);
|
|
@@ -2,7 +2,7 @@ import { spawnSync } from 'node:child_process';
|
|
|
2
2
|
import { cpSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { join, relative, resolve } from 'node:path';
|
|
5
|
+
import { join, relative, resolve, sep } from 'node:path';
|
|
6
6
|
import process from 'node:process';
|
|
7
7
|
import { Writable } from 'node:stream';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
@@ -14,17 +14,18 @@ import { validatePlanDocument } from '../plan-validation.js';
|
|
|
14
14
|
import { getReviewFindingsArtifactPaths } from '../review-findings/artifacts.js';
|
|
15
15
|
import { createAgentReviewFindingsProviderAdapter } from '../review-findings/provider.js';
|
|
16
16
|
import { runNealReviewCli } from '../review-findings/run.js';
|
|
17
|
+
import { enableAgentSettingsIsolation } from '../providers/agent-settings-isolation.js';
|
|
17
18
|
import { isNealProviderError } from '../providers/types.js';
|
|
18
19
|
import { getRunDir } from '../storage-paths.js';
|
|
19
20
|
import { verifyConfiguredProviders } from './check.js';
|
|
20
21
|
import { executeRun } from './runtime.js';
|
|
21
22
|
// Native adapters drive their providers' own CLIs/SDKs directly; every other
|
|
22
|
-
// provider id is "
|
|
23
|
+
// provider id is "openai-compatible" and should be qualified via `neal compat`.
|
|
23
24
|
export const NATIVE_PROVIDER_IDS = new Set([
|
|
24
25
|
'openai-codex',
|
|
25
26
|
'anthropic-claude',
|
|
26
27
|
]);
|
|
27
|
-
export function
|
|
28
|
+
export function isOpenAICompatibleProvider(provider) {
|
|
28
29
|
return !NATIVE_PROVIDER_IDS.has(provider);
|
|
29
30
|
}
|
|
30
31
|
// Default model per native reference provider. `neal compat` routes the
|
|
@@ -35,7 +36,7 @@ export const REFERENCE_DEFAULT_MODELS = {
|
|
|
35
36
|
'anthropic-claude': 'claude-opus-4-8',
|
|
36
37
|
};
|
|
37
38
|
// A compat reference is either a native provider id (run at its built-in default model)
|
|
38
|
-
// or `
|
|
39
|
+
// or `openai-compatible:<openrouter-slug>` to use a *validated* OpenRouter model as the
|
|
39
40
|
// known-good partner. The latter is the follow-up the routing plan anticipated once a
|
|
40
41
|
// rock-solid OpenRouter reference was identified: it removes the codex reference and so
|
|
41
42
|
// uncaps parallelism (no shared native-provider capacity pool). Use ONLY a model that
|
|
@@ -183,11 +184,11 @@ export function parseCompatArgs(args) {
|
|
|
183
184
|
const colon = value.indexOf(':');
|
|
184
185
|
if (colon === -1) {
|
|
185
186
|
if (!NATIVE_PROVIDER_IDS.has(value)) {
|
|
186
|
-
throw new Error('neal compat --reference must be a native provider id (openai-codex or anthropic-claude) or
|
|
187
|
+
throw new Error('neal compat --reference must be a native provider id (openai-codex or anthropic-claude) or openai-compatible:<openrouter-model>');
|
|
187
188
|
}
|
|
188
189
|
}
|
|
189
|
-
else if (value.slice(0, colon) !== '
|
|
190
|
-
throw new Error('neal compat --reference provider:model form must be
|
|
190
|
+
else if (value.slice(0, colon) !== 'openai-compatible' || value.slice(colon + 1).trim() === '') {
|
|
191
|
+
throw new Error('neal compat --reference provider:model form must be openai-compatible:<openrouter-model>');
|
|
191
192
|
}
|
|
192
193
|
reference = value;
|
|
193
194
|
index += 2;
|
|
@@ -223,14 +224,14 @@ export function deriveCandidateConfig(base, options) {
|
|
|
223
224
|
};
|
|
224
225
|
const routingActive = options.model !== null || options.reference !== null;
|
|
225
226
|
const reference = resolveReference(options.reference);
|
|
226
|
-
// Tested role: when a candidate slug is given, force the
|
|
227
|
+
// Tested role: when a candidate slug is given, force the openai-compatible
|
|
227
228
|
// provider and drop any configured effort so the slug drives a clean
|
|
228
229
|
// OpenRouter call. Otherwise leave the configured provider/model untouched.
|
|
229
230
|
if (options.model !== null) {
|
|
230
|
-
next[options.testedRole] = { provider: '
|
|
231
|
+
next[options.testedRole] = { provider: 'openai-compatible', model: options.model };
|
|
231
232
|
}
|
|
232
233
|
// Non-tested roles: when either flag is given, route them onto the reference
|
|
233
|
-
// (a native adapter at its default model, or
|
|
234
|
+
// (a native adapter at its default model, or openai-compatible:<slug> for a validated
|
|
234
235
|
// OpenRouter partner). When neither flag is given, leave every role as configured
|
|
235
236
|
// (pure pass-through).
|
|
236
237
|
if (routingActive) {
|
|
@@ -290,7 +291,12 @@ function applyAndCommitDiff(cwd, diffPath, message) {
|
|
|
290
291
|
}
|
|
291
292
|
function copyFixtureProject(compatDir, fixture) {
|
|
292
293
|
const dir = mkdtempSync(join(tmpdir(), `neal-compat-${fixture.id}-`));
|
|
293
|
-
cpSync(join(compatDir, fixture.projectDir), dir, {
|
|
294
|
+
cpSync(join(compatDir, fixture.projectDir), dir, {
|
|
295
|
+
recursive: true,
|
|
296
|
+
// Never hand committed neal run state to a candidate model: those artifacts
|
|
297
|
+
// describe the pre-fix state and contradict an applied good diff.
|
|
298
|
+
filter: (source) => !source.split(sep).includes('.neal'),
|
|
299
|
+
});
|
|
294
300
|
return dir;
|
|
295
301
|
}
|
|
296
302
|
function cleanProcessEnv() {
|
|
@@ -431,8 +437,8 @@ export function classifyWriterFailure(args) {
|
|
|
431
437
|
// status 'failed' with no conclusive structural signal.
|
|
432
438
|
return 'provider_failed';
|
|
433
439
|
}
|
|
434
|
-
function makeCell(role, fixtureId, pass, failureMode, detail, diffKind = null) {
|
|
435
|
-
return { role, fixtureId, diffKind, pass, failureMode, detail };
|
|
440
|
+
function makeCell(role, fixtureId, pass, failureMode, detail, diffKind = null, blockingCount = null) {
|
|
441
|
+
return { role, fixtureId, diffKind, blockingCount, pass, failureMode, detail };
|
|
436
442
|
}
|
|
437
443
|
// ---------------------------------------------------------------------------
|
|
438
444
|
// Coder fixture evaluation
|
|
@@ -513,8 +519,21 @@ export async function evaluatePlannerFixture(args) {
|
|
|
513
519
|
// ---------------------------------------------------------------------------
|
|
514
520
|
// Reviewer fixture evaluation
|
|
515
521
|
// ---------------------------------------------------------------------------
|
|
516
|
-
export function
|
|
517
|
-
return result.draft.findings.
|
|
522
|
+
export function collectBlockingFindings(result) {
|
|
523
|
+
return result.draft.findings.filter((finding) => finding.severity === 'blocking');
|
|
524
|
+
}
|
|
525
|
+
export function countBlockingFindings(result) {
|
|
526
|
+
return collectBlockingFindings(result).length;
|
|
527
|
+
}
|
|
528
|
+
function normalizeFindingText(text) {
|
|
529
|
+
const collapsed = text.replace(/\s+/gu, ' ').trim();
|
|
530
|
+
return collapsed.length > 120 ? `${collapsed.slice(0, 120)}…` : collapsed;
|
|
531
|
+
}
|
|
532
|
+
export function summarizeBlockingFindings(findings, limit = 2) {
|
|
533
|
+
return findings
|
|
534
|
+
.slice(0, limit)
|
|
535
|
+
.map((finding) => `${normalizeFindingText(finding.claim)} — ${normalizeFindingText(finding.requiredAction)}`)
|
|
536
|
+
.join('; ');
|
|
518
537
|
}
|
|
519
538
|
export function classifyNonAcceptedReviewOutcome(outcome) {
|
|
520
539
|
switch (outcome) {
|
|
@@ -570,7 +589,7 @@ function buildReviewParsedArgs() {
|
|
|
570
589
|
selector: { kind: 'last', count: 1 },
|
|
571
590
|
};
|
|
572
591
|
}
|
|
573
|
-
async function
|
|
592
|
+
async function runReviewerDiff(args) {
|
|
574
593
|
const { fixture, diffKind } = args;
|
|
575
594
|
if (!fixture.reviewer || !fixture.verifyCommand) {
|
|
576
595
|
throw new Error(`compat: reviewer fixture ${fixture.id} is missing diffs/verifyCommand`);
|
|
@@ -604,20 +623,18 @@ async function evaluateReviewerDiff(args) {
|
|
|
604
623
|
// schema failures collapse to `structured_output`.
|
|
605
624
|
const outcome = await readReviewOutcome(throwawayCwd, reviewId);
|
|
606
625
|
const failureMode = classifyReviewerThrownFailure(error, outcome);
|
|
607
|
-
return
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
if (!blocking) {
|
|
612
|
-
return makeCell('reviewer', fixture.id, true, null, null, 'good');
|
|
613
|
-
}
|
|
614
|
-
return makeCell('reviewer', fixture.id, false, 'wrong_or_empty_output', 'reviewer flagged the known-good diff with a blocking finding', 'good');
|
|
626
|
+
return {
|
|
627
|
+
status: 'error',
|
|
628
|
+
cell: makeCell('reviewer', fixture.id, false, failureMode, error instanceof Error ? error.message : String(error), diffKind, null),
|
|
629
|
+
};
|
|
615
630
|
}
|
|
616
|
-
//
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
631
|
+
// No pass/fail judgement here: the verdict is a property of the pair.
|
|
632
|
+
const blocking = collectBlockingFindings(result);
|
|
633
|
+
return {
|
|
634
|
+
status: 'scored',
|
|
635
|
+
blockingCount: blocking.length,
|
|
636
|
+
blockingSummary: summarizeBlockingFindings(blocking),
|
|
637
|
+
};
|
|
621
638
|
}
|
|
622
639
|
finally {
|
|
623
640
|
rmSync(throwawayCwd, { recursive: true, force: true });
|
|
@@ -641,9 +658,51 @@ async function readReviewOutcome(cwd, reviewId) {
|
|
|
641
658
|
}
|
|
642
659
|
}
|
|
643
660
|
export async function evaluateReviewerFixture(args) {
|
|
644
|
-
const good = await
|
|
645
|
-
const broken = await
|
|
646
|
-
|
|
661
|
+
const good = await runReviewerDiff({ ...args, diffKind: 'good' });
|
|
662
|
+
const broken = await runReviewerDiff({ ...args, diffKind: 'broken' });
|
|
663
|
+
// Either diff erroring makes the pair unscoreable: no discrimination verdict
|
|
664
|
+
// is computable, so attribute the systematic cause to both cells.
|
|
665
|
+
if (good.status === 'error' && broken.status === 'error') {
|
|
666
|
+
return { good: good.cell, broken: broken.cell };
|
|
667
|
+
}
|
|
668
|
+
if (good.status === 'error') {
|
|
669
|
+
return {
|
|
670
|
+
good: good.cell,
|
|
671
|
+
broken: broken.status === 'scored'
|
|
672
|
+
? unscoreableCell(args.fixture.id, 'broken', broken.blockingCount, good.cell)
|
|
673
|
+
: broken.cell,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
if (broken.status === 'error') {
|
|
677
|
+
return {
|
|
678
|
+
good: unscoreableCell(args.fixture.id, 'good', good.blockingCount, broken.cell),
|
|
679
|
+
broken: broken.cell,
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
// Discrimination criterion: severity calibration is not graded. A reviewer may
|
|
683
|
+
// raise blocking findings on the good diff and still PASS, as long as the
|
|
684
|
+
// broken diff draws strictly more.
|
|
685
|
+
const pass = broken.blockingCount >= 1 && good.blockingCount < broken.blockingCount;
|
|
686
|
+
if (pass) {
|
|
687
|
+
return {
|
|
688
|
+
good: makeCell('reviewer', args.fixture.id, true, null, null, 'good', good.blockingCount),
|
|
689
|
+
broken: makeCell('reviewer', args.fixture.id, true, null, null, 'broken', broken.blockingCount),
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
const base = broken.blockingCount === 0
|
|
693
|
+
? `reviewer raised no blocking finding on the broken diff (blocking good=${good.blockingCount}, broken=0)`
|
|
694
|
+
: `reviewer did not discriminate: blocking good=${good.blockingCount} >= broken=${broken.blockingCount}`;
|
|
695
|
+
const detailFor = (summary) => summary === '' ? base : `${base} | blocking: ${summary}`;
|
|
696
|
+
return {
|
|
697
|
+
good: makeCell('reviewer', args.fixture.id, false, 'wrong_or_empty_output', detailFor(good.blockingSummary), 'good', good.blockingCount),
|
|
698
|
+
broken: makeCell('reviewer', args.fixture.id, false, 'wrong_or_empty_output', detailFor(broken.blockingSummary), 'broken', broken.blockingCount),
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
// A scored diff whose partner errored: carry the partner's failure mode so
|
|
702
|
+
// smoke aggregation attributes the systematic cause rather than a bogus verdict.
|
|
703
|
+
function unscoreableCell(fixtureId, diffKind, blockingCount, partnerCell) {
|
|
704
|
+
const partnerDiffKind = diffKind === 'good' ? 'broken' : 'good';
|
|
705
|
+
return makeCell('reviewer', fixtureId, false, partnerCell.failureMode, `pair unscoreable: the ${partnerDiffKind} diff review failed (${partnerCell.failureMode ?? 'unknown'})`, diffKind, blockingCount);
|
|
647
706
|
}
|
|
648
707
|
// ---------------------------------------------------------------------------
|
|
649
708
|
// Output formatting
|
|
@@ -704,6 +763,11 @@ export function formatCompatTable(report) {
|
|
|
704
763
|
}
|
|
705
764
|
export async function runCompat(args) {
|
|
706
765
|
const { cwd, parsed } = args;
|
|
766
|
+
// compat is a hermetic capability probe that runs the whole role matrix
|
|
767
|
+
// through the native SDKs. Isolate those adapters from the operator's
|
|
768
|
+
// interactive config so the probe stays quiet (no per-turn notifier hooks)
|
|
769
|
+
// and repeatable. Normal neal runs never call this, so they honor the config.
|
|
770
|
+
enableAgentSettingsIsolation();
|
|
707
771
|
const deps = args.deps ?? {};
|
|
708
772
|
const compatDir = deps.compatDir ?? getCompatExamplesDir();
|
|
709
773
|
const manifest = deps.manifest ?? loadCompatManifest(compatDir);
|
|
@@ -778,14 +842,14 @@ export async function runCompat(args) {
|
|
|
778
842
|
const roles = buildRoleRollups(cells, targetedRoles);
|
|
779
843
|
const overallPass = roles.length > 0 && roles.every((rollup) => rollup.pass);
|
|
780
844
|
return {
|
|
781
|
-
schemaVersion:
|
|
845
|
+
schemaVersion: 2,
|
|
782
846
|
model: parsed.model,
|
|
783
847
|
reference: parsed.reference,
|
|
784
848
|
role: parsed.role,
|
|
785
849
|
candidateProviders: {
|
|
786
|
-
coder: parsed.model !== null ? '
|
|
787
|
-
reviewer: parsed.model !== null ? '
|
|
788
|
-
planner: parsed.model !== null ? '
|
|
850
|
+
coder: parsed.model !== null ? 'openai-compatible' : baseConfig.coder.provider,
|
|
851
|
+
reviewer: parsed.model !== null ? 'openai-compatible' : baseConfig.reviewer.provider,
|
|
852
|
+
planner: parsed.model !== null ? 'openai-compatible' : baseConfig.planner.provider,
|
|
789
853
|
},
|
|
790
854
|
cells,
|
|
791
855
|
roles,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Writable } from 'node:stream';
|
|
2
2
|
import { ActivityFooter } from '../activity-footer.js';
|
|
3
3
|
import { clearDiagnosticFooter, configureDiagnosticFooter, resetDiagnosticDetailState, setDiagnosticDetailContext, writeNarrative, } from '../diagnostic.js';
|
|
4
|
-
import { createInteractiveKeyController } from '../interactive-controls.js';
|
|
4
|
+
import { createInteractiveKeyController, renderInteractiveKeyHint } from '../interactive-controls.js';
|
|
5
5
|
export async function withInteractiveActivity(options, action) {
|
|
6
6
|
let detailContext = {};
|
|
7
7
|
resetDiagnosticDetailState(detailContext);
|
|
@@ -34,7 +34,7 @@ export async function withInteractiveActivity(options, action) {
|
|
|
34
34
|
};
|
|
35
35
|
try {
|
|
36
36
|
if (process.stdin.isTTY) {
|
|
37
|
-
writeNarrative(
|
|
37
|
+
writeNarrative(renderInteractiveKeyHint(false));
|
|
38
38
|
}
|
|
39
39
|
return await action({
|
|
40
40
|
stderr: createNarrativeWritable(),
|
|
@@ -6,6 +6,7 @@ import { assertAgentConfigSupportsResume } from '../providers/registry.js';
|
|
|
6
6
|
import { formatPublicPhase } from '../phase-display.js';
|
|
7
7
|
import { logUserGuidanceApplied } from '../prompts/guidance.js';
|
|
8
8
|
import { decideResumeAction } from '../resume-decision.js';
|
|
9
|
+
import { getPlanReviewGuidanceOriginPhase } from '../state-views.js';
|
|
9
10
|
import { loadState, saveState } from '../state.js';
|
|
10
11
|
function getSelectedRunId(state, selectedRunId) {
|
|
11
12
|
return selectedRunId && selectedRunId !== 'latest' ? selectedRunId : basename(state.runDir);
|
|
@@ -67,11 +68,10 @@ function formatGuidanceDecisionRejection(decision) {
|
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
function isEligiblePlanReviewGuidanceState(state) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
state.pendingPlanReviewGuidance === null);
|
|
71
|
+
// Shared discriminator: reviewer_plan plus coder-authored *response* blocks
|
|
72
|
+
// (blockerReason non-null). A dirty-worktree safety block lands at the same
|
|
73
|
+
// response phase with blockerReason null and is not answerable via --message.
|
|
74
|
+
return getPlanReviewGuidanceOriginPhase(state) !== null;
|
|
75
75
|
}
|
|
76
76
|
async function recordPlanReviewGuidance(args) {
|
|
77
77
|
const resumeCommand = getResumeCommand(args.state, args.target);
|
|
@@ -97,6 +97,16 @@ async function recordPlanReviewGuidance(args) {
|
|
|
97
97
|
if (!isEligiblePlanReviewGuidanceState(args.state)) {
|
|
98
98
|
throw new Error(describePlanReviewGuidanceStateProblem(args.state, args.target));
|
|
99
99
|
}
|
|
100
|
+
const originPhase = getPlanReviewGuidanceOriginPhase(args.state);
|
|
101
|
+
if (originPhase === null) {
|
|
102
|
+
throw new Error(describePlanReviewGuidanceStateProblem(args.state, args.target));
|
|
103
|
+
}
|
|
104
|
+
// Map the origin block to the phase that will actually consume the guidance:
|
|
105
|
+
// a reviewer_plan block delivers to coder_plan_response (unchanged behavior),
|
|
106
|
+
// while a coder-authored response block returns to its own origin phase so the
|
|
107
|
+
// resumed round selects the right open findings and delivers the guidance
|
|
108
|
+
// instead of accepting on an empty blocking set.
|
|
109
|
+
const resumePhase = originPhase === 'reviewer_plan' ? 'coder_plan_response' : originPhase;
|
|
100
110
|
assertAgentConfigSupportsResume(args.state.agentConfig, args.state, {
|
|
101
111
|
context: 'record plan-review guidance for resume',
|
|
102
112
|
});
|
|
@@ -110,19 +120,21 @@ async function recordPlanReviewGuidance(args) {
|
|
|
110
120
|
await logUserGuidanceApplied(logger);
|
|
111
121
|
const nextState = await saveState(args.target.statePath, {
|
|
112
122
|
...args.state,
|
|
113
|
-
phase:
|
|
123
|
+
phase: resumePhase,
|
|
114
124
|
status: 'running',
|
|
115
125
|
blockedFromPhase: null,
|
|
126
|
+
// Returning to running clears the durable coder-authored blocker reason.
|
|
127
|
+
blockerReason: null,
|
|
116
128
|
pendingPlanReviewGuidance: {
|
|
117
129
|
message: trimmedGuidance,
|
|
118
|
-
sourcePhase:
|
|
130
|
+
sourcePhase: originPhase,
|
|
119
131
|
recordedAt: new Date().toISOString(),
|
|
120
132
|
},
|
|
121
133
|
});
|
|
122
134
|
await writeExecutionArtifacts(nextState);
|
|
123
135
|
await logger.event('plan_review_guidance.recorded', {
|
|
124
136
|
statePath: args.target.statePath,
|
|
125
|
-
sourcePhase:
|
|
137
|
+
sourcePhase: originPhase,
|
|
126
138
|
guidanceBytes: Buffer.byteLength(trimmedGuidance, 'utf8'),
|
|
127
139
|
});
|
|
128
140
|
return {
|
|
@@ -3,7 +3,7 @@ import { basename, dirname, join } from 'node:path';
|
|
|
3
3
|
import process from 'node:process';
|
|
4
4
|
import { clearDiagnosticFooter, configureDiagnosticFooter, resetDiagnosticDetailState, setDiagnosticDetailContext, writeDiagnostic, } from '../diagnostic.js';
|
|
5
5
|
import { buildBlockedGuidance, renderBlockedGuidanceSections, } from '../blocked-guidance.js';
|
|
6
|
-
import { createInteractiveKeyController } from '../interactive-controls.js';
|
|
6
|
+
import { createInteractiveKeyController, renderInteractiveKeyHint } from '../interactive-controls.js';
|
|
7
7
|
import { createRunId } from '../logger.js';
|
|
8
8
|
import { runOnePass } from '../orchestrator.js';
|
|
9
9
|
import { writeExecutionArtifacts } from '../orchestrator/artifacts.js';
|
|
@@ -399,14 +399,16 @@ export async function executeRun(state, statePath, logger, options = {}) {
|
|
|
399
399
|
await footer.setState(state, displayedPhaseStartedAt);
|
|
400
400
|
const narrator = createTerminalNarrator({ logger });
|
|
401
401
|
await narrator.start(state);
|
|
402
|
+
const allowStopRequest = state.topLevelMode === 'execute';
|
|
402
403
|
const stopController = createInteractiveKeyController({
|
|
404
|
+
allowStopRequest,
|
|
403
405
|
getDetailFilter() {
|
|
404
406
|
return currentDetailContext;
|
|
405
407
|
},
|
|
406
408
|
});
|
|
407
409
|
try {
|
|
408
410
|
if (process.stdin.isTTY) {
|
|
409
|
-
writeDiagnostic(
|
|
411
|
+
writeDiagnostic(renderInteractiveKeyHint(allowStopRequest));
|
|
410
412
|
}
|
|
411
413
|
let finalState;
|
|
412
414
|
try {
|
package/dist/neal/config.js
CHANGED
|
@@ -13,13 +13,14 @@ const DEFAULT_CONFIG = {
|
|
|
13
13
|
phase_heartbeat_ms: 60_000,
|
|
14
14
|
max_review_rounds: 20,
|
|
15
15
|
review_stuck_window: 5,
|
|
16
|
+
plan_review_debt_round_threshold: 3,
|
|
16
17
|
inactivity_timeout_ms: 600_000,
|
|
17
18
|
api_retry_limit: 10,
|
|
18
19
|
agent_turn_startup_timeout_ms: 300_000,
|
|
19
20
|
agent_turn_retry_limit: 1,
|
|
20
21
|
interactive_blocked_recovery_max_turns: 3,
|
|
21
22
|
final_completion_continue_execution_max: 2,
|
|
22
|
-
|
|
23
|
+
consultant_max_attempts: 1,
|
|
23
24
|
notify_bin: null,
|
|
24
25
|
},
|
|
25
26
|
agent: {
|
|
@@ -51,7 +52,7 @@ function buildWriterProviderSetupLines(guidance, missingProviderKeys) {
|
|
|
51
52
|
}
|
|
52
53
|
return [
|
|
53
54
|
'Neal is not set up yet.',
|
|
54
|
-
`Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in
|
|
55
|
+
`Run \`neal setup\` to choose providers, or set ${formatInlineList(missingKeys)} in neal.yml.`,
|
|
55
56
|
];
|
|
56
57
|
}
|
|
57
58
|
function formatInlineList(values) {
|
|
@@ -173,7 +174,7 @@ function mergeConfig(base, override) {
|
|
|
173
174
|
};
|
|
174
175
|
}
|
|
175
176
|
export function getConfigSourceInfo(cwd = process.cwd()) {
|
|
176
|
-
const repoPath = resolve(cwd, '
|
|
177
|
+
const repoPath = resolve(cwd, 'neal.yml');
|
|
177
178
|
const userPath = join(homedir(), '.neal', 'config.yml');
|
|
178
179
|
return {
|
|
179
180
|
repo: {
|
|
@@ -293,11 +294,19 @@ export function getReviewStuckWindow(cwd = process.cwd()) {
|
|
|
293
294
|
return (parseNumberValue(config.neal?.review_stuck_window) ??
|
|
294
295
|
DEFAULT_CONFIG.neal.review_stuck_window);
|
|
295
296
|
}
|
|
296
|
-
|
|
297
|
+
// The reviewer round at or past which a novel verification-hardening plan-review
|
|
298
|
+
// finding converts to recorded plan-review debt instead of forcing another
|
|
299
|
+
// revision round. Read at the decision site by cwd, mirroring getMaxReviewRounds.
|
|
300
|
+
export function getPlanReviewDebtRoundThreshold(cwd = process.cwd()) {
|
|
297
301
|
const config = loadConfigFile(cwd);
|
|
298
|
-
|
|
299
|
-
DEFAULT_CONFIG.neal.
|
|
300
|
-
|
|
302
|
+
return (parseNumberValue(config.neal?.plan_review_debt_round_threshold) ??
|
|
303
|
+
DEFAULT_CONFIG.neal.plan_review_debt_round_threshold);
|
|
304
|
+
}
|
|
305
|
+
export function getConsultantMaxAttempts(cwd = process.cwd()) {
|
|
306
|
+
const config = loadConfigFile(cwd);
|
|
307
|
+
const raw = parseNumberValue(config.neal?.consultant_max_attempts) ??
|
|
308
|
+
DEFAULT_CONFIG.neal.consultant_max_attempts;
|
|
309
|
+
// Clamp to a non-negative integer (0 disables the consultant entirely).
|
|
301
310
|
if (!Number.isFinite(raw) || raw <= 0) {
|
|
302
311
|
return 0;
|
|
303
312
|
}
|
|
@@ -346,6 +355,28 @@ function parseOpenAICompatibleHeaders(value) {
|
|
|
346
355
|
}
|
|
347
356
|
return headers;
|
|
348
357
|
}
|
|
358
|
+
function parseOpenAICompatiblePricing(value) {
|
|
359
|
+
if (value === undefined || value === null) {
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
if (typeof value !== 'object' || Array.isArray(value)) {
|
|
363
|
+
throw new Error('Invalid providers.openai_compatible.pricing: expected a map with input_per_million, ' +
|
|
364
|
+
'cached_input_per_million, and output_per_million rates.');
|
|
365
|
+
}
|
|
366
|
+
const record = value;
|
|
367
|
+
const parseRate = (key) => {
|
|
368
|
+
const rate = record[key];
|
|
369
|
+
if (typeof rate !== 'number' || !Number.isFinite(rate) || rate < 0) {
|
|
370
|
+
throw new Error(`Invalid providers.openai_compatible.pricing: "${key}" must be a finite non-negative number.`);
|
|
371
|
+
}
|
|
372
|
+
return rate;
|
|
373
|
+
};
|
|
374
|
+
return {
|
|
375
|
+
inputPerMillion: parseRate('input_per_million'),
|
|
376
|
+
cachedInputPerMillion: parseRate('cached_input_per_million'),
|
|
377
|
+
outputPerMillion: parseRate('output_per_million'),
|
|
378
|
+
};
|
|
379
|
+
}
|
|
349
380
|
export function getOpenAICompatibleSettings(cwd = process.cwd(), env = process.env) {
|
|
350
381
|
const config = loadConfigFile(cwd).providers?.openai_compatible;
|
|
351
382
|
const baseUrl = parseStringValue(config?.base_url) ??
|
|
@@ -357,12 +388,14 @@ export function getOpenAICompatibleSettings(cwd = process.cwd(), env = process.e
|
|
|
357
388
|
parseStringValue(env.OPENAI_COMPATIBLE_MODEL) ??
|
|
358
389
|
null;
|
|
359
390
|
const headers = parseOpenAICompatibleHeaders(config?.headers);
|
|
391
|
+
const pricing = parseOpenAICompatiblePricing(config?.pricing);
|
|
360
392
|
return {
|
|
361
393
|
baseUrl,
|
|
362
394
|
apiKeyEnv,
|
|
363
395
|
apiKey,
|
|
364
396
|
defaultModel,
|
|
365
397
|
headers,
|
|
398
|
+
pricing,
|
|
366
399
|
};
|
|
367
400
|
}
|
|
368
401
|
export function getDefaultCoderProvider(cwd = process.cwd()) {
|