@navels/neal 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -209
- package/SECURITY.md +16 -19
- package/dist/neal/adjudicator/{blocked-adjudicator.js → consultant.js} +21 -21
- package/dist/neal/adjudicator/execute.js +33 -33
- package/dist/neal/adjudicator/final-completion.js +2 -31
- package/dist/neal/adjudicator/planning.js +230 -38
- package/dist/neal/agents/prompts.js +6 -17
- package/dist/neal/agents/rounds.js +43 -73
- package/dist/neal/agents/schemas.js +47 -13
- package/dist/neal/blocked-guidance.js +16 -9
- package/dist/neal/cli.js +2 -2
- package/dist/neal/commands/check.js +2 -2
- package/dist/neal/commands/compat.js +100 -36
- package/dist/neal/commands/interactive-activity.js +2 -2
- package/dist/neal/commands/recovery-guidance.js +20 -8
- package/dist/neal/commands/runtime.js +4 -2
- package/dist/neal/config.js +16 -7
- package/dist/neal/context/inline-review-context.js +13 -28
- package/dist/neal/context/reviewer-context.js +19 -14
- package/dist/neal/eval/reviewer-eval-manifest.js +86 -0
- package/dist/neal/eval/reviewer-eval.js +155 -0
- package/dist/neal/interactive-controls.js +5 -0
- package/dist/neal/orchestrator/completion.js +49 -2
- package/dist/neal/orchestrator/failures.js +7 -0
- package/dist/neal/orchestrator/notifications.js +9 -9
- package/dist/neal/orchestrator/phases/planning.js +117 -80
- package/dist/neal/orchestrator/phases/recovery.js +151 -115
- package/dist/neal/orchestrator/phases/review.js +28 -1
- package/dist/neal/orchestrator/transitions.js +7 -1
- package/dist/neal/orchestrator.js +3 -0
- package/dist/neal/plan-queue.js +29 -1
- package/dist/neal/prompts/execute.js +7 -19
- package/dist/neal/prompts/planning.js +64 -55
- package/dist/neal/prompts/review-doctrine.js +13 -50
- package/dist/neal/prompts/specialized.js +7 -17
- package/dist/neal/prompts/specs.js +180 -32
- package/dist/neal/providers/agent-settings-isolation.js +35 -0
- package/dist/neal/providers/anthropic-claude.js +18 -1
- package/dist/neal/providers/detection.js +3 -7
- package/dist/neal/providers/git-config-isolation.js +83 -0
- package/dist/neal/providers/openai-codex.js +25 -3
- package/dist/neal/providers/{generic-agentic-tools.js → openai-compatible-tools.js} +11 -5
- package/dist/neal/providers/openai-compatible.js +1078 -430
- package/dist/neal/providers/openrouter-routing.js +68 -0
- package/dist/neal/providers/pricing.js +1 -1
- package/dist/neal/providers/rate-card.js +6 -32
- package/dist/neal/providers/registry.js +6 -17
- package/dist/neal/providers/types.js +12 -0
- package/dist/neal/resume-decision.js +7 -6
- package/dist/neal/resume-planner.js +9 -0
- package/dist/neal/retrospective.js +3 -3
- package/dist/neal/review-debt.js +30 -0
- package/dist/neal/review-findings/provider.js +17 -29
- package/dist/neal/review.js +17 -1
- package/dist/neal/state-invariants.js +11 -3
- package/dist/neal/state-views.js +27 -5
- package/dist/neal/state.js +59 -15
- package/dist/neal/status.js +28 -1
- package/dist/neal/support.js +3 -3
- package/docs/README.md +22 -18
- package/docs/{ADJUDICATOR_INVENTORY.md → adjudicator-inventory.md} +58 -57
- package/docs/architecture.md +79 -83
- package/docs/automation.md +11 -11
- package/docs/compat.md +91 -61
- package/docs/compatible-models.md +145 -121
- package/docs/demo.md +15 -5
- package/docs/maintenance.md +65 -34
- package/docs/plan-format.md +61 -33
- package/docs/prompt-evals.md +96 -0
- package/docs/{PROMPT_SPECS.md → prompt-specs.md} +52 -48
- package/docs/providers.md +279 -349
- package/docs/release.md +63 -81
- package/docs/review-convergence.md +246 -0
- package/docs/state-machine.md +61 -59
- package/docs/storage.md +40 -31
- package/docs/troubleshooting.md +32 -29
- package/examples/compat/add-edit-verify/PLAN.md +2 -0
- package/examples/compat/add-edit-verify/broken.diff +2 -2
- package/examples/compat/add-edit-verify/good.diff +2 -2
- package/examples/compat/add-edit-verify/src/add.js +1 -1
- package/examples/compat/manifest.json +25 -29
- package/examples/compat/sum-grep-edit/PLAN.md +33 -0
- package/examples/compat/sum-grep-edit/broken.diff +12 -0
- package/examples/compat/sum-grep-edit/good.diff +12 -0
- package/examples/compat/sum-grep-edit/package.json +5 -0
- package/examples/compat/sum-grep-edit/src/numbers.js +9 -0
- package/examples/compat/sum-grep-edit/test/numbers.test.js +15 -0
- package/examples/issue-triage-js/README.md +0 -15
- package/examples/reviewer-eval/clean-doc/base/src/clamp.ts +3 -0
- package/examples/reviewer-eval/clean-doc/change.diff +9 -0
- package/examples/reviewer-eval/clean-extract/base/src/greet.ts +3 -0
- package/examples/reviewer-eval/clean-extract/change.diff +13 -0
- package/examples/reviewer-eval/dropped-error/base/src/load.ts +4 -0
- package/examples/reviewer-eval/dropped-error/change.diff +15 -0
- package/examples/reviewer-eval/manifest.json +45 -0
- package/examples/reviewer-eval/off-by-one/base/src/paginate.ts +5 -0
- package/examples/reviewer-eval/off-by-one/change.diff +11 -0
- package/{config.yml → neal.yml} +1 -1
- package/package.json +15 -14
- package/dist/neal/providers/generic-agentic.js +0 -1256
- package/docs/comparison.md +0 -105
- package/docs/issue-pipeline.md +0 -124
- package/examples/compat/is-even-add-test/PLAN.md +0 -30
- package/examples/compat/is-even-add-test/broken.diff +0 -11
- package/examples/compat/is-even-add-test/good.diff +0 -11
- package/examples/compat/is-even-add-test/package.json +0 -5
- package/examples/compat/is-even-add-test/src/is-even.js +0 -3
- package/examples/compat/is-even-add-test/test/is-even.test.js +0 -9
- package/examples/compat/reverse-grep-edit/PLAN.md +0 -32
- package/examples/compat/reverse-grep-edit/broken.diff +0 -12
- package/examples/compat/reverse-grep-edit/good.diff +0 -12
- package/examples/compat/reverse-grep-edit/package.json +0 -5
- package/examples/compat/reverse-grep-edit/src/strings.js +0 -10
- package/examples/compat/reverse-grep-edit/test/strings.test.js +0 -15
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { normalizeExecutionShapeDeclaration, validatePlanDocument } from '../plan-validation.js';
|
|
2
3
|
import { repairReviewerSquashMessageDraft, validateReviewerSquashMessageDraft } from '../squash-message.js';
|
|
3
4
|
export const EXECUTE_SCOPE_PROGRESS_PAYLOAD_START = 'NEAL_PROGRESS_JUSTIFICATION_JSON_START';
|
|
4
5
|
export const EXECUTE_SCOPE_PROGRESS_PAYLOAD_END = 'NEAL_PROGRESS_JUSTIFICATION_JSON_END';
|
|
5
6
|
// --- zod payload definitions (single source of truth) -----------------------
|
|
6
7
|
const REVIEWER_FINDING_SEVERITIES = ['blocking', 'non_blocking'];
|
|
8
|
+
export const PLAN_REVIEWER_FINDING_CLASSES = [
|
|
9
|
+
'plan_correctness',
|
|
10
|
+
'verification_hardening',
|
|
11
|
+
];
|
|
7
12
|
const REVIEWER_MEANINGFUL_PROGRESS_ACTIONS = [
|
|
8
13
|
'accept',
|
|
9
14
|
'block_for_operator',
|
|
@@ -11,7 +16,7 @@ const REVIEWER_MEANINGFUL_PROGRESS_ACTIONS = [
|
|
|
11
16
|
'advance_parent',
|
|
12
17
|
];
|
|
13
18
|
const EXECUTION_SHAPES = ['one_shot', 'multi_scope', 'multi_scope_unknown'];
|
|
14
|
-
const
|
|
19
|
+
const CONSULTANT_TRIAGE_CATEGORIES = [
|
|
15
20
|
'misunderstanding',
|
|
16
21
|
'authorization',
|
|
17
22
|
'external_precondition',
|
|
@@ -56,15 +61,24 @@ const planReviewerFindingSchema = z.object({
|
|
|
56
61
|
files: z.array(z.string()),
|
|
57
62
|
claim: z.string(),
|
|
58
63
|
requiredAction: z.string(),
|
|
64
|
+
// Optional at the payload boundary: an absent class normalizes to the fail-safe
|
|
65
|
+
// plan_correctness downstream (see validatePlanReviewerPayload), while a
|
|
66
|
+
// present-but-invalid value is rejected by the sequential validator before any
|
|
67
|
+
// normalization runs. The reviewer round is a `neal-json-block-v1` structured
|
|
68
|
+
// round (the schema is advisory prompt context, not native strict output), so —
|
|
69
|
+
// like the consultant's optional targetCanonicalIds — this property is
|
|
70
|
+
// intentionally absent from the emitted `required` tuple and buildPlanReviewerSchema
|
|
71
|
+
// is excluded from the strict all-required contract.
|
|
72
|
+
findingClass: z.enum(PLAN_REVIEWER_FINDING_CLASSES).optional(),
|
|
59
73
|
});
|
|
60
74
|
const planReviewerPayloadSchema = z.object({
|
|
61
75
|
summary: z.string(),
|
|
62
76
|
executionShape: z.enum(EXECUTION_SHAPES),
|
|
63
77
|
findings: z.array(planReviewerFindingSchema),
|
|
64
78
|
});
|
|
65
|
-
const
|
|
79
|
+
const consultantPayloadSchema = z.object({
|
|
66
80
|
recoverable: z.boolean(),
|
|
67
|
-
triageCategory: z.enum(
|
|
81
|
+
triageCategory: z.enum(CONSULTANT_TRIAGE_CATEGORIES),
|
|
68
82
|
resolutionDirective: z.string(),
|
|
69
83
|
targetCanonicalIds: z.array(z.string()).optional(),
|
|
70
84
|
rationale: z.string(),
|
|
@@ -257,8 +271,8 @@ export function buildReviewerSchema() {
|
|
|
257
271
|
export function buildPlanReviewerSchema() {
|
|
258
272
|
return emitJsonSchema(planReviewerPayloadSchema);
|
|
259
273
|
}
|
|
260
|
-
export function
|
|
261
|
-
return emitJsonSchema(
|
|
274
|
+
export function buildConsultantSchema() {
|
|
275
|
+
return emitJsonSchema(consultantPayloadSchema);
|
|
262
276
|
}
|
|
263
277
|
export function buildCoderResponseSchema() {
|
|
264
278
|
return emitJsonSchema(coderResponsePayloadSchema);
|
|
@@ -401,7 +415,7 @@ function walkObjectSequential(schema, record, basePath) {
|
|
|
401
415
|
// Historical optional/nullable field patterns performed one property
|
|
402
416
|
// read for the undefined/null test and a FRESH read for validation when
|
|
403
417
|
// the field was present, so stateful accessors observe both reads (the
|
|
404
|
-
//
|
|
418
|
+
// consultant targetCanonicalIds ternary and the coder-scope
|
|
405
419
|
// manualGate ternary behaved this way).
|
|
406
420
|
if (fieldSchema instanceof z.ZodOptional) {
|
|
407
421
|
if (record[key] === undefined) {
|
|
@@ -495,15 +509,15 @@ export function validateReviewerPayload(payload) {
|
|
|
495
509
|
};
|
|
496
510
|
}
|
|
497
511
|
// --- permissive validators ----------------------------------------------------
|
|
498
|
-
// Validates a read-only review_stuck
|
|
512
|
+
// Validates a read-only review_stuck consultant verdict. Exactly one triage
|
|
499
513
|
// category is autonomously recoverable: `misunderstanding` requires
|
|
500
514
|
// recoverable=true plus a non-empty resolutionDirective; the three genuine-wall
|
|
501
515
|
// categories (`authorization`, `external_precondition`, `impossible_task`)
|
|
502
516
|
// require recoverable=false. Anything else (recoverable=true with another
|
|
503
517
|
// triage, or recoverable=false paired with `misunderstanding`) is rejected so a
|
|
504
518
|
// malformed verdict can never drive an autonomous recovery.
|
|
505
|
-
export function
|
|
506
|
-
const parsed = parsePayload(
|
|
519
|
+
export function validateConsultantVerdictPayload(rawPayload) {
|
|
520
|
+
const parsed = parsePayload(consultantPayloadSchema, rawPayload, 'Consultant payload');
|
|
507
521
|
const payload = {
|
|
508
522
|
recoverable: parsed.recoverable,
|
|
509
523
|
triageCategory: parsed.triageCategory,
|
|
@@ -514,18 +528,18 @@ export function validateBlockedAdjudicatorVerdictPayload(rawPayload) {
|
|
|
514
528
|
rationale: parsed.rationale,
|
|
515
529
|
};
|
|
516
530
|
if (!payload.rationale.trim()) {
|
|
517
|
-
throw new Error('
|
|
531
|
+
throw new Error('Consultant returned an empty rationale.');
|
|
518
532
|
}
|
|
519
533
|
if (payload.recoverable) {
|
|
520
534
|
if (payload.triageCategory !== 'misunderstanding') {
|
|
521
|
-
throw new Error('
|
|
535
|
+
throw new Error('Consultant returned recoverable=true with a triageCategory other than misunderstanding.');
|
|
522
536
|
}
|
|
523
537
|
if (!payload.resolutionDirective.trim()) {
|
|
524
|
-
throw new Error('
|
|
538
|
+
throw new Error('Consultant returned recoverable=true without a non-empty resolutionDirective.');
|
|
525
539
|
}
|
|
526
540
|
}
|
|
527
541
|
else if (payload.triageCategory === 'misunderstanding') {
|
|
528
|
-
throw new Error('
|
|
542
|
+
throw new Error('Consultant returned recoverable=false paired with triageCategory=misunderstanding.');
|
|
529
543
|
}
|
|
530
544
|
return payload;
|
|
531
545
|
}
|
|
@@ -542,6 +556,10 @@ export function validatePlanReviewerPayload(payload) {
|
|
|
542
556
|
// always forces an empty string.
|
|
543
557
|
evidence: '',
|
|
544
558
|
requiredAction: finding.requiredAction,
|
|
559
|
+
// Default only the absent case to plan_correctness (the fail-safe class).
|
|
560
|
+
// A present-but-invalid class was already rejected upstream in parsePayload,
|
|
561
|
+
// so this never silently downgrades a bad value.
|
|
562
|
+
findingClass: finding.findingClass ?? 'plan_correctness',
|
|
545
563
|
})),
|
|
546
564
|
};
|
|
547
565
|
}
|
|
@@ -639,6 +657,22 @@ export function validateCoderPlanPayload(rawPayload) {
|
|
|
639
657
|
if (payload.action === 'blocked' && !blockedReason) {
|
|
640
658
|
throw new Error('Planner plan round returned action=blocked without a blockedReason payload.');
|
|
641
659
|
}
|
|
660
|
+
// The refined plan is persisted over the plan document and sent straight to
|
|
661
|
+
// plan review, so it must satisfy the plan contract here — inside the
|
|
662
|
+
// structured-output validator — where a failure triggers the repair loop
|
|
663
|
+
// (the planner gets the errors and retries) instead of burning a reviewer
|
|
664
|
+
// round on an invalid document. Split-plan payloads already have this gate
|
|
665
|
+
// (validateSplitPlanPayload); a live run persisted a planner payload whose
|
|
666
|
+
// planBody was a 39-line refinement summary declaring multi_scope with no
|
|
667
|
+
// Execution Queue, and the reviewer round was spent rediscovering that.
|
|
668
|
+
if (payload.action === 'ready_for_review') {
|
|
669
|
+
const normalizedBody = normalizeExecutionShapeDeclaration(planBody, payload.executionShape);
|
|
670
|
+
const validation = validatePlanDocument(normalizedBody);
|
|
671
|
+
if (!validation.ok) {
|
|
672
|
+
throw new Error(`Planner plan round returned a planBody that is not a valid Neal plan document: ${validation.errors.join('; ')}. ` +
|
|
673
|
+
'Return the complete refined plan document, not a summary of the refinement.');
|
|
674
|
+
}
|
|
675
|
+
}
|
|
642
676
|
return {
|
|
643
677
|
...payload,
|
|
644
678
|
message: payload.message.trim(),
|
|
@@ -4,6 +4,7 @@ import { formatPublicPhase } from './phase-display.js';
|
|
|
4
4
|
import { getRunDisplayStatus } from './run-status.js';
|
|
5
5
|
import { getCurrentScopeLabel } from './scopes.js';
|
|
6
6
|
import { sanitizeSensitiveText } from './sensitive-text.js';
|
|
7
|
+
import { getPlanReviewGuidanceOriginPhase } from './state-views.js';
|
|
7
8
|
import { getRunStatePath } from './state.js';
|
|
8
9
|
const FALLBACK_REASON = 'Neal is waiting for operator guidance before it can continue.';
|
|
9
10
|
const PLAN_REVIEW_REASON = 'Plan review requires operator guidance before Neal can continue.';
|
|
@@ -134,11 +135,11 @@ export function renderBlockedGuidanceSections(guidance) {
|
|
|
134
135
|
return lines;
|
|
135
136
|
}
|
|
136
137
|
function isPlanReviewGuidanceWaiting(state) {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
138
|
+
// Shared discriminator: reviewer_plan plus coder-authored *response* blocks
|
|
139
|
+
// (blockerReason non-null). A dirty-worktree safety block at the same response
|
|
140
|
+
// phase has blockerReason null and is not a guidance wait, so it renders as a
|
|
141
|
+
// normal blocked state rather than a plan-review-guidance panel.
|
|
142
|
+
return getPlanReviewGuidanceOriginPhase(state) !== null;
|
|
142
143
|
}
|
|
143
144
|
function getReasonAndPhase(state, planReviewWaiting) {
|
|
144
145
|
if (state.interactiveBlockedRecovery) {
|
|
@@ -148,9 +149,15 @@ function getReasonAndPhase(state, planReviewWaiting) {
|
|
|
148
149
|
};
|
|
149
150
|
}
|
|
150
151
|
if (planReviewWaiting) {
|
|
152
|
+
// A coder-authored *response* block carries a durable, operator-facing reason;
|
|
153
|
+
// surface it (and its origin phase) so the guidance render shows the coder's
|
|
154
|
+
// concrete question rather than the generic reviewer-plan message. The
|
|
155
|
+
// reviewer-plan message-resume path leaves blockerReason null and keeps the
|
|
156
|
+
// generic reason + `reviewer_plan` source phase unchanged.
|
|
157
|
+
const blockerReason = state.blockerReason?.trim();
|
|
151
158
|
return {
|
|
152
|
-
reason: PLAN_REVIEW_REASON,
|
|
153
|
-
sourcePhase: 'reviewer_plan',
|
|
159
|
+
reason: blockerReason ? state.blockerReason : PLAN_REVIEW_REASON,
|
|
160
|
+
sourcePhase: state.blockedFromPhase ?? 'reviewer_plan',
|
|
154
161
|
};
|
|
155
162
|
}
|
|
156
163
|
return {
|
|
@@ -167,8 +174,8 @@ function buildCandidateText(state, reason) {
|
|
|
167
174
|
state.currentScopeMeaningfulProgressVerdict?.rationale,
|
|
168
175
|
recovery?.blockedReason,
|
|
169
176
|
recovery?.pendingDirective?.operatorGuidance,
|
|
170
|
-
recovery?.
|
|
171
|
-
recovery?.
|
|
177
|
+
recovery?.consultantAdvice?.resolutionDirective,
|
|
178
|
+
recovery?.consultantAdvice?.rationale,
|
|
172
179
|
].filter(Boolean).join(' '));
|
|
173
180
|
}
|
|
174
181
|
function classifyBlockedGuidance(state, reason, candidateText, planReviewWaiting) {
|
package/dist/neal/cli.js
CHANGED
|
@@ -12,7 +12,7 @@ export function buildUsageLines(version) {
|
|
|
12
12
|
' or: neal review [message] (--last <n> | --since <base>)',
|
|
13
13
|
' or: neal squash [plan.md]',
|
|
14
14
|
' or: neal check',
|
|
15
|
-
' or: neal compat [--model <slug>] [--role coder|reviewer|planner|all] [--reference openai-codex|anthropic-claude|
|
|
15
|
+
' or: neal compat [--model <slug>] [--role coder|reviewer|planner|all] [--reference openai-codex|anthropic-claude|openai-compatible:<model>] [--json]',
|
|
16
16
|
' or: neal status [--json] [--run <run-id>]',
|
|
17
17
|
' or: neal status [--json] --all',
|
|
18
18
|
' or: neal version',
|
|
@@ -38,7 +38,7 @@ export function buildUsageLines(version) {
|
|
|
38
38
|
' neal status',
|
|
39
39
|
' neal status --all',
|
|
40
40
|
' neal check',
|
|
41
|
-
' neal compat --model deepseek/deepseek-chat --role all --reference openai-codex --json # --model runs the slug on
|
|
41
|
+
' neal compat --model deepseek/deepseek-chat --role all --reference openai-codex --json # --model runs the slug on openai-compatible; reference roles run on the native adapter',
|
|
42
42
|
' neal setup',
|
|
43
43
|
' neal setup --provider anthropic-claude --all-roles',
|
|
44
44
|
' neal version',
|
|
@@ -345,11 +345,11 @@ export async function runNealCheckCli(options = {}) {
|
|
|
345
345
|
writeLine(stdout, ` ${describeNotificationScript(notifyBin)}`);
|
|
346
346
|
writeLine(stdout, '');
|
|
347
347
|
// Native adapters drive their providers directly; any other writer provider is
|
|
348
|
-
//
|
|
348
|
+
// an openai-compatible model that should be qualified end-to-end with `neal compat`.
|
|
349
349
|
const NATIVE_WRITER_PROVIDERS = new Set(['openai-codex', 'anthropic-claude']);
|
|
350
350
|
if (!NATIVE_WRITER_PROVIDERS.has(agentConfig.coder.provider) ||
|
|
351
351
|
!NATIVE_WRITER_PROVIDERS.has(agentConfig.reviewer.provider)) {
|
|
352
|
-
writeLine(stdout, 'This is
|
|
352
|
+
writeLine(stdout, 'This is an openai-compatible model - run `neal compat` to confirm it can drive the full loop.');
|
|
353
353
|
writeLine(stdout, '');
|
|
354
354
|
}
|
|
355
355
|
const ignoreStatus = await getNealDirGitIgnoreStatus(cwd);
|
|
@@ -2,7 +2,7 @@ import { spawnSync } from 'node:child_process';
|
|
|
2
2
|
import { cpSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
3
3
|
import { readFile } from 'node:fs/promises';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
|
-
import { join, relative, resolve } from 'node:path';
|
|
5
|
+
import { join, relative, resolve, sep } from 'node:path';
|
|
6
6
|
import process from 'node:process';
|
|
7
7
|
import { Writable } from 'node:stream';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
@@ -14,17 +14,18 @@ import { validatePlanDocument } from '../plan-validation.js';
|
|
|
14
14
|
import { getReviewFindingsArtifactPaths } from '../review-findings/artifacts.js';
|
|
15
15
|
import { createAgentReviewFindingsProviderAdapter } from '../review-findings/provider.js';
|
|
16
16
|
import { runNealReviewCli } from '../review-findings/run.js';
|
|
17
|
+
import { enableAgentSettingsIsolation } from '../providers/agent-settings-isolation.js';
|
|
17
18
|
import { isNealProviderError } from '../providers/types.js';
|
|
18
19
|
import { getRunDir } from '../storage-paths.js';
|
|
19
20
|
import { verifyConfiguredProviders } from './check.js';
|
|
20
21
|
import { executeRun } from './runtime.js';
|
|
21
22
|
// Native adapters drive their providers' own CLIs/SDKs directly; every other
|
|
22
|
-
// provider id is "
|
|
23
|
+
// provider id is "openai-compatible" and should be qualified via `neal compat`.
|
|
23
24
|
export const NATIVE_PROVIDER_IDS = new Set([
|
|
24
25
|
'openai-codex',
|
|
25
26
|
'anthropic-claude',
|
|
26
27
|
]);
|
|
27
|
-
export function
|
|
28
|
+
export function isOpenAICompatibleProvider(provider) {
|
|
28
29
|
return !NATIVE_PROVIDER_IDS.has(provider);
|
|
29
30
|
}
|
|
30
31
|
// Default model per native reference provider. `neal compat` routes the
|
|
@@ -35,7 +36,7 @@ export const REFERENCE_DEFAULT_MODELS = {
|
|
|
35
36
|
'anthropic-claude': 'claude-opus-4-8',
|
|
36
37
|
};
|
|
37
38
|
// A compat reference is either a native provider id (run at its built-in default model)
|
|
38
|
-
// or `
|
|
39
|
+
// or `openai-compatible:<openrouter-slug>` to use a *validated* OpenRouter model as the
|
|
39
40
|
// known-good partner. The latter is the follow-up the routing plan anticipated once a
|
|
40
41
|
// rock-solid OpenRouter reference was identified: it removes the codex reference and so
|
|
41
42
|
// uncaps parallelism (no shared native-provider capacity pool). Use ONLY a model that
|
|
@@ -183,11 +184,11 @@ export function parseCompatArgs(args) {
|
|
|
183
184
|
const colon = value.indexOf(':');
|
|
184
185
|
if (colon === -1) {
|
|
185
186
|
if (!NATIVE_PROVIDER_IDS.has(value)) {
|
|
186
|
-
throw new Error('neal compat --reference must be a native provider id (openai-codex or anthropic-claude) or
|
|
187
|
+
throw new Error('neal compat --reference must be a native provider id (openai-codex or anthropic-claude) or openai-compatible:<openrouter-model>');
|
|
187
188
|
}
|
|
188
189
|
}
|
|
189
|
-
else if (value.slice(0, colon) !== '
|
|
190
|
-
throw new Error('neal compat --reference provider:model form must be
|
|
190
|
+
else if (value.slice(0, colon) !== 'openai-compatible' || value.slice(colon + 1).trim() === '') {
|
|
191
|
+
throw new Error('neal compat --reference provider:model form must be openai-compatible:<openrouter-model>');
|
|
191
192
|
}
|
|
192
193
|
reference = value;
|
|
193
194
|
index += 2;
|
|
@@ -223,14 +224,14 @@ export function deriveCandidateConfig(base, options) {
|
|
|
223
224
|
};
|
|
224
225
|
const routingActive = options.model !== null || options.reference !== null;
|
|
225
226
|
const reference = resolveReference(options.reference);
|
|
226
|
-
// Tested role: when a candidate slug is given, force the
|
|
227
|
+
// Tested role: when a candidate slug is given, force the openai-compatible
|
|
227
228
|
// provider and drop any configured effort so the slug drives a clean
|
|
228
229
|
// OpenRouter call. Otherwise leave the configured provider/model untouched.
|
|
229
230
|
if (options.model !== null) {
|
|
230
|
-
next[options.testedRole] = { provider: '
|
|
231
|
+
next[options.testedRole] = { provider: 'openai-compatible', model: options.model };
|
|
231
232
|
}
|
|
232
233
|
// Non-tested roles: when either flag is given, route them onto the reference
|
|
233
|
-
// (a native adapter at its default model, or
|
|
234
|
+
// (a native adapter at its default model, or openai-compatible:<slug> for a validated
|
|
234
235
|
// OpenRouter partner). When neither flag is given, leave every role as configured
|
|
235
236
|
// (pure pass-through).
|
|
236
237
|
if (routingActive) {
|
|
@@ -290,7 +291,12 @@ function applyAndCommitDiff(cwd, diffPath, message) {
|
|
|
290
291
|
}
|
|
291
292
|
function copyFixtureProject(compatDir, fixture) {
|
|
292
293
|
const dir = mkdtempSync(join(tmpdir(), `neal-compat-${fixture.id}-`));
|
|
293
|
-
cpSync(join(compatDir, fixture.projectDir), dir, {
|
|
294
|
+
cpSync(join(compatDir, fixture.projectDir), dir, {
|
|
295
|
+
recursive: true,
|
|
296
|
+
// Never hand committed neal run state to a candidate model: those artifacts
|
|
297
|
+
// describe the pre-fix state and contradict an applied good diff.
|
|
298
|
+
filter: (source) => !source.split(sep).includes('.neal'),
|
|
299
|
+
});
|
|
294
300
|
return dir;
|
|
295
301
|
}
|
|
296
302
|
function cleanProcessEnv() {
|
|
@@ -431,8 +437,8 @@ export function classifyWriterFailure(args) {
|
|
|
431
437
|
// status 'failed' with no conclusive structural signal.
|
|
432
438
|
return 'provider_failed';
|
|
433
439
|
}
|
|
434
|
-
function makeCell(role, fixtureId, pass, failureMode, detail, diffKind = null) {
|
|
435
|
-
return { role, fixtureId, diffKind, pass, failureMode, detail };
|
|
440
|
+
function makeCell(role, fixtureId, pass, failureMode, detail, diffKind = null, blockingCount = null) {
|
|
441
|
+
return { role, fixtureId, diffKind, blockingCount, pass, failureMode, detail };
|
|
436
442
|
}
|
|
437
443
|
// ---------------------------------------------------------------------------
|
|
438
444
|
// Coder fixture evaluation
|
|
@@ -513,8 +519,21 @@ export async function evaluatePlannerFixture(args) {
|
|
|
513
519
|
// ---------------------------------------------------------------------------
|
|
514
520
|
// Reviewer fixture evaluation
|
|
515
521
|
// ---------------------------------------------------------------------------
|
|
516
|
-
export function
|
|
517
|
-
return result.draft.findings.
|
|
522
|
+
export function collectBlockingFindings(result) {
|
|
523
|
+
return result.draft.findings.filter((finding) => finding.severity === 'blocking');
|
|
524
|
+
}
|
|
525
|
+
export function countBlockingFindings(result) {
|
|
526
|
+
return collectBlockingFindings(result).length;
|
|
527
|
+
}
|
|
528
|
+
function normalizeFindingText(text) {
|
|
529
|
+
const collapsed = text.replace(/\s+/gu, ' ').trim();
|
|
530
|
+
return collapsed.length > 120 ? `${collapsed.slice(0, 120)}…` : collapsed;
|
|
531
|
+
}
|
|
532
|
+
export function summarizeBlockingFindings(findings, limit = 2) {
|
|
533
|
+
return findings
|
|
534
|
+
.slice(0, limit)
|
|
535
|
+
.map((finding) => `${normalizeFindingText(finding.claim)} — ${normalizeFindingText(finding.requiredAction)}`)
|
|
536
|
+
.join('; ');
|
|
518
537
|
}
|
|
519
538
|
export function classifyNonAcceptedReviewOutcome(outcome) {
|
|
520
539
|
switch (outcome) {
|
|
@@ -570,7 +589,7 @@ function buildReviewParsedArgs() {
|
|
|
570
589
|
selector: { kind: 'last', count: 1 },
|
|
571
590
|
};
|
|
572
591
|
}
|
|
573
|
-
async function
|
|
592
|
+
async function runReviewerDiff(args) {
|
|
574
593
|
const { fixture, diffKind } = args;
|
|
575
594
|
if (!fixture.reviewer || !fixture.verifyCommand) {
|
|
576
595
|
throw new Error(`compat: reviewer fixture ${fixture.id} is missing diffs/verifyCommand`);
|
|
@@ -604,20 +623,18 @@ async function evaluateReviewerDiff(args) {
|
|
|
604
623
|
// schema failures collapse to `structured_output`.
|
|
605
624
|
const outcome = await readReviewOutcome(throwawayCwd, reviewId);
|
|
606
625
|
const failureMode = classifyReviewerThrownFailure(error, outcome);
|
|
607
|
-
return
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
if (!blocking) {
|
|
612
|
-
return makeCell('reviewer', fixture.id, true, null, null, 'good');
|
|
613
|
-
}
|
|
614
|
-
return makeCell('reviewer', fixture.id, false, 'wrong_or_empty_output', 'reviewer flagged the known-good diff with a blocking finding', 'good');
|
|
626
|
+
return {
|
|
627
|
+
status: 'error',
|
|
628
|
+
cell: makeCell('reviewer', fixture.id, false, failureMode, error instanceof Error ? error.message : String(error), diffKind, null),
|
|
629
|
+
};
|
|
615
630
|
}
|
|
616
|
-
//
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
631
|
+
// No pass/fail judgement here: the verdict is a property of the pair.
|
|
632
|
+
const blocking = collectBlockingFindings(result);
|
|
633
|
+
return {
|
|
634
|
+
status: 'scored',
|
|
635
|
+
blockingCount: blocking.length,
|
|
636
|
+
blockingSummary: summarizeBlockingFindings(blocking),
|
|
637
|
+
};
|
|
621
638
|
}
|
|
622
639
|
finally {
|
|
623
640
|
rmSync(throwawayCwd, { recursive: true, force: true });
|
|
@@ -641,9 +658,51 @@ async function readReviewOutcome(cwd, reviewId) {
|
|
|
641
658
|
}
|
|
642
659
|
}
|
|
643
660
|
export async function evaluateReviewerFixture(args) {
|
|
644
|
-
const good = await
|
|
645
|
-
const broken = await
|
|
646
|
-
|
|
661
|
+
const good = await runReviewerDiff({ ...args, diffKind: 'good' });
|
|
662
|
+
const broken = await runReviewerDiff({ ...args, diffKind: 'broken' });
|
|
663
|
+
// Either diff erroring makes the pair unscoreable: no discrimination verdict
|
|
664
|
+
// is computable, so attribute the systematic cause to both cells.
|
|
665
|
+
if (good.status === 'error' && broken.status === 'error') {
|
|
666
|
+
return { good: good.cell, broken: broken.cell };
|
|
667
|
+
}
|
|
668
|
+
if (good.status === 'error') {
|
|
669
|
+
return {
|
|
670
|
+
good: good.cell,
|
|
671
|
+
broken: broken.status === 'scored'
|
|
672
|
+
? unscoreableCell(args.fixture.id, 'broken', broken.blockingCount, good.cell)
|
|
673
|
+
: broken.cell,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
if (broken.status === 'error') {
|
|
677
|
+
return {
|
|
678
|
+
good: unscoreableCell(args.fixture.id, 'good', good.blockingCount, broken.cell),
|
|
679
|
+
broken: broken.cell,
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
// Discrimination criterion: severity calibration is not graded. A reviewer may
|
|
683
|
+
// raise blocking findings on the good diff and still PASS, as long as the
|
|
684
|
+
// broken diff draws strictly more.
|
|
685
|
+
const pass = broken.blockingCount >= 1 && good.blockingCount < broken.blockingCount;
|
|
686
|
+
if (pass) {
|
|
687
|
+
return {
|
|
688
|
+
good: makeCell('reviewer', args.fixture.id, true, null, null, 'good', good.blockingCount),
|
|
689
|
+
broken: makeCell('reviewer', args.fixture.id, true, null, null, 'broken', broken.blockingCount),
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
const base = broken.blockingCount === 0
|
|
693
|
+
? `reviewer raised no blocking finding on the broken diff (blocking good=${good.blockingCount}, broken=0)`
|
|
694
|
+
: `reviewer did not discriminate: blocking good=${good.blockingCount} >= broken=${broken.blockingCount}`;
|
|
695
|
+
const detailFor = (summary) => summary === '' ? base : `${base} | blocking: ${summary}`;
|
|
696
|
+
return {
|
|
697
|
+
good: makeCell('reviewer', args.fixture.id, false, 'wrong_or_empty_output', detailFor(good.blockingSummary), 'good', good.blockingCount),
|
|
698
|
+
broken: makeCell('reviewer', args.fixture.id, false, 'wrong_or_empty_output', detailFor(broken.blockingSummary), 'broken', broken.blockingCount),
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
// A scored diff whose partner errored: carry the partner's failure mode so
|
|
702
|
+
// smoke aggregation attributes the systematic cause rather than a bogus verdict.
|
|
703
|
+
function unscoreableCell(fixtureId, diffKind, blockingCount, partnerCell) {
|
|
704
|
+
const partnerDiffKind = diffKind === 'good' ? 'broken' : 'good';
|
|
705
|
+
return makeCell('reviewer', fixtureId, false, partnerCell.failureMode, `pair unscoreable: the ${partnerDiffKind} diff review failed (${partnerCell.failureMode ?? 'unknown'})`, diffKind, blockingCount);
|
|
647
706
|
}
|
|
648
707
|
// ---------------------------------------------------------------------------
|
|
649
708
|
// Output formatting
|
|
@@ -704,6 +763,11 @@ export function formatCompatTable(report) {
|
|
|
704
763
|
}
|
|
705
764
|
export async function runCompat(args) {
|
|
706
765
|
const { cwd, parsed } = args;
|
|
766
|
+
// compat is a hermetic capability probe that runs the whole role matrix
|
|
767
|
+
// through the native SDKs. Isolate those adapters from the operator's
|
|
768
|
+
// interactive config so the probe stays quiet (no per-turn notifier hooks)
|
|
769
|
+
// and repeatable. Normal neal runs never call this, so they honor the config.
|
|
770
|
+
enableAgentSettingsIsolation();
|
|
707
771
|
const deps = args.deps ?? {};
|
|
708
772
|
const compatDir = deps.compatDir ?? getCompatExamplesDir();
|
|
709
773
|
const manifest = deps.manifest ?? loadCompatManifest(compatDir);
|
|
@@ -778,14 +842,14 @@ export async function runCompat(args) {
|
|
|
778
842
|
const roles = buildRoleRollups(cells, targetedRoles);
|
|
779
843
|
const overallPass = roles.length > 0 && roles.every((rollup) => rollup.pass);
|
|
780
844
|
return {
|
|
781
|
-
schemaVersion:
|
|
845
|
+
schemaVersion: 2,
|
|
782
846
|
model: parsed.model,
|
|
783
847
|
reference: parsed.reference,
|
|
784
848
|
role: parsed.role,
|
|
785
849
|
candidateProviders: {
|
|
786
|
-
coder: parsed.model !== null ? '
|
|
787
|
-
reviewer: parsed.model !== null ? '
|
|
788
|
-
planner: parsed.model !== null ? '
|
|
850
|
+
coder: parsed.model !== null ? 'openai-compatible' : baseConfig.coder.provider,
|
|
851
|
+
reviewer: parsed.model !== null ? 'openai-compatible' : baseConfig.reviewer.provider,
|
|
852
|
+
planner: parsed.model !== null ? 'openai-compatible' : baseConfig.planner.provider,
|
|
789
853
|
},
|
|
790
854
|
cells,
|
|
791
855
|
roles,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Writable } from 'node:stream';
|
|
2
2
|
import { ActivityFooter } from '../activity-footer.js';
|
|
3
3
|
import { clearDiagnosticFooter, configureDiagnosticFooter, resetDiagnosticDetailState, setDiagnosticDetailContext, writeNarrative, } from '../diagnostic.js';
|
|
4
|
-
import { createInteractiveKeyController } from '../interactive-controls.js';
|
|
4
|
+
import { createInteractiveKeyController, renderInteractiveKeyHint } from '../interactive-controls.js';
|
|
5
5
|
export async function withInteractiveActivity(options, action) {
|
|
6
6
|
let detailContext = {};
|
|
7
7
|
resetDiagnosticDetailState(detailContext);
|
|
@@ -34,7 +34,7 @@ export async function withInteractiveActivity(options, action) {
|
|
|
34
34
|
};
|
|
35
35
|
try {
|
|
36
36
|
if (process.stdin.isTTY) {
|
|
37
|
-
writeNarrative(
|
|
37
|
+
writeNarrative(renderInteractiveKeyHint(false));
|
|
38
38
|
}
|
|
39
39
|
return await action({
|
|
40
40
|
stderr: createNarrativeWritable(),
|
|
@@ -6,6 +6,7 @@ import { assertAgentConfigSupportsResume } from '../providers/registry.js';
|
|
|
6
6
|
import { formatPublicPhase } from '../phase-display.js';
|
|
7
7
|
import { logUserGuidanceApplied } from '../prompts/guidance.js';
|
|
8
8
|
import { decideResumeAction } from '../resume-decision.js';
|
|
9
|
+
import { getPlanReviewGuidanceOriginPhase } from '../state-views.js';
|
|
9
10
|
import { loadState, saveState } from '../state.js';
|
|
10
11
|
function getSelectedRunId(state, selectedRunId) {
|
|
11
12
|
return selectedRunId && selectedRunId !== 'latest' ? selectedRunId : basename(state.runDir);
|
|
@@ -67,11 +68,10 @@ function formatGuidanceDecisionRejection(decision) {
|
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
function isEligiblePlanReviewGuidanceState(state) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
state.pendingPlanReviewGuidance === null);
|
|
71
|
+
// Shared discriminator: reviewer_plan plus coder-authored *response* blocks
|
|
72
|
+
// (blockerReason non-null). A dirty-worktree safety block lands at the same
|
|
73
|
+
// response phase with blockerReason null and is not answerable via --message.
|
|
74
|
+
return getPlanReviewGuidanceOriginPhase(state) !== null;
|
|
75
75
|
}
|
|
76
76
|
async function recordPlanReviewGuidance(args) {
|
|
77
77
|
const resumeCommand = getResumeCommand(args.state, args.target);
|
|
@@ -97,6 +97,16 @@ async function recordPlanReviewGuidance(args) {
|
|
|
97
97
|
if (!isEligiblePlanReviewGuidanceState(args.state)) {
|
|
98
98
|
throw new Error(describePlanReviewGuidanceStateProblem(args.state, args.target));
|
|
99
99
|
}
|
|
100
|
+
const originPhase = getPlanReviewGuidanceOriginPhase(args.state);
|
|
101
|
+
if (originPhase === null) {
|
|
102
|
+
throw new Error(describePlanReviewGuidanceStateProblem(args.state, args.target));
|
|
103
|
+
}
|
|
104
|
+
// Map the origin block to the phase that will actually consume the guidance:
|
|
105
|
+
// a reviewer_plan block delivers to coder_plan_response (unchanged behavior),
|
|
106
|
+
// while a coder-authored response block returns to its own origin phase so the
|
|
107
|
+
// resumed round selects the right open findings and delivers the guidance
|
|
108
|
+
// instead of accepting on an empty blocking set.
|
|
109
|
+
const resumePhase = originPhase === 'reviewer_plan' ? 'coder_plan_response' : originPhase;
|
|
100
110
|
assertAgentConfigSupportsResume(args.state.agentConfig, args.state, {
|
|
101
111
|
context: 'record plan-review guidance for resume',
|
|
102
112
|
});
|
|
@@ -110,19 +120,21 @@ async function recordPlanReviewGuidance(args) {
|
|
|
110
120
|
await logUserGuidanceApplied(logger);
|
|
111
121
|
const nextState = await saveState(args.target.statePath, {
|
|
112
122
|
...args.state,
|
|
113
|
-
phase:
|
|
123
|
+
phase: resumePhase,
|
|
114
124
|
status: 'running',
|
|
115
125
|
blockedFromPhase: null,
|
|
126
|
+
// Returning to running clears the durable coder-authored blocker reason.
|
|
127
|
+
blockerReason: null,
|
|
116
128
|
pendingPlanReviewGuidance: {
|
|
117
129
|
message: trimmedGuidance,
|
|
118
|
-
sourcePhase:
|
|
130
|
+
sourcePhase: originPhase,
|
|
119
131
|
recordedAt: new Date().toISOString(),
|
|
120
132
|
},
|
|
121
133
|
});
|
|
122
134
|
await writeExecutionArtifacts(nextState);
|
|
123
135
|
await logger.event('plan_review_guidance.recorded', {
|
|
124
136
|
statePath: args.target.statePath,
|
|
125
|
-
sourcePhase:
|
|
137
|
+
sourcePhase: originPhase,
|
|
126
138
|
guidanceBytes: Buffer.byteLength(trimmedGuidance, 'utf8'),
|
|
127
139
|
});
|
|
128
140
|
return {
|
|
@@ -3,7 +3,7 @@ import { basename, dirname, join } from 'node:path';
|
|
|
3
3
|
import process from 'node:process';
|
|
4
4
|
import { clearDiagnosticFooter, configureDiagnosticFooter, resetDiagnosticDetailState, setDiagnosticDetailContext, writeDiagnostic, } from '../diagnostic.js';
|
|
5
5
|
import { buildBlockedGuidance, renderBlockedGuidanceSections, } from '../blocked-guidance.js';
|
|
6
|
-
import { createInteractiveKeyController } from '../interactive-controls.js';
|
|
6
|
+
import { createInteractiveKeyController, renderInteractiveKeyHint } from '../interactive-controls.js';
|
|
7
7
|
import { createRunId } from '../logger.js';
|
|
8
8
|
import { runOnePass } from '../orchestrator.js';
|
|
9
9
|
import { writeExecutionArtifacts } from '../orchestrator/artifacts.js';
|
|
@@ -399,14 +399,16 @@ export async function executeRun(state, statePath, logger, options = {}) {
|
|
|
399
399
|
await footer.setState(state, displayedPhaseStartedAt);
|
|
400
400
|
const narrator = createTerminalNarrator({ logger });
|
|
401
401
|
await narrator.start(state);
|
|
402
|
+
const allowStopRequest = state.topLevelMode === 'execute';
|
|
402
403
|
const stopController = createInteractiveKeyController({
|
|
404
|
+
allowStopRequest,
|
|
403
405
|
getDetailFilter() {
|
|
404
406
|
return currentDetailContext;
|
|
405
407
|
},
|
|
406
408
|
});
|
|
407
409
|
try {
|
|
408
410
|
if (process.stdin.isTTY) {
|
|
409
|
-
writeDiagnostic(
|
|
411
|
+
writeDiagnostic(renderInteractiveKeyHint(allowStopRequest));
|
|
410
412
|
}
|
|
411
413
|
let finalState;
|
|
412
414
|
try {
|