@navels/neal 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/neal/adjudicator/planning.js +48 -0
- package/dist/neal/agents/prompts.js +3 -0
- package/dist/neal/agents/rounds.js +8 -0
- package/dist/neal/agents/schemas.js +575 -496
- package/dist/neal/agents/structured-json.js +36 -0
- package/dist/neal/config.js +24 -0
- package/dist/neal/git.js +9 -3
- package/dist/neal/orchestrator/completion.js +167 -112
- package/dist/neal/orchestrator/phases/planning.js +14 -39
- package/dist/neal/orchestrator/split-plan.js +12 -11
- package/dist/neal/orchestrator/transitions.js +30 -71
- package/dist/neal/plan-doc.js +24 -1
- package/dist/neal/prompts/assert-builder.js +8 -1
- package/dist/neal/prompts/execute.js +4 -0
- package/dist/neal/prompts/specialized.js +22 -6
- package/dist/neal/prompts/specs.js +58 -0
- package/dist/neal/providers/anthropic-claude.js +291 -247
- package/dist/neal/providers/generic-agentic.js +18 -0
- package/dist/neal/providers/openai-codex.js +77 -201
- package/dist/neal/providers/openai-compatible.js +33 -5
- package/dist/neal/providers/pricing.js +124 -0
- package/dist/neal/providers/rate-card.js +2301 -0
- package/dist/neal/providers/telemetry.js +4 -0
- package/dist/neal/retrospective.js +33 -4
- package/dist/neal/run-metrics.js +74 -9
- package/docs/compatible-models.md +11 -0
- package/docs/issue-pipeline.md +124 -0
- package/docs/maintenance.md +30 -19
- package/docs/providers.md +117 -0
- package/docs/release.md +29 -25
- package/package.json +7 -3
|
@@ -63,6 +63,8 @@ async function writeRunEvent(event, options) {
|
|
|
63
63
|
await logger.event('provider.turn_completed', {
|
|
64
64
|
...commonEventData(event),
|
|
65
65
|
...(event.usage !== undefined ? { usage: event.usage } : {}),
|
|
66
|
+
...(event.costUsd !== undefined ? { costUsd: event.costUsd } : {}),
|
|
67
|
+
...(event.costSource !== undefined ? { costSource: event.costSource } : {}),
|
|
66
68
|
});
|
|
67
69
|
break;
|
|
68
70
|
case 'tool_started':
|
|
@@ -121,6 +123,8 @@ async function writeRunEvent(event, options) {
|
|
|
121
123
|
await logger.event('provider.usage_reported', {
|
|
122
124
|
...commonEventData(event),
|
|
123
125
|
usage: event.usage,
|
|
126
|
+
...(event.costUsd !== undefined ? { costUsd: event.costUsd } : {}),
|
|
127
|
+
...(event.costSource !== undefined ? { costSource: event.costSource } : {}),
|
|
124
128
|
});
|
|
125
129
|
break;
|
|
126
130
|
case 'provider_error':
|
|
@@ -28,6 +28,26 @@ function getArchivedRetrospectivePath(state, kind) {
|
|
|
28
28
|
const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
|
|
29
29
|
return join(state.runDir, `RETROSPECTIVE-final${suffix}.md`);
|
|
30
30
|
}
|
|
31
|
+
function getCurrentRunMetricsPath(runDir) {
|
|
32
|
+
return join(runDir, 'RUN_METRICS.json');
|
|
33
|
+
}
|
|
34
|
+
// Mirrors getArchivedRetrospectivePath's scope/kind/commit naming so the
|
|
35
|
+
// machine-readable metrics archive lines up with the retrospective archive.
|
|
36
|
+
function getArchivedRunMetricsPath(state, kind) {
|
|
37
|
+
const scopeLabel = getCurrentScopeLabel(state);
|
|
38
|
+
if (kind === 'scope_accepted') {
|
|
39
|
+
const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
|
|
40
|
+
return join(state.runDir, `RUN_METRICS-scope-${scopeLabel}${suffix}.json`);
|
|
41
|
+
}
|
|
42
|
+
if (kind === 'blocked') {
|
|
43
|
+
return join(state.runDir, `RUN_METRICS-blocked-scope-${scopeLabel}.json`);
|
|
44
|
+
}
|
|
45
|
+
if (kind === 'failed') {
|
|
46
|
+
return join(state.runDir, `RUN_METRICS-failed-scope-${scopeLabel}.json`);
|
|
47
|
+
}
|
|
48
|
+
const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
|
|
49
|
+
return join(state.runDir, `RUN_METRICS-final${suffix}.json`);
|
|
50
|
+
}
|
|
31
51
|
async function loadRunEvents(runDir) {
|
|
32
52
|
try {
|
|
33
53
|
const content = await readFile(getEventsPath(runDir), 'utf8');
|
|
@@ -306,7 +326,8 @@ async function renderRetrospective(state, kind) {
|
|
|
306
326
|
const changedFiles = await summarizeChangedFiles(state);
|
|
307
327
|
const verificationSummary = summarizeVerification(scopeEvents);
|
|
308
328
|
const metricsEvents = kind === 'done' ? events : scopeEvents;
|
|
309
|
-
const
|
|
329
|
+
const metrics = summarizeRunMetrics(metricsEvents);
|
|
330
|
+
const runMetricsSummary = renderRunMetricsMarkdown(metrics);
|
|
310
331
|
const assessment = buildAssessment(state, scopeEvents);
|
|
311
332
|
const narrativeRetrospective = buildNarrativeRetrospective({
|
|
312
333
|
state,
|
|
@@ -320,7 +341,7 @@ async function renderRetrospective(state, kind) {
|
|
|
320
341
|
const derivedPlan = getDerivedPlanView(state);
|
|
321
342
|
const showDerivedPlanContext = Boolean(derivedPlan);
|
|
322
343
|
const parentScopeLabel = derivedPlan?.executing ? getParentScopeLabel(state) : null;
|
|
323
|
-
|
|
344
|
+
const content = [
|
|
324
345
|
`# Neal Retrospective`,
|
|
325
346
|
'',
|
|
326
347
|
`## Outcome`,
|
|
@@ -377,15 +398,23 @@ async function renderRetrospective(state, kind) {
|
|
|
377
398
|
...renderInteractiveBlockedRecoveryHistoryLines(state.interactiveBlockedRecoveryHistory),
|
|
378
399
|
'',
|
|
379
400
|
].join('\n');
|
|
401
|
+
return { content, metrics };
|
|
380
402
|
}
|
|
381
403
|
async function writeRetrospectiveFile(path, content) {
|
|
382
404
|
await writeTextAtomic(path, content);
|
|
383
405
|
}
|
|
406
|
+
async function writeRunMetricsFile(path, metrics) {
|
|
407
|
+
await writeTextAtomic(path, `${JSON.stringify(metrics, null, 2)}\n`);
|
|
408
|
+
}
|
|
384
409
|
export async function writeCheckpointRetrospective(state, kind) {
|
|
385
|
-
const content = await renderRetrospective(state, kind);
|
|
410
|
+
const { content, metrics } = await renderRetrospective(state, kind);
|
|
386
411
|
const currentPath = getCurrentRetrospectivePath(state.runDir);
|
|
387
412
|
const archivedPath = getArchivedRetrospectivePath(state, kind);
|
|
388
413
|
await writeRetrospectiveFile(currentPath, content);
|
|
389
414
|
await writeRetrospectiveFile(archivedPath, content);
|
|
390
|
-
|
|
415
|
+
const currentMetricsPath = getCurrentRunMetricsPath(state.runDir);
|
|
416
|
+
const archivedMetricsPath = getArchivedRunMetricsPath(state, kind);
|
|
417
|
+
await writeRunMetricsFile(currentMetricsPath, metrics);
|
|
418
|
+
await writeRunMetricsFile(archivedMetricsPath, metrics);
|
|
419
|
+
return { currentPath, archivedPath, currentMetricsPath, archivedMetricsPath };
|
|
391
420
|
}
|
package/dist/neal/run-metrics.js
CHANGED
|
@@ -48,6 +48,21 @@ function addUsage(target, value) {
|
|
|
48
48
|
function hasUsage(usage) {
|
|
49
49
|
return Object.values(usage).some((value) => value > 0);
|
|
50
50
|
}
|
|
51
|
+
// Accumulate cost from one counted event into the provider bucket. Presence is
|
|
52
|
+
// tracked separately from value: costUsd stays null until an event carries a
|
|
53
|
+
// numeric costUsd, then it holds the running sum. A single provider/role bucket
|
|
54
|
+
// has one source; if both appear, 'provider' wins.
|
|
55
|
+
function accumulateCost(bucket, data) {
|
|
56
|
+
const raw = data.costUsd;
|
|
57
|
+
if (typeof raw !== 'number' || !Number.isFinite(raw)) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
bucket.costUsd = (bucket.costUsd ?? 0) + raw;
|
|
61
|
+
const source = data.costSource;
|
|
62
|
+
if (source === 'provider' || source === 'rate') {
|
|
63
|
+
bucket.costSource = bucket.costSource === 'provider' || source === 'provider' ? 'provider' : 'rate';
|
|
64
|
+
}
|
|
65
|
+
}
|
|
51
66
|
function cloneEmptyUsage() {
|
|
52
67
|
return { ...EMPTY_USAGE };
|
|
53
68
|
}
|
|
@@ -210,10 +225,13 @@ export function summarizeRunMetrics(events) {
|
|
|
210
225
|
label: identity.label,
|
|
211
226
|
turns: 0,
|
|
212
227
|
usage: cloneEmptyUsage(),
|
|
228
|
+
costUsd: null,
|
|
229
|
+
costSource: null,
|
|
213
230
|
};
|
|
214
231
|
existing.turns += 1;
|
|
215
232
|
if (!hasDedicatedUsageEvents) {
|
|
216
233
|
addUsage(existing.usage, data.usage);
|
|
234
|
+
accumulateCost(existing, data);
|
|
217
235
|
}
|
|
218
236
|
providers.set(identity.key, existing);
|
|
219
237
|
}
|
|
@@ -226,8 +244,11 @@ export function summarizeRunMetrics(events) {
|
|
|
226
244
|
label: identity.label,
|
|
227
245
|
turns: 0,
|
|
228
246
|
usage: cloneEmptyUsage(),
|
|
247
|
+
costUsd: null,
|
|
248
|
+
costSource: null,
|
|
229
249
|
};
|
|
230
250
|
addUsage(existing.usage, data.usage);
|
|
251
|
+
accumulateCost(existing, data);
|
|
231
252
|
providers.set(identity.key, existing);
|
|
232
253
|
}
|
|
233
254
|
else if (event.type === 'provider.command_completed') {
|
|
@@ -276,6 +297,25 @@ export function summarizeRunMetrics(events) {
|
|
|
276
297
|
phase.durationMs = start !== null && end !== null && end >= start ? end - start : null;
|
|
277
298
|
}
|
|
278
299
|
const publicPhases = phases.map(({ startIndex: _startIndex, endIndex: _endIndex, ...phase }) => phase);
|
|
300
|
+
const providerSummaries = [...providers.values()].sort((left, right) => {
|
|
301
|
+
const turns = right.turns - left.turns;
|
|
302
|
+
if (turns !== 0) {
|
|
303
|
+
return turns;
|
|
304
|
+
}
|
|
305
|
+
return usageSortValue(right) - usageSortValue(left);
|
|
306
|
+
});
|
|
307
|
+
// Coverage is computed only over usage-bearing buckets so a partial subtotal
|
|
308
|
+
// never masquerades as a complete run total.
|
|
309
|
+
const usageBearing = providerSummaries.filter((provider) => hasUsage(provider.usage));
|
|
310
|
+
const pricedUsageBearing = usageBearing.filter((provider) => provider.costUsd !== null);
|
|
311
|
+
const costCoverage = pricedUsageBearing.length === 0
|
|
312
|
+
? 'none'
|
|
313
|
+
: pricedUsageBearing.length === usageBearing.length
|
|
314
|
+
? 'complete'
|
|
315
|
+
: 'partial';
|
|
316
|
+
const totalCostUsd = costCoverage === 'none'
|
|
317
|
+
? null
|
|
318
|
+
: providerSummaries.reduce((sum, provider) => (provider.costUsd !== null ? sum + provider.costUsd : sum), 0);
|
|
279
319
|
return {
|
|
280
320
|
observedStartedAt: firstTimestamp,
|
|
281
321
|
observedCompletedAt: lastTimestamp,
|
|
@@ -288,13 +328,9 @@ export function summarizeRunMetrics(events) {
|
|
|
288
328
|
toolEventCount,
|
|
289
329
|
fileChangeEventCount,
|
|
290
330
|
phases: publicPhases,
|
|
291
|
-
providers:
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
return turns;
|
|
295
|
-
}
|
|
296
|
-
return usageSortValue(right) - usageSortValue(left);
|
|
297
|
-
}),
|
|
331
|
+
providers: providerSummaries,
|
|
332
|
+
totalCostUsd,
|
|
333
|
+
costCoverage,
|
|
298
334
|
};
|
|
299
335
|
}
|
|
300
336
|
function formatDuration(value) {
|
|
@@ -327,6 +363,31 @@ function formatProviderName(summary) {
|
|
|
327
363
|
function usageCell(value) {
|
|
328
364
|
return value > 0 ? formatNumber(value) : '-';
|
|
329
365
|
}
|
|
366
|
+
function formatCostAmount(value) {
|
|
367
|
+
return `$${value.toFixed(4)}`;
|
|
368
|
+
}
|
|
369
|
+
// Cost cell for the Provider Usage table: '-' when the bucket reported no cost,
|
|
370
|
+
// otherwise the fixed-precision USD amount with a footnote marker on
|
|
371
|
+
// rate-computed cells.
|
|
372
|
+
function formatCost(value, source) {
|
|
373
|
+
if (value === null) {
|
|
374
|
+
return '-';
|
|
375
|
+
}
|
|
376
|
+
return source === 'rate' ? `${formatCostAmount(value)}*` : formatCostAmount(value);
|
|
377
|
+
}
|
|
378
|
+
// The top-summary cost line, labeled by coverage so a partial subtotal is never
|
|
379
|
+
// presented as a complete run total.
|
|
380
|
+
function formatEstimatedCostLine(metrics) {
|
|
381
|
+
if (metrics.costCoverage === 'none' || metrics.totalCostUsd === null) {
|
|
382
|
+
return '- Estimated cost: unknown';
|
|
383
|
+
}
|
|
384
|
+
if (metrics.costCoverage === 'partial') {
|
|
385
|
+
const usageBearing = metrics.providers.filter((provider) => hasUsage(provider.usage));
|
|
386
|
+
const priced = usageBearing.filter((provider) => provider.costUsd !== null);
|
|
387
|
+
return `- Estimated cost (partial — ${priced.length} of ${usageBearing.length} priced providers): ${formatCostAmount(metrics.totalCostUsd)}`;
|
|
388
|
+
}
|
|
389
|
+
return `- Estimated cost: ${formatCostAmount(metrics.totalCostUsd)}`;
|
|
390
|
+
}
|
|
330
391
|
export function renderRunMetricsMarkdown(metrics) {
|
|
331
392
|
const commandSummary = metrics.resolvedNonZeroCommandCount > 0
|
|
332
393
|
? `- Commands: ${metrics.commandCount} total, ${metrics.unresolvedNonZeroCommandCount} unresolved non-zero or failed, ${metrics.resolvedNonZeroCommandCount} resolved by later passing rerun`
|
|
@@ -337,16 +398,20 @@ export function renderRunMetricsMarkdown(metrics) {
|
|
|
337
398
|
commandSummary,
|
|
338
399
|
`- Tool events: ${metrics.toolEventCount}`,
|
|
339
400
|
`- File change events: ${metrics.fileChangeEventCount}`,
|
|
401
|
+
formatEstimatedCostLine(metrics),
|
|
340
402
|
];
|
|
341
403
|
if (metrics.phases.length > 0) {
|
|
342
404
|
lines.push('', '### Phase Timing', '', '| Phase | Duration | Provider turns | Commands | Non-zero commands | Tool events | File changes |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', ...metrics.phases.map((phase) => `| ${formatPhaseName(phase.phase)} | ${formatPhaseDuration(phase)} | ${phase.providerTurns} | ${phase.commandCount} | ${phase.nonZeroCommandCount} | ${phase.toolEventCount} | ${phase.fileChangeEventCount} |`));
|
|
343
405
|
}
|
|
344
406
|
const providersWithUsage = metrics.providers.filter((provider) => provider.turns > 0 || hasUsage(provider.usage));
|
|
345
407
|
if (providersWithUsage.length > 0) {
|
|
346
|
-
lines.push('', '### Provider Usage', '', '| Provider / role | Turns | Input | Cached input | Cache created | Cache read | Output | Reasoning output | Total |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...providersWithUsage.map((provider) => {
|
|
408
|
+
lines.push('', '### Provider Usage', '', '| Provider / role | Turns | Input | Cached input | Cache created | Cache read | Output | Reasoning output | Total | Cost |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...providersWithUsage.map((provider) => {
|
|
347
409
|
const usage = provider.usage;
|
|
348
|
-
return `| ${formatProviderName(provider)} | ${provider.turns} | ${usageCell(usage.inputTokens)} | ${usageCell(usage.cachedInputTokens)} | ${usageCell(usage.cacheCreationInputTokens)} | ${usageCell(usage.cacheReadInputTokens)} | ${usageCell(usage.outputTokens)} | ${usageCell(usage.reasoningOutputTokens)} | ${usageCell(usage.totalTokens)} |`;
|
|
410
|
+
return `| ${formatProviderName(provider)} | ${provider.turns} | ${usageCell(usage.inputTokens)} | ${usageCell(usage.cachedInputTokens)} | ${usageCell(usage.cacheCreationInputTokens)} | ${usageCell(usage.cacheReadInputTokens)} | ${usageCell(usage.outputTokens)} | ${usageCell(usage.reasoningOutputTokens)} | ${usageCell(usage.totalTokens)} | ${formatCost(provider.costUsd, provider.costSource)} |`;
|
|
349
411
|
}));
|
|
412
|
+
if (providersWithUsage.some((provider) => provider.costUsd !== null && provider.costSource === 'rate')) {
|
|
413
|
+
lines.push('', '\\* Cost estimated from published or configured rates, not reported by the provider.');
|
|
414
|
+
}
|
|
350
415
|
}
|
|
351
416
|
else {
|
|
352
417
|
lines.push('', '### Provider Usage', '', '- No provider usage events recorded.');
|
|
@@ -24,6 +24,17 @@
|
|
|
24
24
|
|
|
25
25
|
Sorted cheapest first ($/Mtok in·out, from the live OpenRouter catalog).
|
|
26
26
|
|
|
27
|
+
> **Drift observed (2026-07-14):** `google/gemma-4-26b-a4b-it` — an all-roles PASS on
|
|
28
|
+
> 2026-06-18 — failed the coder role 3/3 with `structured_output` in the CI smoke
|
|
29
|
+
> (self-reference configuration). Slugs rot as providers re-route or re-quantize;
|
|
30
|
+
> treat its row below as stale until re-qualified against the codex reference.
|
|
31
|
+
>
|
|
32
|
+
> **Self-reference caveat (2026-07-14):** `inclusionai/ling-2.6-1t` repeatedly failed the
|
|
33
|
+
> reviewer broken-diff cell (2/3 attempts) in the CI smoke when serving as **its own
|
|
34
|
+
> partner**. Its codex-referenced row below stands — this is a partner-duty weakness,
|
|
35
|
+
> the same class the reference investigation documents. The smoke pin is
|
|
36
|
+
> `minimax/minimax-m2.5`, which held up under self-reference.
|
|
37
|
+
|
|
27
38
|
| Model | Coder | Reviewer | Planner | $/Mtok | ctx |
|
|
28
39
|
|---|---|---|---|---|---|
|
|
29
40
|
| `google/gemma-4-26b-a4b-it` | 3/3 | 6/6 | 1/1 | 0.06·0.33 | 262k |
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# GitHub Issue Pipeline
|
|
2
|
+
|
|
3
|
+
Assign a GitHub issue to neal and get a reviewed pull request back. A maintainer
|
|
4
|
+
applies a trigger label to an issue; a GitHub Actions runner turns the issue
|
|
5
|
+
into a neal plan, runs neal's planner/coder/reviewer loop to completion, and
|
|
6
|
+
opens a pull request assigned to a human — or reports back on the issue if neal
|
|
7
|
+
blocks. Humans own merge; the pipeline never pushes to a protected branch.
|
|
8
|
+
|
|
9
|
+
This is a self-hosting use of neal itself: neal already *is* the middle of the
|
|
10
|
+
loop (cross-model review, deterministic gating, bounded recovery), so the
|
|
11
|
+
pipeline is a thin trigger plus a thin publish around `neal run`.
|
|
12
|
+
|
|
13
|
+
## What it does
|
|
14
|
+
|
|
15
|
+
```
|
|
16
|
+
maintainer applies the trigger label to an issue
|
|
17
|
+
│ (gate: the labeler must have write access — the issue author is irrelevant)
|
|
18
|
+
▼
|
|
19
|
+
issue title + body ──▶ seed .neal/PLAN.md ──▶ neal run --unattended
|
|
20
|
+
│ (Opus 4.8 writes, sol reviews each diff)
|
|
21
|
+
├─ exit 0 ──▶ push neal/issue-<n>, open a PR, assign the maintainer
|
|
22
|
+
├─ exit 2 ──▶ comment the blocker on the issue, keep the branch
|
|
23
|
+
└─ exit 3 ──▶ comment the failure on the issue, open no PR
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The plan is seeded inside `.neal/` — wrapper-owned, gitignored territory — so
|
|
27
|
+
it can never collide with a tracked file or appear in review diffs, and it is
|
|
28
|
+
never committed. The refined plan travels with the outcome instead: the PR
|
|
29
|
+
body (or the blocked/failed issue comment) embeds it in a collapsed details
|
|
30
|
+
block, which outlives the 3-day `.neal/` artifact retention.
|
|
31
|
+
|
|
32
|
+
The outcome also carries a cost ledger — per-provider turns, tokens, and
|
|
33
|
+
dollars aggregated from the run's `RUN_METRICS-final*.json` files. Claude
|
|
34
|
+
cost is provider-reported; reviewer cost is rate-computed from neal's
|
|
35
|
+
vendored published rate card (see `docs/providers.md`) and marked as such.
|
|
36
|
+
|
|
37
|
+
## Running an issue locally
|
|
38
|
+
|
|
39
|
+
`scripts/neal-issue-local.sh <issue-number> [plan-doc]` is the pipeline's
|
|
40
|
+
local twin for subscription billing: the coder rides the Claude CLI login and
|
|
41
|
+
the reviewer the codex CLI login (per `~/.neal/config.yml`) instead of the
|
|
42
|
+
API keys CI uses. It shares the plan seed and PR-body construction with
|
|
43
|
+
`pipeline/run.sh` through `pipeline/lib.sh`, so a local run produces the same
|
|
44
|
+
branch name, the same PR format, and the same embedded plan and cost ledger.
|
|
45
|
+
The optional second argument substitutes a pre-authored plan (for example a
|
|
46
|
+
`multi_scope` plan spanning several issues) for the seeded `one_shot` one.
|
|
47
|
+
The script unsets `OPENAI_API_KEY`/`ANTHROPIC_API_KEY` for the run so a stray
|
|
48
|
+
exported key cannot silently switch billing from subscription to API.
|
|
49
|
+
|
|
50
|
+
Every run executes in its own git worktree under `<repo>-worktrees/issue-<n>`,
|
|
51
|
+
branched from `origin/main`. Runs on different issues can proceed in parallel
|
|
52
|
+
(git enforces one worktree per branch), and the main checkout stays free for
|
|
53
|
+
your own work throughout. On success the worktree is removed after the push —
|
|
54
|
+
the branch lives on the PR. On blocked/failed outcomes the worktree is kept:
|
|
55
|
+
it is the resume state (`cd` in and `neal resume`). The practical concurrency
|
|
56
|
+
cap is your subscriptions, not git: parallel runs share your Claude and
|
|
57
|
+
ChatGPT rate limits.
|
|
58
|
+
|
|
59
|
+
- **Coder:** Claude Opus 4.8 (`anthropic-claude` adapter, `ANTHROPIC_API_KEY`).
|
|
60
|
+
- **Reviewer:** GPT-5.6-sol via the `openai-compatible` adapter pointed at the
|
|
61
|
+
OpenAI API (`OPENAI_API_KEY`). The reviewer judges neal-inlined diff/context;
|
|
62
|
+
it has no repository tools, which keeps the runner setup to two env vars and
|
|
63
|
+
no extra CLI.
|
|
64
|
+
|
|
65
|
+
## Setup
|
|
66
|
+
|
|
67
|
+
One-time, by a maintainer. The workflow no-ops safely until this is done.
|
|
68
|
+
|
|
69
|
+
1. **Secrets** (repo → Settings → Secrets → Actions, on the `neal-pipeline`
|
|
70
|
+
environment): `ANTHROPIC_API_KEY` and `OPENAI_API_KEY`. Use short-lived,
|
|
71
|
+
minimally-scoped keys and rotate them (see Security below).
|
|
72
|
+
2. **Environment**: create a `neal-pipeline` environment (Settings →
|
|
73
|
+
Environments). Optionally add required reviewers so each run needs a manual
|
|
74
|
+
click before the secrets are exposed — recommended for a public repo.
|
|
75
|
+
3. **Reviewer model**: set the repo variable `NEAL_REVIEWER_MODEL` to the sol
|
|
76
|
+
model slug your OpenAI key serves. Before the first run, qualify it:
|
|
77
|
+
`neal compat --model <slug> --role reviewer --reference anthropic-claude`
|
|
78
|
+
(locally, with the same key). If sol is not reachable over the OpenAI
|
|
79
|
+
Chat Completions API, point `NEAL_REVIEWER_BASE_URL`/`NEAL_REVIEWER_MODEL` at
|
|
80
|
+
an endpoint that serves it, or switch the reviewer to the `openai-codex`
|
|
81
|
+
adapter in `pipeline/config.yml`.
|
|
82
|
+
4. **Trigger label**: create a label (default `neal:go`; override with the repo
|
|
83
|
+
variable `NEAL_TRIGGER_LABEL`). Optionally set `NEAL_READY_LABEL` (e.g.
|
|
84
|
+
`ready-for-human`) to tag the PRs neal opens.
|
|
85
|
+
5. **Branch protection**: protect `main` (require PR + review). The pipeline can
|
|
86
|
+
push `neal/issue-*` branches and open PRs, but must never be able to merge or
|
|
87
|
+
push to `main`.
|
|
88
|
+
|
|
89
|
+
## Security model
|
|
90
|
+
|
|
91
|
+
This runs on a public repository, so the trust boundary is "arbitrary internet
|
|
92
|
+
text becomes instructions to an agent with push access." The controls:
|
|
93
|
+
|
|
94
|
+
- **Maintainer-gated trigger.** The gate checks the *labeler's* repo permission
|
|
95
|
+
(`write`/`maintain`/`admin`), not the issue author. An untrusted issue cannot
|
|
96
|
+
start a run — only a maintainer applying the label can.
|
|
97
|
+
- **Untrusted issue input.** The issue title/body are passed as environment
|
|
98
|
+
variables (never interpolated into a shell command) and written to the seed
|
|
99
|
+
plan with `printf '%s'`, so they cannot inject shell commands. neal then
|
|
100
|
+
treats the seed as plan input, refined through the planner/reviewer loop.
|
|
101
|
+
- **Least privilege.** The workflow token can push `neal/issue-*` branches, open
|
|
102
|
+
PRs, and comment — nothing else. Branch protection keeps the human merge gate.
|
|
103
|
+
- **Boundaries in the plan.** The seed forbids edits to `.github/**` /
|
|
104
|
+
`pipeline/**` and forbids reading or printing secrets — defense-in-depth on
|
|
105
|
+
top of the token scope.
|
|
106
|
+
- **Secret exposure caveat.** The coder runs with shell in the runner, where
|
|
107
|
+
`ANTHROPIC_API_KEY` and `OPENAI_API_KEY` live in the environment. A
|
|
108
|
+
prompt-injected coder could try to exfiltrate them; the maintainer gate is
|
|
109
|
+
what makes that acceptable (only a maintainer's labeled issue ever runs). Use
|
|
110
|
+
short-lived, minimally-scoped keys, the `neal-pipeline` environment's approval
|
|
111
|
+
gate, and rotate on any suspicion.
|
|
112
|
+
|
|
113
|
+
## v1 limitations
|
|
114
|
+
|
|
115
|
+
- **Seed shape is `one_shot`.** Best for well-scoped issues ("fix X", "add Y").
|
|
116
|
+
For large multi-part work neal's coder will split the scope into a sub-plan;
|
|
117
|
+
if it can't be done in scope, neal blocks and comments asking for a smaller
|
|
118
|
+
issue. Write issues with a clear, bounded objective for best results.
|
|
119
|
+
- **Reviewer reads inlined context, not the repo.** The `openai-compatible`
|
|
120
|
+
reviewer judges the diff neal inlines, not the working tree. Switch to an
|
|
121
|
+
`openai-codex` reviewer (OS-sandboxed, repo-reading) once sol is confirmed on
|
|
122
|
+
that adapter's API path, if repo-reading review is wanted.
|
|
123
|
+
- **No comment-driven iteration yet.** On a blocked run you re-apply the label to
|
|
124
|
+
retry; a maintainer-comment → `neal resume --message` loop is a v2 addition.
|
package/docs/maintenance.md
CHANGED
|
@@ -4,37 +4,48 @@ neal's behavior is defined in large part by the agent SDKs it drives, so
|
|
|
4
4
|
dependency updates are a first-class concern, not routine hygiene. This document
|
|
5
5
|
is the policy.
|
|
6
6
|
|
|
7
|
-
##
|
|
7
|
+
## Three dependency tiers
|
|
8
8
|
|
|
9
9
|
| Tier | Packages | Pinning | Update posture |
|
|
10
10
|
| --- | --- | --- | --- |
|
|
11
|
-
| **
|
|
11
|
+
| **Native SDKs** | `@openai/codex-sdk`, `@anthropic-ai/claude-agent-sdk` | **exact** (enforced by `scripts/validate-release.mjs`) | **deliberate** — behaviorally qualified on a subscription-authenticated machine via `scripts/qualify-sdk.sh`; never auto-merge |
|
|
12
|
+
| **AI-SDK tier** | `ai`, `@ai-sdk/openai-compatible`, `zod` | **exact** (enforced by `scripts/validate-release.mjs`) | minor/patch **auto-merge** once CI **and the live smoke** are green (3-day soak); majors stay manual |
|
|
12
13
|
| **Utility** | `dotenv`, `yaml`, `@types/node`, `tsx`, `typescript` | caret OK | routine — auto-merge on green CI after a soak period |
|
|
13
14
|
|
|
14
15
|
The asymmetry exists because an agentic-SDK bump can change tool-calling,
|
|
15
16
|
structured output, or sandbox behavior — i.e. break neal's loop **without**
|
|
16
|
-
breaking compilation. Utility deps cannot.
|
|
17
|
+
breaking compilation. Utility deps cannot. The native/AI-SDK split exists
|
|
18
|
+
because CI can behaviorally exercise the AI-SDK tier (the smoke drives it
|
|
19
|
+
live through `generic-agentic`) but not the native adapters, whose auth is
|
|
20
|
+
subscription-based and lives only on a maintainer's machine.
|
|
17
21
|
|
|
18
22
|
## The update flow
|
|
19
23
|
|
|
20
|
-
1. **Detect.** [Renovate](../renovate.json) opens PRs weekly: utility deps
|
|
21
|
-
|
|
22
|
-
`
|
|
24
|
+
1. **Detect.** [Renovate](../renovate.json) opens PRs weekly: utility deps
|
|
25
|
+
grouped, AI-SDK and native SDKs one-per-PR, pinned exact, labelled
|
|
26
|
+
`agentic-sdk` (native and AI-SDK majors additionally `needs-qualification`).
|
|
23
27
|
2. **Verify (automatic).** CI (`.github/workflows/ci.yml`) runs typecheck + unit
|
|
24
|
-
tests + package verification — catches **API-shape / contract** breaks.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
28
|
+
tests + package verification — catches **API-shape / contract** breaks. The
|
|
29
|
+
live smoke (`.github/workflows/smoke.yml`) runs on every package.json /
|
|
30
|
+
lockfile PR: a real `neal compat` run against a cheap OpenRouter model
|
|
31
|
+
through `generic-agentic`, catching **behavioral** breaks in the AI-SDK
|
|
32
|
+
tier. AI-SDK minor/patch PRs auto-merge when both are green.
|
|
33
|
+
**The smoke requires the `OPENROUTER_API_KEY` repo secret** — without it
|
|
34
|
+
the smoke skips (green) and the AI-SDK auto-merge gate is compile-only.
|
|
35
|
+
3. **Verify (behavioral, native tier).** `@openai/codex-sdk` and
|
|
36
|
+
`@anthropic-ai/claude-agent-sdk` cannot be smoked in CI, so qualify them
|
|
37
|
+
from any checkout with authenticated Claude/Codex CLIs:
|
|
38
|
+
```
|
|
39
|
+
scripts/qualify-sdk.sh <pr-number>
|
|
40
|
+
```
|
|
41
|
+
It runs the full suite plus a live `neal compat --role all` pass-through on
|
|
42
|
+
the bumped adapter (in a throwaway worktree, with roles and models pinned
|
|
43
|
+
explicitly so nothing leaks from `~/.neal/config.yml`), posts the compat
|
|
44
|
+
matrix to the PR, and approves on PASS (`--merge` also squash-merges).
|
|
36
45
|
4. **Adopt.** Merge, bump neal's version, add a CHANGELOG entry noting the bump +
|
|
37
|
-
any behavior change, and cut a release via the existing workflow.
|
|
46
|
+
any behavior change, and cut a release via the existing workflow. Urgent
|
|
47
|
+
bumps (a fix neal needs immediately) may skip the Renovate soak with a
|
|
48
|
+
manual PR — qualify them the same way.
|
|
38
49
|
|
|
39
50
|
## Versioning
|
|
40
51
|
|
package/docs/providers.md
CHANGED
|
@@ -524,6 +524,69 @@ Settings resolve config-first with environment fallbacks:
|
|
|
524
524
|
One of these is required.
|
|
525
525
|
- `headers`: optional string-to-string map of extra HTTP headers (useful for
|
|
526
526
|
OpenRouter attribution headers).
|
|
527
|
+
- `pricing`: an **optional override** for per-million-token rates. It is no
|
|
528
|
+
longer required to get a dollar cost (see "Cost pricing" below); set it only
|
|
529
|
+
for models the vendored rate card does not key exactly, or to pin different
|
|
530
|
+
rates. All three rates are required when the block is present (a partial block
|
|
531
|
+
is a configuration error); rates are in USD per one million tokens:
|
|
532
|
+
|
|
533
|
+
```yaml
|
|
534
|
+
providers:
|
|
535
|
+
openai_compatible:
|
|
536
|
+
base_url: https://api.deepseek.com
|
|
537
|
+
api_key_env: DEEPSEEK_API_KEY
|
|
538
|
+
default_model: deepseek-chat
|
|
539
|
+
pricing:
|
|
540
|
+
input_per_million: 0.27
|
|
541
|
+
cached_input_per_million: 0.07
|
|
542
|
+
output_per_million: 1.10
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
#### Cost pricing
|
|
546
|
+
|
|
547
|
+
Neal resolves each run's dollar cost per provider/role bucket in this order:
|
|
548
|
+
|
|
549
|
+
1. **Provider-reported cost** — the Claude adapter passes through the provider's
|
|
550
|
+
own `total_cost_usd`; it always wins when present.
|
|
551
|
+
2. **Configured `pricing` override** — the
|
|
552
|
+
`providers.openai_compatible.pricing` block above (shared by the
|
|
553
|
+
`openai-compatible` and `generic-agentic` providers).
|
|
554
|
+
3. **Vendored published rate card** — the default rate source, keyed by model
|
|
555
|
+
slug. Neal ships a trimmed copy of LiteLLM's
|
|
556
|
+
[`model_prices_and_context_window.json`](https://raw.githubusercontent.com/BerriAI/litellm/f1f33f560f7a39e86f7a2e5b26b9fa032f9dcaba/model_prices_and_context_window.json)
|
|
557
|
+
(retrieved 2026-07-15), so any adapter whose resolved model slug is exactly a
|
|
558
|
+
card key gets a rate-computed cost with **zero configuration**.
|
|
559
|
+
4. **Tokens only** — if none of the above yields pricing, the run shows token
|
|
560
|
+
counts only. Neal never invents dollars.
|
|
561
|
+
|
|
562
|
+
Card lookup is **exact-match only**: the resolved model slug must be an exact
|
|
563
|
+
card key. Neal does not strip provider prefixes and does not fall back to the
|
|
564
|
+
basename after a `/`, so a slash-qualified slug (for example a local or gateway
|
|
565
|
+
slug like `local/gpt-5.5` or `azure/<deployment>`) is priced only if the card
|
|
566
|
+
lists that exact string; otherwise it stays tokens-only. LiteLLM lists many
|
|
567
|
+
`vendor/model` keys directly, so provider-qualified slugs are still priced when
|
|
568
|
+
the card carries that exact key. Set an explicit `pricing` override for slugs the
|
|
569
|
+
card does not key exactly.
|
|
570
|
+
|
|
571
|
+
**Base-tier only.** Card cost uses each model's published base /
|
|
572
|
+
standard-context per-token rates only. Neal does **not** apply long-context
|
|
573
|
+
surcharges, tiered, batch, or priority rates. A turn whose prompt crosses a
|
|
574
|
+
model's long-context threshold (for example GPT-5.6 Sol above 272K prompt tokens,
|
|
575
|
+
which upstream prices at 2× input / 1.5× output) is priced at the base tier and
|
|
576
|
+
is therefore an **underestimate** for that turn. Operators who need exact
|
|
577
|
+
long-context cost can pin explicit rates via the `pricing` override.
|
|
578
|
+
|
|
579
|
+
The `openai-codex` provider is now card-priced by its configured model (each
|
|
580
|
+
role is priced by the model it actually ran); there is no codex pricing config
|
|
581
|
+
block by design, so a codex role with no configured model (SDK default) stays
|
|
582
|
+
tokens-only.
|
|
583
|
+
|
|
584
|
+
The card is community-maintained list prices, and rate-computed cost is an
|
|
585
|
+
estimate. Rate-computed cost (from either the vendored card or a configured
|
|
586
|
+
override) is flagged with a footnote (`*`) in the retrospective's Provider Usage
|
|
587
|
+
table — "Cost estimated from published or configured rates, not reported by the
|
|
588
|
+
provider." — to distinguish it from cost a provider reports directly. Cached
|
|
589
|
+
input is billed once, at the cached rate.
|
|
527
590
|
|
|
528
591
|
`neal setup` offers `openai-compatible` for the reviewer role only and prints
|
|
529
592
|
guidance when the base URL, API key, or model is unresolved. `neal setup`
|
|
@@ -676,6 +739,60 @@ else `OPENAI_COMPATIBLE_MODEL`; one of these is required. Structured-advisor
|
|
|
676
739
|
rounds additionally honor a Neal-internal round-level model override first,
|
|
677
740
|
matching the `openai-compatible` adapter.
|
|
678
741
|
|
|
742
|
+
### Local Endpoints (Ollama, llama.cpp, vLLM)
|
|
743
|
+
|
|
744
|
+
A local server that speaks the OpenAI-compatible Chat Completions API — Ollama,
|
|
745
|
+
llama.cpp's `server`, vLLM, and similar — is just another endpoint for this
|
|
746
|
+
provider. It reuses the same `providers.openai_compatible` block documented
|
|
747
|
+
above (there is still no separate generic-agentic config surface), with the
|
|
748
|
+
same config-first, environment-fallback resolution rules from
|
|
749
|
+
[OpenAI-Compatible Endpoints](#openai-compatible-endpoints); only the
|
|
750
|
+
`base_url`, model slug, and auth variable change.
|
|
751
|
+
|
|
752
|
+
```yaml
|
|
753
|
+
providers:
|
|
754
|
+
openai_compatible:
|
|
755
|
+
base_url: http://localhost:11434/v1
|
|
756
|
+
api_key_env: OLLAMA_API_KEY # any non-empty placeholder for servers that ignore auth
|
|
757
|
+
default_model: qwen3-coder:30b
|
|
758
|
+
|
|
759
|
+
agent:
|
|
760
|
+
coder:
|
|
761
|
+
provider: openai-codex
|
|
762
|
+
model: null
|
|
763
|
+
reviewer:
|
|
764
|
+
provider: generic-agentic
|
|
765
|
+
model: null
|
|
766
|
+
```
|
|
767
|
+
|
|
768
|
+
Neal has no `api_key` config field: it resolves the key from the environment
|
|
769
|
+
variable named by `api_key_env` (default `OPENAI_COMPATIBLE_API_KEY`), and the
|
|
770
|
+
AI SDK requires that value to be non-empty even when the local server ignores
|
|
771
|
+
auth. So export a non-empty placeholder to the variable you named —
|
|
772
|
+
`export OLLAMA_API_KEY=ollama` — before starting a run. Use the fully qualified
|
|
773
|
+
local slug your server reports (Ollama tags such as `qwen3-coder:30b`), not a
|
|
774
|
+
bare family name.
|
|
775
|
+
|
|
776
|
+
Qualify a local model with [`neal compat`](compat.md)
|
|
777
|
+
(`neal compat --model <slug> --role all`) before trusting it with writer runs;
|
|
778
|
+
`--model` forces the candidate onto `generic-agentic`. Note the reference
|
|
779
|
+
caveat: `--model` defaults `--reference` to `openai-codex`, which needs a Codex
|
|
780
|
+
login, so a purely-local operator should pass
|
|
781
|
+
`--reference generic-agentic:<same-slug>` for a self-contained check — while
|
|
782
|
+
being aware that self-reference is a weaker qualification partner (a candidate
|
|
783
|
+
grading itself), so treat its PASS with more caution than a native reference
|
|
784
|
+
(see [compatible-models.md](compatible-models.md)).
|
|
785
|
+
|
|
786
|
+
Set realistic role expectations. Local models below the whitelist bar usually
|
|
787
|
+
fail the **coder** role on structured output — `structured_output` dominates the
|
|
788
|
+
coder-FAIL rows in [compatible-models.md](compatible-models.md) — so the config
|
|
789
|
+
above is the realistic starting point: keep the coder on a native provider
|
|
790
|
+
(`openai-codex` or `anthropic-claude`) and give the local `generic-agentic`
|
|
791
|
+
endpoint reviewer-only duty. That split only works if the local model calls
|
|
792
|
+
tools reliably; if it is chat-only (no tool calling), use `openai-compatible`
|
|
793
|
+
instead of `generic-agentic`, per the "Choosing a reviewer adapter for generic
|
|
794
|
+
endpoints" guidance above.
|
|
795
|
+
|
|
679
796
|
## Adding A Built-In Provider
|
|
680
797
|
|
|
681
798
|
To add another built-in provider:
|
package/docs/release.md
CHANGED
|
@@ -92,38 +92,42 @@ node scripts/verify-package.mjs
|
|
|
92
92
|
```
|
|
93
93
|
|
|
94
94
|
Review the dry-run result before any real publish. Run `Publish` again with
|
|
95
|
-
`dry_run: false` only when a
|
|
96
|
-
|
|
95
|
+
`dry_run: false` only when a release-preparation plan has explicitly authorized
|
|
96
|
+
the real publish.
|
|
97
97
|
|
|
98
|
-
|
|
98
|
+
The real-publish path is **staged**: the workflow runs `npm stage publish`,
|
|
99
|
+
which places the version in a staged, not-publicly-available state. A
|
|
100
|
+
maintainer then reviews and approves it with 2FA — `npm stage list`,
|
|
101
|
+
`npm stage view <stage-id>` / `npm stage download <stage-id>`, and
|
|
102
|
+
`npm stage approve <stage-id>` (or the npmjs.com UI). Nothing reaches `latest`
|
|
103
|
+
without that human approval, so a compromised workflow cannot ship directly.
|
|
99
104
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
105
|
+
## First-Publish History
|
|
106
|
+
|
|
107
|
+
`0.1.0` was published manually on 2026-07-12 with an interactive 2FA publish
|
|
108
|
+
from a maintainer terminal: npm trusted publishing cannot be configured for a
|
|
109
|
+
package that has never been published, so the first publish had to
|
|
110
|
+
authenticate directly. Every release after `0.1.0` goes through the `Publish`
|
|
111
|
+
workflow and the staged flow above.
|
|
106
112
|
|
|
107
113
|
## Trusted Publishing Setup
|
|
108
114
|
|
|
109
115
|
The publish workflow relies on trusted publishing/OIDC and intentionally has no
|
|
110
116
|
npm-token fallback. It grants `id-token: write` for OIDC, keeps repository
|
|
111
|
-
contents read-only, uses the `npm-publish` GitHub environment, and
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
filename `publish.yml`, environment `npm-publish`, and allowed action
|
|
126
|
-
`npm publish`.
|
|
117
|
+
contents read-only, uses the `npm-publish` GitHub environment, and runs
|
|
118
|
+
`npm stage publish` only on the real-publish path (staged publishing requires
|
|
119
|
+
npm >= 11.15.0; the workflow upgrades npm accordingly).
|
|
120
|
+
|
|
121
|
+
The npm-side configuration for `@navels/neal`:
|
|
122
|
+
|
|
123
|
+
- Publishing access: **Require two-factor authentication and disallow tokens**.
|
|
124
|
+
This blocks every traditional token permanently; trusted publishers are
|
|
125
|
+
unaffected because they use OIDC, and staged approvals always require a
|
|
126
|
+
maintainer's 2FA.
|
|
127
|
+
- Trusted publisher: owner/repo `navels/neal`, workflow filename
|
|
128
|
+
`publish.yml`, environment `npm-publish`, allowed action **`npm stage
|
|
129
|
+
publish` only** (stage-only; plain `npm publish` is deliberately not
|
|
130
|
+
granted).
|
|
127
131
|
|
|
128
132
|
## Release Boundaries
|
|
129
133
|
|