@navels/neal 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -63,6 +63,8 @@ async function writeRunEvent(event, options) {
63
63
  await logger.event('provider.turn_completed', {
64
64
  ...commonEventData(event),
65
65
  ...(event.usage !== undefined ? { usage: event.usage } : {}),
66
+ ...(event.costUsd !== undefined ? { costUsd: event.costUsd } : {}),
67
+ ...(event.costSource !== undefined ? { costSource: event.costSource } : {}),
66
68
  });
67
69
  break;
68
70
  case 'tool_started':
@@ -121,6 +123,8 @@ async function writeRunEvent(event, options) {
121
123
  await logger.event('provider.usage_reported', {
122
124
  ...commonEventData(event),
123
125
  usage: event.usage,
126
+ ...(event.costUsd !== undefined ? { costUsd: event.costUsd } : {}),
127
+ ...(event.costSource !== undefined ? { costSource: event.costSource } : {}),
124
128
  });
125
129
  break;
126
130
  case 'provider_error':
@@ -28,6 +28,26 @@ function getArchivedRetrospectivePath(state, kind) {
28
28
  const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
29
29
  return join(state.runDir, `RETROSPECTIVE-final${suffix}.md`);
30
30
  }
31
+ function getCurrentRunMetricsPath(runDir) {
32
+ return join(runDir, 'RUN_METRICS.json');
33
+ }
34
+ // Mirrors getArchivedRetrospectivePath's scope/kind/commit naming so the
35
+ // machine-readable metrics archive lines up with the retrospective archive.
36
+ function getArchivedRunMetricsPath(state, kind) {
37
+ const scopeLabel = getCurrentScopeLabel(state);
38
+ if (kind === 'scope_accepted') {
39
+ const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
40
+ return join(state.runDir, `RUN_METRICS-scope-${scopeLabel}${suffix}.json`);
41
+ }
42
+ if (kind === 'blocked') {
43
+ return join(state.runDir, `RUN_METRICS-blocked-scope-${scopeLabel}.json`);
44
+ }
45
+ if (kind === 'failed') {
46
+ return join(state.runDir, `RUN_METRICS-failed-scope-${scopeLabel}.json`);
47
+ }
48
+ const suffix = state.finalCommit ? `-${state.finalCommit}` : '';
49
+ return join(state.runDir, `RUN_METRICS-final${suffix}.json`);
50
+ }
31
51
  async function loadRunEvents(runDir) {
32
52
  try {
33
53
  const content = await readFile(getEventsPath(runDir), 'utf8');
@@ -306,7 +326,8 @@ async function renderRetrospective(state, kind) {
306
326
  const changedFiles = await summarizeChangedFiles(state);
307
327
  const verificationSummary = summarizeVerification(scopeEvents);
308
328
  const metricsEvents = kind === 'done' ? events : scopeEvents;
309
- const runMetricsSummary = renderRunMetricsMarkdown(summarizeRunMetrics(metricsEvents));
329
+ const metrics = summarizeRunMetrics(metricsEvents);
330
+ const runMetricsSummary = renderRunMetricsMarkdown(metrics);
310
331
  const assessment = buildAssessment(state, scopeEvents);
311
332
  const narrativeRetrospective = buildNarrativeRetrospective({
312
333
  state,
@@ -320,7 +341,7 @@ async function renderRetrospective(state, kind) {
320
341
  const derivedPlan = getDerivedPlanView(state);
321
342
  const showDerivedPlanContext = Boolean(derivedPlan);
322
343
  const parentScopeLabel = derivedPlan?.executing ? getParentScopeLabel(state) : null;
323
- return [
344
+ const content = [
324
345
  `# Neal Retrospective`,
325
346
  '',
326
347
  `## Outcome`,
@@ -377,15 +398,23 @@ async function renderRetrospective(state, kind) {
377
398
  ...renderInteractiveBlockedRecoveryHistoryLines(state.interactiveBlockedRecoveryHistory),
378
399
  '',
379
400
  ].join('\n');
401
+ return { content, metrics };
380
402
  }
381
403
  async function writeRetrospectiveFile(path, content) {
382
404
  await writeTextAtomic(path, content);
383
405
  }
406
+ async function writeRunMetricsFile(path, metrics) {
407
+ await writeTextAtomic(path, `${JSON.stringify(metrics, null, 2)}\n`);
408
+ }
384
409
  export async function writeCheckpointRetrospective(state, kind) {
385
- const content = await renderRetrospective(state, kind);
410
+ const { content, metrics } = await renderRetrospective(state, kind);
386
411
  const currentPath = getCurrentRetrospectivePath(state.runDir);
387
412
  const archivedPath = getArchivedRetrospectivePath(state, kind);
388
413
  await writeRetrospectiveFile(currentPath, content);
389
414
  await writeRetrospectiveFile(archivedPath, content);
390
- return { currentPath, archivedPath };
415
+ const currentMetricsPath = getCurrentRunMetricsPath(state.runDir);
416
+ const archivedMetricsPath = getArchivedRunMetricsPath(state, kind);
417
+ await writeRunMetricsFile(currentMetricsPath, metrics);
418
+ await writeRunMetricsFile(archivedMetricsPath, metrics);
419
+ return { currentPath, archivedPath, currentMetricsPath, archivedMetricsPath };
391
420
  }
@@ -48,6 +48,21 @@ function addUsage(target, value) {
48
48
  function hasUsage(usage) {
49
49
  return Object.values(usage).some((value) => value > 0);
50
50
  }
51
+ // Accumulate cost from one counted event into the provider bucket. Presence is
52
+ // tracked separately from value: costUsd stays null until an event carries a
53
+ // numeric costUsd, then it holds the running sum. A single provider/role bucket
54
+ // has one source; if both appear, 'provider' wins.
55
+ function accumulateCost(bucket, data) {
56
+ const raw = data.costUsd;
57
+ if (typeof raw !== 'number' || !Number.isFinite(raw)) {
58
+ return;
59
+ }
60
+ bucket.costUsd = (bucket.costUsd ?? 0) + raw;
61
+ const source = data.costSource;
62
+ if (source === 'provider' || source === 'rate') {
63
+ bucket.costSource = bucket.costSource === 'provider' || source === 'provider' ? 'provider' : 'rate';
64
+ }
65
+ }
51
66
  function cloneEmptyUsage() {
52
67
  return { ...EMPTY_USAGE };
53
68
  }
@@ -210,10 +225,13 @@ export function summarizeRunMetrics(events) {
210
225
  label: identity.label,
211
226
  turns: 0,
212
227
  usage: cloneEmptyUsage(),
228
+ costUsd: null,
229
+ costSource: null,
213
230
  };
214
231
  existing.turns += 1;
215
232
  if (!hasDedicatedUsageEvents) {
216
233
  addUsage(existing.usage, data.usage);
234
+ accumulateCost(existing, data);
217
235
  }
218
236
  providers.set(identity.key, existing);
219
237
  }
@@ -226,8 +244,11 @@ export function summarizeRunMetrics(events) {
226
244
  label: identity.label,
227
245
  turns: 0,
228
246
  usage: cloneEmptyUsage(),
247
+ costUsd: null,
248
+ costSource: null,
229
249
  };
230
250
  addUsage(existing.usage, data.usage);
251
+ accumulateCost(existing, data);
231
252
  providers.set(identity.key, existing);
232
253
  }
233
254
  else if (event.type === 'provider.command_completed') {
@@ -276,6 +297,25 @@ export function summarizeRunMetrics(events) {
276
297
  phase.durationMs = start !== null && end !== null && end >= start ? end - start : null;
277
298
  }
278
299
  const publicPhases = phases.map(({ startIndex: _startIndex, endIndex: _endIndex, ...phase }) => phase);
300
+ const providerSummaries = [...providers.values()].sort((left, right) => {
301
+ const turns = right.turns - left.turns;
302
+ if (turns !== 0) {
303
+ return turns;
304
+ }
305
+ return usageSortValue(right) - usageSortValue(left);
306
+ });
307
+ // Coverage is computed only over usage-bearing buckets so a partial subtotal
308
+ // never masquerades as a complete run total.
309
+ const usageBearing = providerSummaries.filter((provider) => hasUsage(provider.usage));
310
+ const pricedUsageBearing = usageBearing.filter((provider) => provider.costUsd !== null);
311
+ const costCoverage = pricedUsageBearing.length === 0
312
+ ? 'none'
313
+ : pricedUsageBearing.length === usageBearing.length
314
+ ? 'complete'
315
+ : 'partial';
316
+ const totalCostUsd = costCoverage === 'none'
317
+ ? null
318
+ : providerSummaries.reduce((sum, provider) => (provider.costUsd !== null ? sum + provider.costUsd : sum), 0);
279
319
  return {
280
320
  observedStartedAt: firstTimestamp,
281
321
  observedCompletedAt: lastTimestamp,
@@ -288,13 +328,9 @@ export function summarizeRunMetrics(events) {
288
328
  toolEventCount,
289
329
  fileChangeEventCount,
290
330
  phases: publicPhases,
291
- providers: [...providers.values()].sort((left, right) => {
292
- const turns = right.turns - left.turns;
293
- if (turns !== 0) {
294
- return turns;
295
- }
296
- return usageSortValue(right) - usageSortValue(left);
297
- }),
331
+ providers: providerSummaries,
332
+ totalCostUsd,
333
+ costCoverage,
298
334
  };
299
335
  }
300
336
  function formatDuration(value) {
@@ -327,6 +363,31 @@ function formatProviderName(summary) {
327
363
  function usageCell(value) {
328
364
  return value > 0 ? formatNumber(value) : '-';
329
365
  }
366
+ function formatCostAmount(value) {
367
+ return `$${value.toFixed(4)}`;
368
+ }
369
+ // Cost cell for the Provider Usage table: '-' when the bucket reported no cost,
370
+ // otherwise the fixed-precision USD amount with a footnote marker on
371
+ // rate-computed cells.
372
+ function formatCost(value, source) {
373
+ if (value === null) {
374
+ return '-';
375
+ }
376
+ return source === 'rate' ? `${formatCostAmount(value)}*` : formatCostAmount(value);
377
+ }
378
+ // The top-summary cost line, labeled by coverage so a partial subtotal is never
379
+ // presented as a complete run total.
380
+ function formatEstimatedCostLine(metrics) {
381
+ if (metrics.costCoverage === 'none' || metrics.totalCostUsd === null) {
382
+ return '- Estimated cost: unknown';
383
+ }
384
+ if (metrics.costCoverage === 'partial') {
385
+ const usageBearing = metrics.providers.filter((provider) => hasUsage(provider.usage));
386
+ const priced = usageBearing.filter((provider) => provider.costUsd !== null);
387
+ return `- Estimated cost (partial — ${priced.length} of ${usageBearing.length} priced providers): ${formatCostAmount(metrics.totalCostUsd)}`;
388
+ }
389
+ return `- Estimated cost: ${formatCostAmount(metrics.totalCostUsd)}`;
390
+ }
330
391
  export function renderRunMetricsMarkdown(metrics) {
331
392
  const commandSummary = metrics.resolvedNonZeroCommandCount > 0
332
393
  ? `- Commands: ${metrics.commandCount} total, ${metrics.unresolvedNonZeroCommandCount} unresolved non-zero or failed, ${metrics.resolvedNonZeroCommandCount} resolved by later passing rerun`
@@ -337,16 +398,20 @@ export function renderRunMetricsMarkdown(metrics) {
337
398
  commandSummary,
338
399
  `- Tool events: ${metrics.toolEventCount}`,
339
400
  `- File change events: ${metrics.fileChangeEventCount}`,
401
+ formatEstimatedCostLine(metrics),
340
402
  ];
341
403
  if (metrics.phases.length > 0) {
342
404
  lines.push('', '### Phase Timing', '', '| Phase | Duration | Provider turns | Commands | Non-zero commands | Tool events | File changes |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', ...metrics.phases.map((phase) => `| ${formatPhaseName(phase.phase)} | ${formatPhaseDuration(phase)} | ${phase.providerTurns} | ${phase.commandCount} | ${phase.nonZeroCommandCount} | ${phase.toolEventCount} | ${phase.fileChangeEventCount} |`));
343
405
  }
344
406
  const providersWithUsage = metrics.providers.filter((provider) => provider.turns > 0 || hasUsage(provider.usage));
345
407
  if (providersWithUsage.length > 0) {
346
- lines.push('', '### Provider Usage', '', '| Provider / role | Turns | Input | Cached input | Cache created | Cache read | Output | Reasoning output | Total |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...providersWithUsage.map((provider) => {
408
+ lines.push('', '### Provider Usage', '', '| Provider / role | Turns | Input | Cached input | Cache created | Cache read | Output | Reasoning output | Total | Cost |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...providersWithUsage.map((provider) => {
347
409
  const usage = provider.usage;
348
- return `| ${formatProviderName(provider)} | ${provider.turns} | ${usageCell(usage.inputTokens)} | ${usageCell(usage.cachedInputTokens)} | ${usageCell(usage.cacheCreationInputTokens)} | ${usageCell(usage.cacheReadInputTokens)} | ${usageCell(usage.outputTokens)} | ${usageCell(usage.reasoningOutputTokens)} | ${usageCell(usage.totalTokens)} |`;
410
+ return `| ${formatProviderName(provider)} | ${provider.turns} | ${usageCell(usage.inputTokens)} | ${usageCell(usage.cachedInputTokens)} | ${usageCell(usage.cacheCreationInputTokens)} | ${usageCell(usage.cacheReadInputTokens)} | ${usageCell(usage.outputTokens)} | ${usageCell(usage.reasoningOutputTokens)} | ${usageCell(usage.totalTokens)} | ${formatCost(provider.costUsd, provider.costSource)} |`;
349
411
  }));
412
+ if (providersWithUsage.some((provider) => provider.costUsd !== null && provider.costSource === 'rate')) {
413
+ lines.push('', '\\* Cost estimated from published or configured rates, not reported by the provider.');
414
+ }
350
415
  }
351
416
  else {
352
417
  lines.push('', '### Provider Usage', '', '- No provider usage events recorded.');
@@ -24,6 +24,17 @@
24
24
 
25
25
  Sorted cheapest first ($/Mtok in·out, from the live OpenRouter catalog).
26
26
 
27
+ > **Drift observed (2026-07-14):** `google/gemma-4-26b-a4b-it` — an all-roles PASS on
28
+ > 2026-06-18 — failed the coder role 3/3 with `structured_output` in the CI smoke
29
+ > (self-reference configuration). Slugs rot as providers re-route or re-quantize;
30
+ > treat its row below as stale until re-qualified against the codex reference.
31
+ >
32
+ > **Self-reference caveat (2026-07-14):** `inclusionai/ling-2.6-1t` repeatedly failed the
33
+ > reviewer broken-diff cell (2/3 attempts) in the CI smoke when serving as **its own
34
+ > partner**. Its codex-referenced row below stands — this is a partner-duty weakness,
35
+ > the same class the reference investigation documents. The smoke pin is
36
+ > `minimax/minimax-m2.5`, which held up under self-reference.
37
+
27
38
  | Model | Coder | Reviewer | Planner | $/Mtok | ctx |
28
39
  |---|---|---|---|---|---|
29
40
  | `google/gemma-4-26b-a4b-it` | 3/3 | 6/6 | 1/1 | 0.06·0.33 | 262k |
@@ -0,0 +1,124 @@
1
+ # GitHub Issue Pipeline
2
+
3
+ Assign a GitHub issue to neal and get a reviewed pull request back. A maintainer
4
+ applies a trigger label to an issue; a GitHub Actions runner turns the issue
5
+ into a neal plan, runs neal's planner/coder/reviewer loop to completion, and
6
+ opens a pull request assigned to a human — or reports back on the issue if neal
7
+ blocks. Humans own merge; the pipeline never pushes to a protected branch.
8
+
9
+ This is a self-hosting use of neal itself: neal already *is* the middle of the
10
+ loop (cross-model review, deterministic gating, bounded recovery), so the
11
+ pipeline is a thin trigger plus a thin publish around `neal run`.
12
+
13
+ ## What it does
14
+
15
+ ```
16
+ maintainer applies the trigger label to an issue
17
+ │ (gate: the labeler must have write access — the issue author is irrelevant)
18
+
19
+ issue title + body ──▶ seed .neal/PLAN.md ──▶ neal run --unattended
20
+ │ (Opus 4.8 writes, sol reviews each diff)
21
+ ├─ exit 0 ──▶ push neal/issue-<n>, open a PR, assign the maintainer
22
+ ├─ exit 2 ──▶ comment the blocker on the issue, keep the branch
23
+ └─ exit 3 ──▶ comment the failure on the issue, open no PR
24
+ ```
25
+
26
+ The plan is seeded inside `.neal/` — wrapper-owned, gitignored territory — so
27
+ it can never collide with a tracked file or appear in review diffs, and it is
28
+ never committed. The refined plan travels with the outcome instead: the PR
29
+ body (or the blocked/failed issue comment) embeds it in a collapsed details
30
+ block, which outlives the 3-day `.neal/` artifact retention.
31
+
32
+ The outcome also carries a cost ledger — per-provider turns, tokens, and
33
+ dollars aggregated from the run's `RUN_METRICS-final*.json` files. Claude
34
+ cost is provider-reported; reviewer cost is rate-computed from neal's
35
+ vendored published rate card (see `docs/providers.md`) and marked as such.
36
+
37
+ ## Running an issue locally
38
+
39
+ `scripts/neal-issue-local.sh <issue-number> [plan-doc]` is the pipeline's
40
+ local twin for subscription billing: the coder rides the Claude CLI login and
41
+ the reviewer the codex CLI login (per `~/.neal/config.yml`) instead of the
42
+ API keys CI uses. It shares the plan seed and PR-body construction with
43
+ `pipeline/run.sh` through `pipeline/lib.sh`, so a local run produces the same
44
+ branch name, the same PR format, and the same embedded plan and cost ledger.
45
+ The optional second argument substitutes a pre-authored plan (for example a
46
+ `multi_scope` plan spanning several issues) for the seeded `one_shot` one.
47
+ The script unsets `OPENAI_API_KEY`/`ANTHROPIC_API_KEY` for the run so a stray
48
+ exported key cannot silently switch billing from subscription to API.
49
+
50
+ Every run executes in its own git worktree under `<repo>-worktrees/issue-<n>`,
51
+ branched from `origin/main`. Runs on different issues can proceed in parallel
52
+ (git enforces one worktree per branch), and the main checkout stays free for
53
+ your own work throughout. On success the worktree is removed after the push —
54
+ the branch lives on the PR. On blocked/failed outcomes the worktree is kept:
55
+ it is the resume state (`cd` in and `neal resume`). The practical concurrency
56
+ cap is your subscriptions, not git: parallel runs share your Claude and
57
+ ChatGPT rate limits.
58
+
59
+ - **Coder:** Claude Opus 4.8 (`anthropic-claude` adapter, `ANTHROPIC_API_KEY`).
60
+ - **Reviewer:** GPT-5.6-sol via the `openai-compatible` adapter pointed at the
61
+ OpenAI API (`OPENAI_API_KEY`). The reviewer judges neal-inlined diff/context;
62
+ it has no repository tools, which keeps the runner setup to two env vars and
63
+ no extra CLI.
64
+
65
+ ## Setup
66
+
67
+ One-time, by a maintainer. The workflow no-ops safely until this is done.
68
+
69
+ 1. **Secrets** (repo → Settings → Secrets → Actions, on the `neal-pipeline`
70
+ environment): `ANTHROPIC_API_KEY` and `OPENAI_API_KEY`. Use short-lived,
71
+ minimally-scoped keys and rotate them (see Security below).
72
+ 2. **Environment**: create a `neal-pipeline` environment (Settings →
73
+ Environments). Optionally add required reviewers so each run needs a manual
74
+ click before the secrets are exposed — recommended for a public repo.
75
+ 3. **Reviewer model**: set the repo variable `NEAL_REVIEWER_MODEL` to the sol
76
+ model slug your OpenAI key serves. Before the first run, qualify it:
77
+ `neal compat --model <slug> --role reviewer --reference anthropic-claude`
78
+ (locally, with the same key). If sol is not reachable over the OpenAI
79
+ Chat Completions API, point `NEAL_REVIEWER_BASE_URL`/`NEAL_REVIEWER_MODEL` at
80
+ an endpoint that serves it, or switch the reviewer to the `openai-codex`
81
+ adapter in `pipeline/config.yml`.
82
+ 4. **Trigger label**: create a label (default `neal:go`; override with the repo
83
+ variable `NEAL_TRIGGER_LABEL`). Optionally set `NEAL_READY_LABEL` (e.g.
84
+ `ready-for-human`) to tag the PRs neal opens.
85
+ 5. **Branch protection**: protect `main` (require PR + review). The pipeline can
86
+ push `neal/issue-*` branches and open PRs, but must never be able to merge or
87
+ push to `main`.
88
+
89
+ ## Security model
90
+
91
+ This runs on a public repository, so the trust boundary is "arbitrary internet
92
+ text becomes instructions to an agent with push access." The controls:
93
+
94
+ - **Maintainer-gated trigger.** The gate checks the *labeler's* repo permission
95
+ (`write`/`maintain`/`admin`), not the issue author. An untrusted issue cannot
96
+ start a run — only a maintainer applying the label can.
97
+ - **Untrusted issue input.** The issue title/body are passed as environment
98
+ variables (never interpolated into a shell command) and written to the seed
99
+ plan with `printf '%s'`, so they cannot inject shell commands. neal then
100
+ treats the seed as plan input, refined through the planner/reviewer loop.
101
+ - **Least privilege.** The workflow token can push `neal/issue-*` branches, open
102
+ PRs, and comment — nothing else. Branch protection keeps the human merge gate.
103
+ - **Boundaries in the plan.** The seed forbids edits to `.github/**` /
104
+ `pipeline/**` and forbids reading or printing secrets — defense-in-depth on
105
+ top of the token scope.
106
+ - **Secret exposure caveat.** The coder runs with shell in the runner, where
107
+ `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` live in the environment. A
108
+ prompt-injected coder could try to exfiltrate them; the maintainer gate is
109
+ what makes that acceptable (only a maintainer's labeled issue ever runs). Use
110
+ short-lived, minimally-scoped keys, the `neal-pipeline` environment's approval
111
+ gate, and rotate on any suspicion.
112
+
113
+ ## v1 limitations
114
+
115
+ - **Seed shape is `one_shot`.** Best for well-scoped issues ("fix X", "add Y").
116
+ For large multi-part work neal's coder will split the scope into a sub-plan;
117
+ if it can't be done in scope, neal blocks and comments asking for a smaller
118
+ issue. Write issues with a clear, bounded objective for best results.
119
+ - **Reviewer reads inlined context, not the repo.** The `openai-compatible`
120
+ reviewer judges the diff neal inlines, not the working tree. Switch to an
121
+ `openai-codex` reviewer (OS-sandboxed, repo-reading) once sol is confirmed on
122
+ that adapter's API path, if repo-reading review is wanted.
123
+ - **No comment-driven iteration yet.** On a blocked run you re-apply the label to
124
+ retry; a maintainer-comment → `neal resume --message` loop is a v2 addition.
@@ -4,37 +4,48 @@ neal's behavior is defined in large part by the agent SDKs it drives, so
4
4
  dependency updates are a first-class concern, not routine hygiene. This document
5
5
  is the policy.
6
6
 
7
- ## Two dependency tiers
7
+ ## Three dependency tiers
8
8
 
9
9
  | Tier | Packages | Pinning | Update posture |
10
10
  | --- | --- | --- | --- |
11
- | **Behavior-defining** | `@openai/codex-sdk`, `@anthropic-ai/claude-agent-sdk`, `@anthropic-ai/sdk`, `ai`, `@ai-sdk/openai-compatible`, `zod` | **exact** (enforced by `scripts/validate-release.mjs`) | **deliberate** — qualify before adopting; never auto-merge |
11
+ | **Native SDKs** | `@openai/codex-sdk`, `@anthropic-ai/claude-agent-sdk` | **exact** (enforced by `scripts/validate-release.mjs`) | **deliberate** — behaviorally qualified on a subscription-authenticated machine via `scripts/qualify-sdk.sh`; never auto-merge |
12
+ | **AI-SDK tier** | `ai`, `@ai-sdk/openai-compatible`, `zod` | **exact** (enforced by `scripts/validate-release.mjs`) | minor/patch **auto-merge** once CI **and the live smoke** are green (3-day soak); majors stay manual |
12
13
  | **Utility** | `dotenv`, `yaml`, `@types/node`, `tsx`, `typescript` | caret OK | routine — auto-merge on green CI after a soak period |
13
14
 
14
15
  The asymmetry exists because an agentic-SDK bump can change tool-calling,
15
16
  structured output, or sandbox behavior — i.e. break neal's loop **without**
16
- breaking compilation. Utility deps cannot.
17
+ breaking compilation. Utility deps cannot. The native/AI-SDK split exists
18
+ because CI can behaviorally exercise the AI-SDK tier (the smoke drives it
19
+ live through `generic-agentic`) but not the native adapters, whose auth is
20
+ subscription-based and lives only on a maintainer's machine.
17
21
 
18
22
  ## The update flow
19
23
 
20
- 1. **Detect.** [Renovate](../renovate.json) opens PRs weekly: utility deps grouped
21
- (auto-merge), agentic SDKs one-per-PR (labelled `agentic-sdk` /
22
- `needs-qualification`, auto-merge **off**, pinned exact).
24
+ 1. **Detect.** [Renovate](../renovate.json) opens PRs weekly: utility deps
25
+ grouped, AI-SDK and native SDKs one-per-PR, pinned exact, labelled
26
+ `agentic-sdk` (native and AI-SDK majors additionally `needs-qualification`).
23
27
  2. **Verify (automatic).** CI (`.github/workflows/ci.yml`) runs typecheck + unit
24
- tests + package verification — catches **API-shape / contract** breaks.
25
- 3. **Verify (behavioral).**
26
- - The **AI-SDK tier** (`ai`, `@ai-sdk/openai-compatible`, `zod`) is exercised
27
- automatically by the live smoke (`.github/workflows/smoke.yml`): a real
28
- `neal compat` run against a cheap OpenRouter model through `generic-agentic`.
29
- - The **native tier** (`@openai/codex-sdk`, `@anthropic-ai/*`) cannot be smoked
30
- in CI (subscription auth isn't available there), so qualify it **locally**
31
- before merging:
32
- ```
33
- neal compat --model <a-known-good-slug> --role all
34
- ```
35
- and ideally a one-fixture end-to-end run.
28
+ tests + package verification — catches **API-shape / contract** breaks. The
29
+ live smoke (`.github/workflows/smoke.yml`) runs on every package.json /
30
+ lockfile PR: a real `neal compat` run against a cheap OpenRouter model
31
+ through `generic-agentic`, catching **behavioral** breaks in the AI-SDK
32
+ tier. AI-SDK minor/patch PRs auto-merge when both are green.
33
+ **The smoke requires the `OPENROUTER_API_KEY` repo secret** — without it
34
+ the smoke skips (green) and the AI-SDK auto-merge gate is compile-only.
35
+ 3. **Verify (behavioral, native tier).** `@openai/codex-sdk` and
36
+ `@anthropic-ai/claude-agent-sdk` cannot be smoked in CI, so qualify them
37
+ from any checkout with authenticated Claude/Codex CLIs:
38
+ ```
39
+ scripts/qualify-sdk.sh <pr-number>
40
+ ```
41
+ It runs the full suite plus a live `neal compat --role all` pass-through on
42
+ the bumped adapter (in a throwaway worktree, with roles and models pinned
43
+ explicitly so nothing leaks from `~/.neal/config.yml`), posts the compat
44
+ matrix to the PR, and approves on PASS (`--merge` also squash-merges).
36
45
  4. **Adopt.** Merge, bump neal's version, add a CHANGELOG entry noting the bump +
37
- any behavior change, and cut a release via the existing workflow.
46
+ any behavior change, and cut a release via the existing workflow. Urgent
47
+ bumps (a fix neal needs immediately) may skip the Renovate soak with a
48
+ manual PR — qualify them the same way.
38
49
 
39
50
  ## Versioning
40
51
 
package/docs/providers.md CHANGED
@@ -524,6 +524,69 @@ Settings resolve config-first with environment fallbacks:
524
524
  One of these is required.
525
525
  - `headers`: optional string-to-string map of extra HTTP headers (useful for
526
526
  OpenRouter attribution headers).
527
+ - `pricing`: an **optional override** for per-million-token rates. It is no
528
+ longer required to get a dollar cost (see "Cost pricing" below); set it only
529
+ for models the vendored rate card does not key exactly, or to pin different
530
+ rates. All three rates are required when the block is present (a partial block
531
+ is a configuration error); rates are in USD per one million tokens:
532
+
533
+ ```yaml
534
+ providers:
535
+ openai_compatible:
536
+ base_url: https://api.deepseek.com
537
+ api_key_env: DEEPSEEK_API_KEY
538
+ default_model: deepseek-chat
539
+ pricing:
540
+ input_per_million: 0.27
541
+ cached_input_per_million: 0.07
542
+ output_per_million: 1.10
543
+ ```
544
+
545
+ #### Cost pricing
546
+
547
+ Neal resolves each run's dollar cost per provider/role bucket in this order:
548
+
549
+ 1. **Provider-reported cost** — the Claude adapter passes through the provider's
550
+ own `total_cost_usd`; it always wins when present.
551
+ 2. **Configured `pricing` override** — the
552
+ `providers.openai_compatible.pricing` block above (shared by the
553
+ `openai-compatible` and `generic-agentic` providers).
554
+ 3. **Vendored published rate card** — the default rate source, keyed by model
555
+ slug. Neal ships a trimmed copy of LiteLLM's
556
+ [`model_prices_and_context_window.json`](https://raw.githubusercontent.com/BerriAI/litellm/f1f33f560f7a39e86f7a2e5b26b9fa032f9dcaba/model_prices_and_context_window.json)
557
+ (retrieved 2026-07-15), so any adapter whose resolved model slug is exactly a
558
+ card key gets a rate-computed cost with **zero configuration**.
559
+ 4. **Tokens only** — if none of the above yields pricing, the run shows token
560
+ counts only. Neal never invents dollars.
561
+
562
+ Card lookup is **exact-match only**: the resolved model slug must be an exact
563
+ card key. Neal does not strip provider prefixes and does not fall back to the
564
+ basename after a `/`, so a slash-qualified slug (for example a local or gateway
565
+ slug like `local/gpt-5.5` or `azure/<deployment>`) is priced only if the card
566
+ lists that exact string; otherwise it stays tokens-only. LiteLLM lists many
567
+ `vendor/model` keys directly, so provider-qualified slugs are still priced when
568
+ the card carries that exact key. Set an explicit `pricing` override for slugs the
569
+ card does not key exactly.
570
+
571
+ **Base-tier only.** Card cost uses each model's published base /
572
+ standard-context per-token rates only. Neal does **not** apply long-context
573
+ surcharges, tiered, batch, or priority rates. A turn whose prompt crosses a
574
+ model's long-context threshold (for example GPT-5.6 Sol above 272K prompt tokens,
575
+ which upstream prices at 2× input / 1.5× output) is priced at the base tier and
576
+ is therefore an **underestimate** for that turn. Operators who need exact
577
+ long-context cost can pin explicit rates via the `pricing` override.
578
+
579
+ The `openai-codex` provider is now card-priced by its configured model (each
580
+ role is priced by the model it actually ran); there is no codex pricing config
581
+ block by design, so a codex role with no configured model (SDK default) stays
582
+ tokens-only.
583
+
584
+ The card is community-maintained list prices, and rate-computed cost is an
585
+ estimate. Rate-computed cost (from either the vendored card or a configured
586
+ override) is flagged with a footnote (`*`) in the retrospective's Provider Usage
587
+ table — "Cost estimated from published or configured rates, not reported by the
588
+ provider." — to distinguish it from cost a provider reports directly. Cached
589
+ input is billed once, at the cached rate.
527
590
 
528
591
  `neal setup` offers `openai-compatible` for the reviewer role only and prints
529
592
  guidance when the base URL, API key, or model is unresolved. `neal setup`
@@ -676,6 +739,60 @@ else `OPENAI_COMPATIBLE_MODEL`; one of these is required. Structured-advisor
676
739
  rounds additionally honor a Neal-internal round-level model override first,
677
740
  matching the `openai-compatible` adapter.
678
741
 
742
+ ### Local Endpoints (Ollama, llama.cpp, vLLM)
743
+
744
+ A local server that speaks the OpenAI-compatible Chat Completions API — Ollama,
745
+ llama.cpp's `server`, vLLM, and similar — is just another endpoint for this
746
+ provider. It reuses the same `providers.openai_compatible` block documented
747
+ above (there is still no separate generic-agentic config surface), with the
748
+ same config-first, environment-fallback resolution rules from
749
+ [OpenAI-Compatible Endpoints](#openai-compatible-endpoints); only the
750
+ `base_url`, model slug, and auth variable change.
751
+
752
+ ```yaml
753
+ providers:
754
+ openai_compatible:
755
+ base_url: http://localhost:11434/v1
756
+ api_key_env: OLLAMA_API_KEY # any non-empty placeholder for servers that ignore auth
757
+ default_model: qwen3-coder:30b
758
+
759
+ agent:
760
+ coder:
761
+ provider: openai-codex
762
+ model: null
763
+ reviewer:
764
+ provider: generic-agentic
765
+ model: null
766
+ ```
767
+
768
+ Neal has no `api_key` config field: it resolves the key from the environment
769
+ variable named by `api_key_env` (default `OPENAI_COMPATIBLE_API_KEY`), and the
770
+ AI SDK requires that value to be non-empty even when the local server ignores
771
+ auth. So export a non-empty placeholder to the variable you named —
772
+ `export OLLAMA_API_KEY=ollama` — before starting a run. Use the fully qualified
773
+ local slug your server reports (Ollama tags such as `qwen3-coder:30b`), not a
774
+ bare family name.
775
+
776
+ Qualify a local model with [`neal compat`](compat.md)
777
+ (`neal compat --model <slug> --role all`) before trusting it with writer runs;
778
+ `--model` forces the candidate onto `generic-agentic`. Note the reference
779
+ caveat: `--model` defaults `--reference` to `openai-codex`, which needs a Codex
780
+ login, so a purely-local operator should pass
781
+ `--reference generic-agentic:<same-slug>` for a self-contained check — while
782
+ being aware that self-reference is a weaker qualification partner (a candidate
783
+ grading itself), so treat its PASS with more caution than a native reference
784
+ (see [compatible-models.md](compatible-models.md)).
785
+
786
+ Set realistic role expectations. Local models below the whitelist bar usually
787
+ fail the **coder** role on structured output — `structured_output` dominates the
788
+ coder-FAIL rows in [compatible-models.md](compatible-models.md) — so the config
789
+ above is the realistic starting point: keep the coder on a native provider
790
+ (`openai-codex` or `anthropic-claude`) and give the local `generic-agentic`
791
+ endpoint reviewer-only duty. That split only works if the local model calls
792
+ tools reliably; if it is chat-only (no tool calling), use `openai-compatible`
793
+ instead of `generic-agentic`, per the "Choosing a reviewer adapter for generic
794
+ endpoints" guidance above.
795
+
679
796
  ## Adding A Built-In Provider
680
797
 
681
798
  To add another built-in provider:
package/docs/release.md CHANGED
@@ -92,38 +92,42 @@ node scripts/verify-package.mjs
92
92
  ```
93
93
 
94
94
  Review the dry-run result before any real publish. Run `Publish` again with
95
- `dry_run: false` only when a first-publish or later release-preparation plan has
96
- explicitly authorized the real publish.
95
+ `dry_run: false` only when a release-preparation plan has explicitly authorized
96
+ the real publish.
97
97
 
98
- ## First-Publish Boundary
98
+ The real-publish path is **staged**: the workflow runs `npm stage publish`,
99
+ which places the version in a staged, not-publicly-available state. A
100
+ maintainer then reviews and approves it with 2FA — `npm stage list`,
101
+ `npm stage view <stage-id>` / `npm stage download <stage-id>`, and
102
+ `npm stage approve <stage-id>` (or the npmjs.com UI). Nothing reaches `latest`
103
+ without that human approval, so a compromised workflow cannot ship directly.
99
104
 
100
- `@navels/neal` is not yet published to the npm registry. Local validation and
101
- the workflow run dry-run release checks, including `npm publish --dry-run
102
- --access public`. A real public publish (`npm publish --access public`) happens
103
- only on the `dry_run: false` path, under the configured npm trusted publishing
104
- and an explicit authorization from a first-publish or later release-preparation
105
- plan.
105
+ ## First-Publish History
106
+
107
+ `0.1.0` was published manually on 2026-07-12 with an interactive 2FA publish
108
+ from a maintainer terminal: npm trusted publishing cannot be configured for a
109
+ package that has never been published, so the first publish had to
110
+ authenticate directly. Every release after `0.1.0` goes through the `Publish`
111
+ workflow and the staged flow above.
106
112
 
107
113
  ## Trusted Publishing Setup
108
114
 
109
115
  The publish workflow relies on trusted publishing/OIDC and intentionally has no
110
116
  npm-token fallback. It grants `id-token: write` for OIDC, keeps repository
111
- contents read-only, uses the `npm-publish` GitHub environment, and publishes
112
- with `npm publish --access public` only on the real-publish path.
113
-
114
- Use this npm CLI setup command as the trusted publisher reference:
115
-
116
- ```sh
117
- npm install -g npm@^11.10.0
118
- npm trust github @navels/neal --repo navels/neal --file publish.yml --env npm-publish --allow-publish
119
- ```
120
-
121
- Current first-publish limitation: `@navels/neal` is not currently published, and
122
- the `npm trust github` command may be blocked until the package already exists
123
- on the npm registry. The npm trusted publisher configuration must match the
124
- repository workflow: package `@navels/neal`, owner/repo `navels/neal`, workflow
125
- filename `publish.yml`, environment `npm-publish`, and allowed action
126
- `npm publish`.
117
+ contents read-only, uses the `npm-publish` GitHub environment, and runs
118
+ `npm stage publish` only on the real-publish path (staged publishing requires
119
+ npm >= 11.15.0; the workflow upgrades npm accordingly).
120
+
121
+ The npm-side configuration for `@navels/neal`:
122
+
123
+ - Publishing access: **Require two-factor authentication and disallow tokens**.
124
+ This blocks every traditional token permanently; trusted publishers are
125
+ unaffected because they use OIDC, and staged approvals always require a
126
+ maintainer's 2FA.
127
+ - Trusted publisher: owner/repo `navels/neal`, workflow filename
128
+ `publish.yml`, environment `npm-publish`, allowed action **`npm stage
129
+ publish` only** (stage-only; plain `npm publish` is deliberately not
130
+ granted).
127
131
 
128
132
  ## Release Boundaries
129
133