@natjswenson/devlog 0.4.2 → 0.5.1

package/config.example.json

@@ -4,6 +4,10 @@
   "gitAuthor": "Your Name",
   "githubUser": "yourusername",
   "voicePath": "~/.claude/ghostwriter/voice",
+  "deepDive": {
+    "topicDomains": ["AI", "DevOps/SRE", "software engineering"],
+    "minSources": 3
+  },
   "projects": [
     {
       "key": "midnight-side-quest",

package/evals/budget.mjs

@@ -0,0 +1,60 @@
+// Cost control for live eval runs. Ported from ghostwriter's evals/budget.py:
+// deliberately over-estimates, guards BEFORE any call that would exceed the
+// cap, and forces mock mode when there is no API key so CI can never spend.
+
+export const DEFAULT_MAX_SPEND = 0.50; // USD
+
+// Conservative blended $/1K-token prices by model-family substring. These are
+// intentionally pessimistic (treated as if every token cost the output rate).
+const PRICE_PER_1K = [
+  [/haiku/i, 0.005],
+  [/sonnet/i, 0.018],
+  [/opus/i, 0.09],
+];
+const DEFAULT_PRICE_PER_1K = 0.02;
+
+export function pricePer1k(model) {
+  for (const [re, price] of PRICE_PER_1K) {
+    if (re.test(model || '')) return price;
+  }
+  return DEFAULT_PRICE_PER_1K;
+}
+
+// Over-estimate the cost of one `claude -p` judge call: a context baseline for
+// the re-sent system prompt, ~4 chars/token for the prompt itself, plus the
+// output budget.
+export function estimateUsd(text, model, { outputTokens = 800, contextTokens = 6000 } = {}) {
+  const promptTokens = Math.ceil((text || '').length / 4);
+  const totalTokens = contextTokens + promptTokens + outputTokens;
+  return (totalTokens / 1000) * pricePer1k(model);
+}
+
+export class BudgetExceeded extends Error {}
+
+export class Budget {
+  constructor(maxSpend = DEFAULT_MAX_SPEND) {
+    this.maxSpend = maxSpend;
+    this.spent = 0;
+  }
+
+  // Call BEFORE spending: throws if the estimate would push cumulative spend
+  // over the cap. The cap is a hard ceiling, not advisory.
+  guard(estimate) {
+    if (this.spent + estimate > this.maxSpend) {
+      throw new BudgetExceeded(
+        `Refusing call: ~$${estimate.toFixed(3)} would push spend to `
+        + `$${(this.spent + estimate).toFixed(3)} > cap $${this.maxSpend.toFixed(2)}`,
+      );
+    }
+  }
+
+  record(actual) {
+    this.spent += actual;
+  }
+}
+
+// Mock is forced when there is no ANTHROPIC_API_KEY — CI has no key, so a
+// forgotten --mock flag can never turn into a live spend there.
+export function mockEnabled(flag) {
+  return Boolean(flag) || !process.env.ANTHROPIC_API_KEY;
+}

package/evals/fixtures/bad-post.md

@@ -0,0 +1,33 @@
+---
+title: "Release v1.4.0"
+date: 2026-07-10
+project: fixture
+version: v1.4.0
+tags: [update]
+summary: "We shipped some updates."
+---
+
+## Shipped
+
+This release is a game-changer for our workflow. We revolutionized the retry logic and
+made everything more robust, scalable, and future-proof.
+
+## The Update
+
+We updated the code. Here is some of it:
+
+```
+def retry(fn):
+    return fn
+```
+
+It's better now. No more issues. The system is faster. The system is cleaner. The system
+is unstoppable.
+
+## Sources
+
+- [Some blog](https://example.com/blog) — general vibes
+
+## Changelog
+
+- update stuff ([abc1234](https://github.com/example/fixture/commit/abc1234))

package/evals/fixtures/good-post.md

@@ -0,0 +1,118 @@
+---
+title: "Retries that don't stampede: exponential backoff with jitter in 40 lines"
+date: 2026-07-10
+project: fixture
+version: v1.3.0
+tags: [reliability, python, distributed-systems]
+summary: "This release moved our flaky HTTP calls behind a retry wrapper. Here's how to build one with full jitter, and the two traps that bit me."
+---
+
+## Shipped
+
+v1.3.0 wraps every outbound HTTP call in a retry decorator with exponential backoff and
+full jitter. The change itself is small; the interesting part is why naive retries make
+outages worse, and how little code the correct version takes. This post walks the whole
+build.
+
+## Prerequisites
+
+You need Python 3.10+ and `httpx`. No other dependencies — the retry logic is stdlib.
+
+```bash
+pip install httpx==0.27.0
+```
+
+## Build the backoff schedule
+
+Start with the delay calculation, isolated so you can unit-test it. Full jitter means:
+sleep a uniform random amount between 0 and the exponential ceiling, which spreads
+retrying clients across the whole window instead of synchronizing them into waves.
+
+```python
+import random
+
+def backoff_delay(attempt: int, base: float = 0.5, cap: float = 30.0) -> float:
+    """Full-jitter delay for a zero-indexed attempt number."""
+    ceiling = min(cap, base * (2 ** attempt))
+    return random.uniform(0, ceiling)
+```
+
+## Wrap it into a retry decorator
+
+The decorator retries only on retryable failures (connection errors and 5xx), never on
+4xx — a 404 will be a 404 no matter how many times you ask.
+
+```python
+import functools
+import time
+import httpx
+
+RETRYABLE_STATUS = {500, 502, 503, 504}
+
+def with_retries(max_attempts: int = 5):
+    def decorator(fn):
+        @functools.wraps(fn)
+        def wrapper(*args, **kwargs):
+            for attempt in range(max_attempts):
+                try:
+                    response = fn(*args, **kwargs)
+                    if response.status_code not in RETRYABLE_STATUS:
+                        return response
+                except httpx.TransportError:
+                    if attempt == max_attempts - 1:
+                        raise
+                time.sleep(backoff_delay(attempt))
+            return response
+        return wrapper
+    return decorator
+```
+
+## Use it
+
+Apply it to any function that returns an `httpx.Response`:
+
+```python
+@with_retries(max_attempts=5)
+def fetch_profile(user_id: str) -> httpx.Response:
+    return httpx.get(f"https://api.example.com/users/{user_id}", timeout=5.0)
+
+profile = fetch_profile("u_123")
+print(profile.status_code)  # 200 after up to 5 attempts
+```
+
+## Verify it
+
+Prove the schedule spreads load before you trust it. Run this against your copy of
+`backoff_delay` — every delay must fall inside its window and the cap must hold:
+
+```python
+for attempt in range(10):
+    d = backoff_delay(attempt)
+    assert 0 <= d <= min(30.0, 0.5 * 2 ** attempt), (attempt, d)
+print("schedule ok")
+```
+
+Expected output: `schedule ok`. To see the retry path itself, point `fetch_profile` at
+`https://httpbin.org/status/503` — you should observe five attempts spaced by growing,
+randomized delays, then the final 503 returned.
+
+## Gotchas
+
+- **Retrying 4xx responses.** My first version retried everything non-200. Symptom: a
+  bad auth token turned into 5 slow failures instead of 1 fast one, and our alert fired
+  on latency instead of auth. Escape: allowlist the retryable statuses (5xx + transport
+  errors) explicitly.
+- **Equal jitter isn't enough under real outages.** I started with `ceiling/2 +
+  uniform(0, ceiling/2)`. Symptom: load tests showed retry waves still clustering at the
+  half-window mark. Escape: full jitter (`uniform(0, ceiling)`), which the AWS analysis
+  below shows keeps total calls lowest across client counts.
+
+## Sources
+
+- [Exponential Backoff And Jitter (AWS Architecture Blog)](https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/) — full vs. equal jitter comparison
+- [httpx documentation](https://www.python-httpx.org/exceptions/) — TransportError hierarchy
+- [Google SRE Book: Handling Overload](https://sre.google/sre-book/handling-overload/) — why synchronized retries amplify outages
+
+## Changelog
+
+- feat: retry wrapper with full jitter ([a1b2c3d](https://github.com/example/fixture/commit/a1b2c3d4))

package/evals/fixtures/irreproducible-post.md

@@ -0,0 +1,71 @@
+---
+title: "How our pipeline learned to validate itself end-to-end"
+date: 2026-07-10
+project: fixture
+version: v2.1.0
+tags: [testing, data-pipelines, python]
+summary: "v2.1.0 added self-validating pipeline stages. A look at how the validation layer came together."
+---
+
+## Shipped
+
+v2.1.0 wires validation into every pipeline stage. Each stage now declares a schema, and
+the runner checks outputs before they flow downstream. This post walks through how the
+system fits together.
+
+## The validation layer
+
+The heart of it is the stage wrapper. It pulls the declared schema off the stage and
+routes failures to our dead-letter handler:
+
+```python
+def validated(stage):
+    def wrapper(batch):
+        result = stage(batch)
+        schema = registry.schema_for(stage)
+        for row in result:
+            check_row(row, schema, on_error=dead_letter.route)
+        return result
+    return wrapper
+```
+
+`registry` holds all our stage schemas and `dead_letter.route` sends bad rows to the
+usual place. With that in place, the runner just wraps everything:
+
+```python
+pipeline = build_pipeline(config)
+for stage in pipeline.stages:
+    stage.fn = validated(stage.fn)
+run_pipeline(pipeline, source=events_source())
+```
+
+## Watching it work
+
+Once deployed, the validation layer immediately started catching issues. In our repo,
+running the nightly job now prints:
+
+```text
+[validate] stage=enrich rows=48210 rejected=17 (0.04%)
+[validate] stage=aggregate rows=48193 rejected=0
+nightly: OK
+```
+
+Those 17 rejected rows were exactly the malformed events we'd been chasing for weeks.
+The dead-letter queue fills up with them and our dashboard graphs the rejection rate per
+stage, which has already flattened out nicely since launch.
+
+## Gotchas
+
+- The main lesson here is philosophical: validation is a journey, not a destination. We
+  learned to think of schemas as living documents and to stay flexible about where
+  checking happens. Teams should find the balance that works for them.
+
+## Sources
+
+- [Great Expectations documentation](https://docs.greatexpectations.io/docs/) — data validation concepts
+- [Designing Data-Intensive Applications](https://dataintensive.net/) — schema evolution background
+- [Martin Fowler on ContractTest](https://martinfowler.com/bliki/ContractTest.html) — testing at boundaries
+
+## Changelog
+
+- feat: self-validating stages ([f9e8d7c](https://github.com/example/fixture/commit/f9e8d7c6))

package/evals/judge_post.mjs

@@ -0,0 +1,113 @@
+// Two-layer quality judge for generated posts.
+// Layer 1 ($0, deterministic): lint-post findings are an automatic fail.
+// Layer 2 ($, LLM): a cheap judge scores the post against the how-to contract
+// and returns strict JSON. Mock mode substitutes a fixed passing score — it
+// can catch contract violations (via layer 1) but NOT subtle quality drift;
+// that is exactly what the irreproducible fixture documents.
+import { spawnSync } from 'node:child_process';
+import { lintPost } from '../lib/lint_post.mjs';
+import { estimateUsd } from './budget.mjs';
+
+export const DEFAULT_JUDGE_MODEL = 'claude-haiku-4-5-20251001';
+export const DEFAULT_MIN_SCORE = 7.0;
+
+export const DIMENSIONS = [
+  'reproducibility',      // could a stranger build this from the post alone?
+  'code_completeness',    // no phantom fixtures; blocks compose into a runnable whole
+  'gotcha_quality',       // concrete trap → symptom → escape, plausibly from real history
+  'citation_quality',     // distinct reputable sources actually supporting the claims
+  'voice_fidelity',       // authentic first-person, no AI tells
+  'scope_honesty',        // title/summary sized to what actually shipped
+];
+
+export function buildJudgePrompt(post, voiceGuide = '') {
+  return `You are scoring a developer-blog release post against a strict "how-to guide" contract.
+
+The contract: (1) REPRODUCIBILITY — a reader with no access to the author's repository can
+follow the post and build the technique end-to-end (the "stranger test"); (2) CODE
+COMPLETENESS — every symbol a code block references is defined in an earlier block or
+explicitly stubbed; the blocks compose into a runnable whole, not fragments around an
+essay; (3) GOTCHA QUALITY — the Gotchas section gives concrete trap → symptom → escape
+items that read as genuinely experienced, not generic advice; (4) CITATION QUALITY —
+distinct, reputable sources that actually support the specific claims made; (5) VOICE
+FIDELITY — authentic first-person writing with no AI tells (no padded symmetry, no
+hype, no filler)${voiceGuide ? ', judged against the voice guide below' : ''}; (6) SCOPE
+HONESTY — the title and framing match what actually shipped (a single test file is not
+"end-to-end").
+
+Score each dimension 0-10 and give an overall 0-10 score (your holistic judgment, not an
+average — a fatal reproducibility failure caps the overall at 5 even if prose is lovely).
+
+Return ONLY a JSON object, no prose:
+{"score": <0-10 float>, "dimensions": {"reproducibility": <0-10>, "code_completeness": <0-10>, "gotcha_quality": <0-10>, "citation_quality": <0-10>, "voice_fidelity": <0-10>, "scope_honesty": <0-10>}, "worst_problem": "<one sentence>"}
+${voiceGuide ? `\n=== VOICE GUIDE ===\n${voiceGuide}\n` : ''}
+=== POST ===
+${post}`;
+}
+
+// Extract and validate the first {...} JSON blob from the judge's reply.
+export function parseJudgeResponse(text) {
+  const m = /\{[\s\S]*\}/.exec(text || '');
+  if (!m) throw new Error(`Judge returned no JSON object: ${String(text).slice(0, 200)}`);
+  const parsed = JSON.parse(m[0]);
+  if (typeof parsed.score !== 'number' || parsed.score < 0 || parsed.score > 10) {
+    throw new Error(`Judge score out of range: ${JSON.stringify(parsed.score)}`);
+  }
+  const dims = parsed.dimensions || {};
+  for (const d of DIMENSIONS) {
+    if (typeof dims[d] !== 'number' || dims[d] < 0 || dims[d] > 10) {
+      throw new Error(`Judge dimension "${d}" missing or out of range`);
+    }
+  }
+  return { score: parsed.score, dimensions: dims, worstProblem: parsed.worst_problem || null };
+}
+
+// Live call site — injectable so tests never touch the network.
+export function callClaude(prompt, model) {
+  const r = spawnSync('claude', ['-p', prompt, '--model', model], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+    timeout: 120_000,
+  });
+  if (r.status !== 0) throw new Error(`claude -p failed (${r.status}): ${(r.stderr || '').slice(0, 300)}`);
+  return r.stdout;
+}
+
+// Score one post. Returns
+//   { pass, score, dimensions?, lintFindings, mocked, worstProblem? }
+export function scorePost(content, {
+  mock = true,
+  model = DEFAULT_JUDGE_MODEL,
+  minScore = DEFAULT_MIN_SCORE,
+  minSources = 3,
+  voiceGuide = '',
+  budget = null,
+  runJudge = callClaude,
+} = {}) {
+  // Layer 1: contract violations fail before any money is spent.
+  const lint = lintPost(content, { minSources });
+  if (!lint.ok) {
+    return { pass: false, score: 0, lintFindings: lint.findings, mocked: false };
+  }
+
+  if (mock) {
+    // $0 smoke path: lint passed, so report a passing score. Cannot catch
+    // subtle quality drift — only a live judge can.
+    return { pass: true, score: 9.0, lintFindings: [], mocked: true };
+  }
+
+  const prompt = buildJudgePrompt(content, voiceGuide);
+  const estimate = estimateUsd(prompt, model);
+  if (budget) budget.guard(estimate);
+  const raw = runJudge(prompt, model);
+  if (budget) budget.record(estimate); // conservative: record the estimate, not less
+  const judged = parseJudgeResponse(raw);
+  return {
+    pass: judged.score >= minScore,
+    score: judged.score,
+    dimensions: judged.dimensions,
+    worstProblem: judged.worstProblem,
+    lintFindings: [],
+    mocked: false,
+  };
+}

package/evals/run_eval.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+// Eval runner for the non-deterministic half of devlog: scores the golden
+// fixtures with the two-layer judge and checks each lands on its expected
+// side. Mock mode is $0 and runs in CI; live mode quotes its estimated spend
+// up front and refuses to start over the cap.
+//
+//   node evals/run_eval.mjs --mock
+//   node evals/run_eval.mjs --live [--max-spend 0.50] [--model <id>] [--min-score 7]
+import { readFileSync, readdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+import { Budget, DEFAULT_MAX_SPEND, estimateUsd, mockEnabled } from './budget.mjs';
+import { scorePost, buildJudgePrompt, DEFAULT_JUDGE_MODEL, DEFAULT_MIN_SCORE, callClaude } from './judge_post.mjs';
+
+const FIXTURES_DIR = join(dirname(fileURLToPath(import.meta.url)), 'fixtures');
+
+// expect: which side of the gate the fixture must land on.
+// liveOnly: passes the deterministic layer by design — only a live judge can
+// fail it, so mock runs report it as skipped rather than vacuously green.
+export const CASES = [
+  { file: 'good-post.md', expect: 'pass', liveOnly: false },
+  { file: 'bad-post.md', expect: 'fail', liveOnly: false },
+  { file: 'irreproducible-post.md', expect: 'fail', liveOnly: true },
+];
+
+export function runCases({ mock, model = DEFAULT_JUDGE_MODEL, minScore = DEFAULT_MIN_SCORE, budget, runJudge = callClaude, fixturesDir = FIXTURES_DIR }) {
+  return CASES.map((c) => {
+    if (mock && c.liveOnly) {
+      return { ...c, outcome: 'skipped', note: 'needs a live judge (passes the deterministic layer by design)' };
+    }
+    const content = readFileSync(join(fixturesDir, c.file), 'utf8');
+    const result = scorePost(content, { mock, model, minScore, budget, runJudge });
+    const got = result.pass ? 'pass' : 'fail';
+    return { ...c, outcome: got === c.expect ? 'ok' : 'UNEXPECTED', got, result };
+  });
+}
+
+function main() {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      mock: { type: 'boolean', default: false },
+      live: { type: 'boolean', default: false },
+      model: { type: 'string', default: DEFAULT_JUDGE_MODEL },
+      'min-score': { type: 'string', default: String(DEFAULT_MIN_SCORE) },
+      'max-spend': { type: 'string', default: String(DEFAULT_MAX_SPEND) },
+    },
+  });
+
+  const mock = mockEnabled(values.mock || !values.live);
+  const model = values.model;
+  const minScore = Number(values['min-score']);
+  const maxSpend = Number(values['max-spend']);
+  const budget = new Budget(maxSpend);
+
+  if (!mock) {
+    // Quote the worst-case spend before the first call and refuse over cap.
+    const liveCases = CASES.filter((c) => c.file !== 'bad-post.md'); // fails at the $0 layer
+    const estimate = liveCases.reduce((sum, c) => {
+      const content = readFileSync(join(FIXTURES_DIR, c.file), 'utf8');
+      return sum + estimateUsd(buildJudgePrompt(content), model);
+    }, 0);
+    console.log(`Estimated spend ~$${estimate.toFixed(3)} (cap $${maxSpend.toFixed(2)}, model ${model})`);
+    if (estimate > maxSpend) {
+      console.error('Estimated spend exceeds the cap — refusing to start. Raise --max-spend deliberately.');
+      process.exit(2);
+    }
+  } else {
+    console.log('Mock mode ($0). The live-only fixture is skipped; run --live to exercise the judge.');
+  }
+
+  const results = runCases({ mock, model, minScore, budget });
+  let bad = 0;
+  for (const r of results) {
+    if (r.outcome === 'skipped') {
+      console.log(`~ ${r.file}: skipped — ${r.note}`);
+      continue;
+    }
+    const mark = r.outcome === 'ok' ? '✓' : '✗';
+    if (r.outcome !== 'ok') bad += 1;
+    const score = r.result.score !== undefined ? ` score=${r.result.score}` : '';
+    const lint = r.result.lintFindings?.length ? ` lint=[${r.result.lintFindings.map((f) => f.rule).join(', ')}]` : '';
+    const worst = r.result.worstProblem ? ` worst="${r.result.worstProblem}"` : '';
+    console.log(`${mark} ${r.file}: expected ${r.expect}, got ${r.got}${score}${lint}${worst}`);
+    if (r.result.dimensions) {
+      console.log(`    ${Object.entries(r.result.dimensions).map(([k, v]) => `${k}=${v}`).join(' ')}`);
+    }
+  }
+  if (!mock) console.log(`Recorded spend (conservative estimates): $${budget.spent.toFixed(3)}`);
+  process.exit(bad === 0 ? 0 : 1);
+}
+
+const isMain = process.argv[1] === fileURLToPath(import.meta.url);
+if (isMain) main();

package/lib/config_ops.mjs

@@ -0,0 +1,58 @@
+// Pure, agent-drivable config mutations. Each function takes the current
+// config object and returns a NEW validated config — the CLI layer owns
+// reading/writing ~/.claude/skills/devlog/config.json atomically.
+import { validateConfig, expandHome } from './core.mjs';
+
+export function addProject(config, { key, path, remote, label, tagPrefix, pathFilter }) {
+  if (config.projects.some((p) => p.key === key)) {
+    throw new Error(`Project key "${key}" is already registered.`);
+  }
+  const project = { key, path: expandHome(path), remote };
+  if (label) project.label = label;
+  // Only persist tagPrefix when it differs from the default `v` (keeps configs clean).
+  if (tagPrefix && tagPrefix !== 'v') project.tagPrefix = tagPrefix;
+  if (pathFilter) project.pathFilter = pathFilter;
+  return validateConfig({ ...config, projects: [...config.projects, project] });
+}
+
+export function removeProject(config, key) {
+  const projects = config.projects.filter((p) => p.key !== key);
+  if (projects.length === config.projects.length) {
+    throw new Error(`No project with key "${key}". Registered: ${config.projects.map((p) => p.key).join(', ') || '(none)'}`);
+  }
+  return validateConfig({ ...config, projects });
+}
+
+// Fields settable via `devlog set <field> <value>`. Everything funnels through
+// validateConfig, so a bad value can never be persisted.
+const SETTERS = {
+  targetRepo: (c, v) => ({ ...c, targetRepo: v }),
+  branch: (c, v) => ({ ...c, branch: v }),
+  gitAuthor: (c, v) => ({ ...c, gitAuthor: v }),
+  githubUser: (c, v) => ({ ...c, githubUser: v }),
+  voicePath: (c, v) => (v === '' ? omit(c, 'voicePath') : { ...c, voicePath: expandHome(v) }),
+  'deepDive.minSources': (c, v) => {
+    const n = Number(v);
+    return { ...c, deepDive: { ...(c.deepDive || {}), minSources: n } };
+  },
+  'deepDive.topicDomains': (c, v) => {
+    const domains = v.split(',').map((t) => t.trim()).filter(Boolean);
+    return { ...c, deepDive: { ...(c.deepDive || {}), topicDomains: domains } };
+  },
+};
+
+export const SETTABLE_FIELDS = Object.keys(SETTERS);
+
+export function setField(config, field, value) {
+  const setter = SETTERS[field];
+  if (!setter) {
+    throw new Error(`Unknown field "${field}". Settable: ${SETTABLE_FIELDS.join(', ')}`);
+  }
+  if (typeof value !== 'string') throw new Error('Value must be a string.');
+  return validateConfig(setter(config, value));
+}
+
+function omit(obj, key) {
+  const { [key]: _dropped, ...rest } = obj;
+  return rest;
+}