@natjswenson/devlog 0.10.0 → 0.11.1

package/bin/devlog.js

@@ -2,7 +2,7 @@
 import { spawn, spawnSync, execSync } from 'node:child_process';
 import {
   existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, realpathSync,
-  readdirSync, statSync, unlinkSync, rmSync, mkdtempSync,
+  readdirSync, statSync, rmSync, mkdtempSync,
 } from 'node:fs';
 import { dirname, join, resolve, basename } from 'node:path';
 import { fileURLToPath } from 'node:url';
@@ -32,9 +32,10 @@ import {
   validateConfig,
   resolveDeepDive,
 } from '../lib/core.mjs';
-import { scanAll } from '../lib/scan.mjs';
+import { scanAll, summarizeScan } from '../lib/scan.mjs';
 import { lintPost, parseFrontmatter, splitSections } from '../lib/lint_post.mjs';
-import { publishEntry, addCoverToExistingEntry } from '../lib/publish_entry.mjs';
+import { publishEntry, addCoverToExistingEntry, tombstoneEntry, syncEntryFromFrontmatter } from '../lib/publish_entry.mjs';
+import { writeAssembledBlocks } from '../lib/assemble_post.mjs';
 import { addProject, removeProject, setField, SETTABLE_FIELDS } from '../lib/config_ops.mjs';
 import { loadStyleGuide, getRecentCovers, mergeManifestEntries } from '../lib/cover_gen.mjs';
 import { renderCoverImage } from '../lib/render_cover.mjs';
@@ -110,6 +111,18 @@ function emitJSON(obj, exitCode = 0) {
   process.exit(exitCode);
 }
 
+// parseArgs is strict by default, so an unknown flag (`render-cover --force`)
+// used to die with a raw ERR_PARSE_ARGS_UNKNOWN_OPTION stack trace instead of
+// the JSON error shape every agent-facing command promises. Same contract as
+// commit-covers' hand-rolled parser: unknown/malformed flags → bad-flag JSON.
+function safeParseArgs(spec) {
+  try {
+    return parseArgs(spec);
+  } catch (e) {
+    emitJSON({ error: 'bad-flag', message: e.message }, 2);
+  }
+}
+
 function readValidConfigOrExit({ json = false } = {}) {
   if (!existsSync(CONFIG_PATH)) {
     if (json) emitJSON({ error: 'config-missing', path: CONFIG_PATH, hint: 'Run `npx @natjswenson/devlog init` first.' }, 1);
@@ -462,7 +475,7 @@ async function cmdInit() {
 
 // ─── add-project ─────────────────────────────────────────────────────────────
 async function cmdAddProject(rest) {
-  const { values } = parseArgs({
+  const { values } = safeParseArgs({
     args: rest,
     options: {
       path: { type: 'string' },
@@ -541,7 +554,7 @@ async function cmdAddProject(rest) {
 
 // ─── remove-project ──────────────────────────────────────────────────────────
 async function cmdRemoveProject(rest) {
-  const { values, positionals } = parseArgs({
+  const { values, positionals } = safeParseArgs({
     args: rest,
     options: { yes: { type: 'boolean', default: false } },
     allowPositionals: true,
@@ -571,7 +584,7 @@ async function cmdRemoveProject(rest) {
 
 // ─── set ─────────────────────────────────────────────────────────────────────
 function cmdSet(rest) {
-  const { positionals } = parseArgs({ args: rest, options: {}, allowPositionals: true });
+  const { positionals } = safeParseArgs({ args: rest, options: {}, allowPositionals: true });
   const [field, value] = positionals;
   const config = readValidConfigOrExit({ json: true });
   if (!field || value === undefined) {
@@ -588,11 +601,14 @@ function cmdSet(rest) {
 
 // ─── scan ────────────────────────────────────────────────────────────────────
 function cmdScan(rest) {
-  const { values } = parseArgs({
+  const { values } = safeParseArgs({
     args: rest,
     options: {
       project: { type: 'string' },
       'no-fetch': { type: 'boolean', default: false },
+      // Plan-table view: per release, commitCount instead of the commit list
+      // and diffstat; skippedTags collapsed to per-reason counts.
+      summary: { type: 'boolean', default: false },
       // scan always emits JSON; the flag is accepted so `scan --json` (as
       // SKILL.md spells it) is never a crash.
       json: { type: 'boolean', default: true },
@@ -600,19 +616,29 @@ function cmdScan(rest) {
     allowPositionals: false,
   });
   const config = readValidConfigOrExit({ json: true });
-  const result = scanAll(config, { projectKey: values.project || null, fetch: !values['no-fetch'] });
+  let result = scanAll(config, { projectKey: values.project || null, fetch: !values['no-fetch'] });
+  if (values.summary) result = summarizeScan(result);
+  // Which CLI actually ran: npx caches aggressively, and a stale install has
+  // silently missed shipped fixes before — the skill compares this against the
+  // version its own instructions shipped with.
+  if (!result.error) result.cliVersion = readPackageVersion();
   emitJSON(result, result.error ? 1 : 0);
 }
 
 // ─── lint-post ───────────────────────────────────────────────────────────────
 function cmdLintPost(rest) {
-  const { values, positionals } = parseArgs({
+  const { values, positionals } = safeParseArgs({
     args: rest,
-    options: { 'min-sources': { type: 'string' } },
+    options: {
+      'min-sources': { type: 'string' },
+      // Deterministic voice-contract rules (em dashes, banned phrases) —
+      // opt-in so non-voice callers and the eval harness keep their behavior.
+      voice: { type: 'boolean', default: false },
+    },
     allowPositionals: true,
   });
   const file = positionals[0];
-  if (!file) emitJSON({ error: 'missing-arg', message: 'Usage: devlog lint-post <file> [--min-sources N]' }, 2);
+  if (!file) emitJSON({ error: 'missing-arg', message: 'Usage: devlog lint-post <file> [--min-sources N] [--voice]' }, 2);
 
   let minSources;
   if (values['min-sources'] !== undefined) {
@@ -633,13 +659,13 @@ function cmdLintPost(rest) {
   } catch (e) {
     emitJSON({ error: 'unreadable', message: e.message }, 2);
   }
-  const result = lintPost(content, { minSources, filename: file });
+  const result = lintPost(content, { minSources, filename: file, voice: values.voice });
   emitJSON({ ...result, minSources }, result.ok ? 0 : 1);
 }
 
 // ─── publish-entry ───────────────────────────────────────────────────────────
 function cmdPublishEntry(rest) {
-  const { values } = parseArgs({
+  const { values } = safeParseArgs({
     args: rest,
     options: {
       clone: { type: 'string' },
@@ -677,6 +703,92 @@ function cmdPublishEntry(rest) {
   }
 }
 
+// ─── tombstone ───────────────────────────────────────────────────────────────
+// Editorially retire a (project, version) identity after its entry was moved,
+// consolidated, or deleted by hand — scan then reports `entry-tombstoned` and
+// publish-entry refuses it forever.
+function cmdTombstone(rest) {
+  const { values } = safeParseArgs({
+    args: rest,
+    options: {
+      clone: { type: 'string' },
+      project: { type: 'string' },
+      version: { type: 'string' },
+      reason: { type: 'string' },
+    },
+    allowPositionals: false,
+  });
+  for (const flag of ['clone', 'project', 'version', 'reason']) {
+    if (!values[flag]) emitJSON({ error: 'missing-flag', message: `tombstone requires --${flag}` }, 1);
+  }
+  try {
+    const result = tombstoneEntry({
+      cloneDir: expandHome(values.clone),
+      project: values.project,
+      version: values.version,
+      reason: values.reason,
+    });
+    emitJSON({ ok: true, ...result });
+  } catch (e) {
+    emitJSON({ error: 'tombstone-failed', message: e.message }, 1);
+  }
+}
+
+// ─── sync-entry ──────────────────────────────────────────────────────────────
+// Resync a published entry's manifest row (title/summary/date/tags) from its
+// .md frontmatter after a deliberate post-publish edit.
+function cmdSyncEntry(rest) {
+  const { values } = safeParseArgs({
+    args: rest,
+    options: {
+      clone: { type: 'string' },
+      project: { type: 'string' },
+      slug: { type: 'string' },
+    },
+    allowPositionals: false,
+  });
+  for (const flag of ['clone', 'project', 'slug']) {
+    if (!values[flag]) emitJSON({ error: 'missing-flag', message: `sync-entry requires --${flag}` }, 1);
+  }
+  try {
+    const result = syncEntryFromFrontmatter({
+      cloneDir: expandHome(values.clone),
+      project: values.project,
+      slug: values.slug,
+    });
+    emitJSON({ ok: true, ...result });
+  } catch (e) {
+    emitJSON({ error: 'sync-failed', message: e.message }, 1);
+  }
+}
+
+// ─── assemble-post ───────────────────────────────────────────────────────────
+// Extract a draft's fenced code blocks, in order, into numbered files so the
+// Step 4 assemble-and-run check is mechanical instead of hand-copied.
+function cmdAssemblePost(rest) {
+  const { values, positionals } = safeParseArgs({
+    args: rest,
+    options: { out: { type: 'string' } },
+    allowPositionals: true,
+  });
+  const file = positionals[0];
+  if (!file) emitJSON({ error: 'missing-arg', message: 'Usage: devlog assemble-post <draft> --out <dir>' }, 2);
+  if (!values.out) emitJSON({ error: 'missing-flag', message: 'assemble-post requires --out' }, 1);
+
+  let content;
+  try {
+    content = readFileSync(expandHome(file), 'utf8');
+  } catch (e) {
+    emitJSON({ error: 'unreadable', message: e.message }, 2);
+  }
+  try {
+    const result = writeAssembledBlocks(content, expandHome(values.out));
+    emitJSON({ ok: true, ...result });
+  } catch (e) {
+    emitJSON({ error: 'assemble-failed', message: e.message }, 1);
+  }
+}
+
 // ─── backfill-covers list ─────────────────────────────────────────────────────
 function cmdBackfillCovers(rest) {
   const sub = rest[0];
@@ -684,7 +796,7 @@ function cmdBackfillCovers(rest) {
     emitJSON({ error: 'unknown-subcommand', message: 'Usage: devlog backfill-covers list --clone <cloneDir> [--project <key>] [--out <staging-dir>] [--all]' }, 2);
     return;
   }
-  const { values } = parseArgs({
+  const { values } = safeParseArgs({
     args: rest.slice(1),
     options: {
       clone: { type: 'string' },
@@ -761,7 +873,7 @@ function cmdBackfillCovers(rest) {
 
 // ─── cover-context ─────────────────────────────────────────────────────────────
 function cmdCoverContext(rest) {
-  const { positionals, values } = parseArgs({
+  const { positionals, values } = safeParseArgs({
     args: rest,
     options: {
       clone: { type: 'string' },
@@ -826,7 +938,7 @@ function regenerateContactSheet(outDir) {
 }
 
 async function cmdRenderCover(rest) {
-  const { positionals, values } = parseArgs({
+  const { positionals, values } = safeParseArgs({
     args: rest,
     options: {
       project: { type: 'string' },
@@ -852,17 +964,21 @@ async function cmdRenderCover(rest) {
   mkdirSync(projectDir, { recursive: true });
   const pngPath = join(projectDir, `${values.slug}.png`);
 
-  // Idempotent re-run: an existing, valid PNG is left untouched — no re-render.
-  if (existsSync(pngPath) && isValidPngFile(pngPath)) {
-    regenerateContactSheet(outDir);
-    emitJSON({ ok: true, written: pngPath, rendered: false });
-    return;
-  }
-
+  // The HTML file is the source of truth: whenever it's present, render it —
+  // overwriting any stale PNG from a previous attempt. (The old
+  // PNG-exists short-circuit silently ignored freshly edited HTML, which cost
+  // every real retry loop an ls/mtime/md5 debugging dance and a guessed-at
+  // `--force` flag that didn't exist.) Only when the HTML is gone does an
+  // existing valid PNG mean "already rendered, nothing to do".
   let html;
   try {
     html = readFileSync(expandHome(htmlFile), 'utf8');
   } catch (e) {
+    if (existsSync(pngPath) && isValidPngFile(pngPath)) {
+      regenerateContactSheet(outDir);
+      emitJSON({ ok: true, written: pngPath, rendered: false });
+      return;
+    }
     emitJSON({ error: 'html-unreadable', message: e.message }, 1);
     return;
   }
@@ -878,9 +994,10 @@ async function cmdRenderCover(rest) {
   }
   writeFileSync(pngPath, png);
 
-  // Transient source document — deleted immediately after a successful render only.
-  try { unlinkSync(expandHome(htmlFile)); } catch { /* best-effort cleanup */ }
-
+  // The HTML source deliberately stays on disk (it lives in the run's scratch
+  // dir and dies with it): keeping it is what makes "tweak the HTML, re-run
+  // render-cover" work at all — deleting it on success broke every
+  // post-render Edit attempt in real runs.
   regenerateContactSheet(outDir);
   emitJSON({ ok: true, written: pngPath, rendered: true });
 }
@@ -1018,7 +1135,7 @@ async function cmdCommitCovers(rest) {
 
 // ─── config (view) ───────────────────────────────────────────────────────────
 async function cmdConfig(rest) {
-  const { values } = parseArgs({
+  const { values } = safeParseArgs({
     args: rest,
     options: { json: { type: 'boolean', default: false } },
     allowPositionals: false,
@@ -1137,12 +1254,17 @@ Setup & config:
   ${kleur.cyan('npx @natjswenson/devlog config [--json]')}          Show current config (with validation)
 
 Used by the /devlog skill:
-  ${kleur.cyan('npx @natjswenson/devlog scan [--project <key>]')}   JSON plan of new releases needing entries
-  ${kleur.cyan('npx @natjswenson/devlog lint-post <file>')}         Deterministic post-contract check
+  ${kleur.cyan('npx @natjswenson/devlog scan [--project <key>] [--summary]')}   JSON plan of new releases needing entries
+  ${kleur.cyan('npx @natjswenson/devlog lint-post <file> [--voice]')}  Deterministic post-contract check (+ voice rules)
+  ${kleur.cyan('npx @natjswenson/devlog assemble-post <draft> --out <dir>')}  Extract the draft's code blocks for the run-it check
   ${kleur.cyan('npx @natjswenson/devlog publish-entry ...')}        Copy a drafted entry into the clone + update manifest (never overwrites)
   ${kleur.cyan('npx @natjswenson/devlog cover-context <project> <slug> --clone <dir>')}  Style guide + reference-image paths for cover composition
   ${kleur.cyan('npx @natjswenson/devlog render-cover <html> --project <key> --slug <s> --out <dir>')}  Rasterize a composed cover to PNG
 
+Editorial maintenance:
+  ${kleur.cyan('npx @natjswenson/devlog tombstone --clone <dir> --project <key> --version <v> --reason <why>')}  Retire a moved/consolidated entry's identity
+  ${kleur.cyan('npx @natjswenson/devlog sync-entry --clone <dir> --project <key> --slug <v>')}  Resync a manifest row from an edited entry's frontmatter
+
 Backfilling covers onto existing posts:
   ${kleur.cyan('npx @natjswenson/devlog backfill-covers list --clone <dir> [--out <staging-dir>]')}  List posts missing a cover
   ${kleur.cyan('npx @natjswenson/devlog commit-covers <staging-dir> [--force [slug]]')}  Publish staged covers to already-published entries
@@ -1196,6 +1318,15 @@ if (isMain) {
     case 'publish-entry':
       cmdPublishEntry(rest);
       break;
+    case 'tombstone':
+      cmdTombstone(rest);
+      break;
+    case 'sync-entry':
+      cmdSyncEntry(rest);
+      break;
+    case 'assemble-post':
+      cmdAssemblePost(rest);
+      break;
     case 'backfill-covers':
       cmdBackfillCovers(rest);
       break;

package/evals/baseline/published/devlog-v0.10.0.md

@@ -0,0 +1,129 @@
+---
+title: "Folding a content repo into the site repo without breaking idempotent publishing"
+date: 2026-07-19
+project: devlog
+version: v0.10.0
+tags: [github-api, gh-cli, idempotency, cloudflare-pages, static-sites, content-pipeline, release-engineering, shell]
+summary: "Moving generated content into a subfolder of the site repo turns every publish push into a deploy. The catch: your publisher's already-published check is full of path assumptions, and missing one silently re-plans your entire archive."
+---
+
+## Shipped
+
+devlog v0.10.0 adds a `targetDir` setting: the publisher can now write its content tree into a subdirectory of the target repo instead of the repo root. That's the piece that let me delete a whole repo from my pipeline; devlog entries now land in `content/devlog/` inside the site repo itself, and the push that publishes them is the same push that deploys the site. The release also rewrote the cover style guide for the site's PRESS brand (including a fill floor for the hero illustration) and froze entry numbers at publish time so a backdated entry can't renumber the archive. This post is about the migration pattern: what an idempotent publisher checks before it writes, and every place a path prefix has to reach when the content moves.
+
+## The two-repo tax
+
+The old shape was a dedicated content repo. The generator pushed markdown, a manifest, and a cover PNG there; the site fetched it all at build time. Workable, but every publish needed a second step, an empty "rebuild" commit to the site repo, because [Cloudflare Pages triggers a deployment on commits to the production branch](https://developers.cloudflare.com/pages/configuration/branch-build-controls/) of the repo it watches, and content landing in a *different* repo doesn't count.
+
+Move the content into the site repo and that second step disappears. The site reads the files from disk at build time, and the publish push is the deploy trigger. The only real engineering is in the publisher: it has to stay idempotent when its target is no longer a repo root.
+
+## The check that makes publishing idempotent
+
+A release publisher should be safe to re-run. Mine plans work by listing what's already published and diffing against local git tags, one directory listing per project via the [GitHub contents API](https://docs.github.com/en/rest/repos/contents): `GET /repos/{owner}/{repo}/contents/{path}` returns an array of entries when `path` is a directory, and `ref` pins the branch. Through [`gh api`](https://cli.github.com/manual/gh_api) that's a one-liner, authenticated with your existing CLI login:
+
+```bash
+gh api "repos/OWNER/REPO/contents/PROJECT?ref=main" --jq '.[].name'
+```
+
+The Node version, with the two failure modes that matter kept distinct:
+
+```js
+// existing.mjs
+import { spawnSync } from 'node:child_process';
+
+// Which entry files already exist for one project.
+// Returns { files: Set, status: 'ok' | 'empty' | 'failed' }.
+export function fetchExistingEntries(repo, branch, projectKey, targetDir = '') {
+  const contentPath = targetDir ? `${targetDir}/${projectKey}` : projectKey;
+  const r = spawnSync('gh', ['api', `repos/${repo}/contents/${contentPath}?ref=${branch}`, '--jq', '.[].name'], { encoding: 'utf8' });
+  if (r.status === 0) {
+    return { files: new Set(r.stdout.split('\n').filter(Boolean)), status: 'ok' };
+  }
+  // 404 means "this project has no entries yet"; a normal state for a new
+  // project, not an error. Anything else means the check itself failed and
+  // the caller should know its already-published filter may be incomplete.
+  if (/HTTP 404|Not Found/i.test(r.stderr)) {
+    return { files: new Set(), status: 'empty' };
+  }
+  return { files: new Set(), status: 'failed' };
+}
+```
+
+The `targetDir` parameter is the whole feature. Without it, moving content to `content/devlog/` means the check asks GitHub for `contents/ghostwriter` at the repo root, gets a 404, concludes "no entries yet", and the planner happily re-plans every release you've ever published. Fifty-five posts, in my case, all queued for regeneration against a publisher that would then refuse each one.
+
+## Thread the prefix everywhere, and validate it
+
+A path that ends up inside shell commands and API URLs earns strict validation at config time:
+
+```js
+// config.mjs
+export function validateTargetDir(targetDir) {
+  if (
+    typeof targetDir !== 'string' ||
+    !/^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)*$/.test(targetDir) ||
+    targetDir.split('/').some((s) => s === '.' || s === '..')
+  ) {
+    throw new Error(`targetDir must be a relative path like "content/devlog": got ${JSON.stringify(targetDir)}`);
+  }
+  return targetDir;
+}
+```
+
+Relative only, tight charset, no traversal segments. The regex alone would accept `..` as a "word", so the explicit segment check backs it up.
+
+Then find every consumer of the old repo-root assumption. In my publisher there were three:
+
+```js
+// scan.mjs: the planner threads targetDir into the existence check and
+// echoes it in its output, so the publish step can build paths from the
+// same value instead of re-reading config.
+const existing = fetchExistingEntries(config.targetRepo, branch, project.key, config.targetDir || '');
+```
+
+The other two live in the publish step. The writer's content root becomes `<clone>/<targetDir>` while git still operates on the clone root; and any "view it here" URL you print needs the prefix too:
+
+```bash
+git clone --depth=1 "https://github.com/OWNER/REPO.git" "$TMP/REPO"
+CONTENT_ROOT="$TMP/REPO/content/devlog"   # writers target this
+# ... write PROJECT/vX.Y.Z.md, PROJECT/manifest.json, PROJECT/vX.Y.Z.png under $CONTENT_ROOT ...
+git -C "$TMP/REPO" add . && git -C "$TMP/REPO" commit -m 'devlog: add release entries'
+git -C "$TMP/REPO" push origin main       # on Cloudflare Pages, this IS the deploy
+```
+
+## Cut over and verify
+
+The safe order matters: land the content move in the site repo first, deploy it, and only then point the publisher at the new target; the existence check reads the target's live branch, so flipping config before the content exists there recreates the 404-means-empty re-planning problem on purpose.
+
+After flipping, one scan tells you whether the prefix reached everywhere. This is the real output from my cutover:
+
+```text
+targetRepo: natejswenson/natejswenson.io | targetDir: content/devlog | branch: main
+totalNewReleases: 5
+  local-fitness: existenceCheck=ok new=0
+  devlog: existenceCheck=ok new=2
+  ghostwriter: existenceCheck=ok new=2
+  resume: existenceCheck=ok new=0
+  local-budget: existenceCheck=ok new=0
+  personal: existenceCheck=ok new=0
+```
+
+Every project resolves `ok` through the subfolder, and the only "new" releases are genuinely untagged ones. If you see a fully published project reporting `new=<its entire history>`, the prefix missed the existence check.
+
+## Gotchas
+
+- **The missed-prefix failure is silent and looks like work to do.** Trap: any consumer of the old root-relative path that you didn't update. Symptom: not an error; the scan cheerfully reports your whole archive as new releases. Escape: after any path change, run the planner against the live target and assert the re-plan count is zero before letting anything publish.
+- **404 is a state, not a failure.** Trap: treating every non-200 from the contents API the same. Symptom: either a brand-new project blocks publishing (404 treated as failure) or a real outage quietly re-plans everything (failure treated as empty). Escape: three-way status, as in `fetchExistingEntries` above; only 404 means empty.
+- **The check is a filter, not the safety.** Trap: trusting the remote listing as the last line of defense. Symptom: a degraded check plus an overwrite-happy writer equals clobbered history. Escape: the writer itself must refuse to overwrite an existing entry against the fresh clone; then a failed existence check degrades to wasted planning, never to data loss.
+- **zsh eats the `?` in the API path.** Trap: pasting `gh api repos/o/r/contents/x?ref=main` unquoted into zsh while debugging. Symptom: `no matches found: repos/...` before gh even runs, because `?` is a glob character. Escape: quote the whole endpoint argument; I hit this within an hour of shipping the feature.
+- **Directory listings cap at 1,000 files.** Trap: one flat directory per project, forever. Symptom: the [contents API stops listing past 1,000 entries](https://docs.github.com/en/rest/repos/contents) and the existence filter goes blind. Escape: at that scale, switch the check to the Git Trees API; per-project directories buy a lot of headroom first.
+
+## Sources
+
+- [GitHub REST API: repository contents](https://docs.github.com/en/rest/repos/contents) — directory responses are arrays, `ref` pins the branch, 404 for missing paths, 1,000-file listing cap
+- [Cloudflare Pages: branch build controls](https://developers.cloudflare.com/pages/configuration/branch-build-controls/) — deployments trigger on commits to the production branch
+- [gh api manual](https://cli.github.com/manual/gh_api) — authenticated API calls from the CLI, `--jq` for response filtering
+
+## Changelog
+
+- feat(devlog): targetDir — publish the content tree into a subdirectory of targetRepo (#84) ([5bfada2](https://github.com/natejswenson/claude-skills/commit/5bfada2bfd3d0a3826cb78c1eed0056d71436840))
+- feat(devlog): PRESS cover style guide + frozen entry numbers ([2627a4c](https://github.com/natejswenson/claude-skills/commit/2627a4ce8cd59dfd95010e49f0a274c23abce289))

package/evals/baseline/published/devlog-v0.11.0.md

@@ -0,0 +1,139 @@
+---
+title: "Auditing an AI skill against its own past runs"
+date: 2026-07-19
+project: devlog
+version: v0.11.0
+tags: [ai-agents, agent-evaluation, postmortems, claude-code, subagents, git-forensics, guardrails, transcripts]
+summary: "Version 0.11.0 of this skill came entirely out of one exercise: asking the model to review the skill's own previous six runs. Here is the four-lens audit loop that produced it, and what the round cost in tokens."
+---
+
+## Shipped
+
+devlog 0.11.0 is a batch of guardrails: tombstones so a deleted entry can never republish itself, a ground-truth gate that verifies every claim about my own repos against git before publishing, a mechanical version of the run-the-code check, deterministic voice linting, and a fix for a CLI command that silently ignored fresh input. None of it came from a feature idea. All of it came from asking the model to review the skill's own past six runs. That review loop is the technique worth teaching; this guide walks through running it on your own agent skill.
+
+## The four lenses
+
+An agent skill drifts quietly. No single run fails hard enough to file a bug, but small problems repeat until they are load-bearing. The fix is the same discipline SRE applies to incidents: document what happened, understand the causes, and put preventive actions in place so it stops recurring ([Google SRE book, postmortem culture](https://sre.google/sre-book/postmortem-culture/)). The difference is that your "incident" is spread across runs, so you have to go collect it.
+
+I review on four dimensions, each answering one question:
+
+- **Accuracy**: did the runs do what the skill promises? For a writing skill, are the claims in the output true?
+- **Completeness**: what did the runs miss, and what did a human have to clean up afterward?
+- **Efficiency**: where did tokens and wall-clock time go that produced no value?
+- **Agent UX**: where did the skill's own instructions or tools fight the agent executing them?
+
+The last one surprises people. The agent is a user of your skill, and it hits usability bugs the same way humans do; it just cannot file a complaint. You find those bugs in transcripts.
+
+## Three evidence streams
+
+A run leaves three artifact trails, and each lens needs a different one.
+
+First, the outputs themselves: whatever the skill publishes. Grade a sample against the skill's own quality contract, and verify factual claims against the system of record instead of taking the output's word for it.
+
+Second, the correction commits. If your skill writes into a git repo, every manual fix a human made after a run is a finding with a timestamp. Separate the skill's own commits from everything else:
+
+```bash
+# Commits the skill's happy path writes (use your skill's commit message):
+git log --date=short --pretty='%h %ad %s' --grep='add release entries'
+
+# Everything a human had to do around them:
+git log --date=short --pretty='%h %ad %s' --invert-grep --grep='add release entries'
+```
+
+Running that against my dev-log repo (output trimmed to the signal lines):
+
+```text
+9b26494 2026-07-18 devlog: add release entries
+8f8938a 2026-07-17 devlog: add release entries
+...
+83cd9de 2026-07-18 chore: assign permanent entry numbers to all 55 entries
+c160db1 2026-07-17 devlog: file market-research v0.1.0 entry under personal
+```
+
+The second list held the review's best material: an entry reverted the same day a run re-added it, three posts consolidated into one by hand, and a moved file (that `c160db1` line) that the next run would have silently regenerated. Manual cleanup is the skill telling you what it cannot do yet.
+
+Third, the transcripts. Claude Code stores session logs as JSONL under `~/.claude/projects/`, one directory per working directory. Find the sessions where your skill ran by grepping for a command only it uses:
+
+```bash
+grep -l 'devlog scan --json' ~/.claude/projects/*/*.jsonl
+```
+
+Then pull rough per-run metrics. Transcript size is a fair cost proxy, and counting CLI invocations or retries shows where the loop stalled:
+
+```bash
+f=$(grep -l 'devlog scan --json' ~/.claude/projects/*/*.jsonl | head -1)
+wc -c < "$f"                      # transcript bytes, a rough cost proxy
+grep -c '"name":"Bash"' "$f"      # shell tool calls in the session
+```
+
+```text
+ 2762710
+95
+```
+
+## Fan out reviewers that can touch ground truth
+
+One reviewer reading everything runs out of context and blends the lenses together. I dispatch one subagent per evidence stream and give each a narrow brief plus access to the real repos. The prompt shape matters more than the wording:
+
+```markdown
+You are auditing the output quality of the <skill> agent skill.
+The skill's contract is at <path to SKILL.md>; read it first.
+
+For each published output:
+1. Grade it against the contract, point by point.
+2. Verify its factual claims against the SOURCE repo's git history
+   (tags, diffs, commit messages), not against the output's own text.
+3. Report patterns across outputs, each with concrete evidence
+   (file, commit hash, or quote).
+
+Be skeptical and specific; this feeds a fix list, not a report card.
+```
+
+Point 2 is the one you cannot skip. My accuracy reviewer found a published post whose central premise was false: it claimed two of four packages in a release never got git tags, and a single `git tag -l` showed all four tags existed. The post had passed the skill's self-review, because self-review graded the draft against the writing contract, never against the repo. A reviewer without repo access would have graded the same lie the same way.
+
+## Ship findings as gates, then re-run
+
+A review that produces a document has not improved anything yet. The SRE postmortem bar applies: findings become prioritized action items or they are theater ([postmortem culture](https://sre.google/sre-book/postmortem-culture/)). For agent skills I hold a stricter line: every finding ships as a mechanical gate in the next version, because an instruction the agent is supposed to remember is exactly the thing the review just proved gets skipped. Reflexion showed that agents improve when reflections persist somewhere durable instead of evaporating with the episode ([Shinn et al., 2023](https://arxiv.org/abs/2303.11366)); for a skill, the durable place is the code path the agent cannot route around.
+
+Two examples from this round. The false-premise finding became a required pre-publish step: list every claim the draft makes about your own repo, then verify each with a git command run in that session, and delete what you cannot verify. The cleanup-commit findings became identity guardrails; a retired artifact now keeps a tombstone row in its manifest, and the publish path refuses it:
+
+```json
+{ "version": "v0.1.0", "file": "v0.1.0.md", "removed": true,
+  "reason": "consolidated into the 2026-07-17 entry" }
+```
+
+```js
+function refuseTombstoned(manifest, version) {
+  const tombstoned = manifest.entries.find(
+    (e) => e.removed && e.version === version,
+  );
+  if (tombstoned) {
+    throw new Error(
+      `${version} was editorially retired (${tombstoned.reason}); refusing to republish.`,
+    );
+  }
+}
+```
+
+Transcript findings usually fix the tool, not the prompt. Anthropic's agent guidance says to invest in the agent-computer interface the way you would invest in UI design ([Building effective agents](https://www.anthropic.com/engineering/building-effective-agents)), and my transcripts proved why: a render command that silently no-opped when its output already existed cost the agent a multi-call debugging dance in two separate runs. The fix was one reordered check in the CLI, worth more than any added instruction.
+
+Then re-run the skill. The next real run is the acceptance test: this post was generated by the version the audit produced, its opening scan reported the new CLI version, and the retired entry scanned as tombstoned instead of resurfacing as a new release.
+
+On cost, since I had never tallied it: the three review subagents in this round reported 337k tokens between them, and the three code-exploration agents that turned findings into an implementation plan reported another 273k. Call the whole round about 600k subagent tokens plus the main session. That is real money, and it bought eighteen files of shipped fixes; I would not spend it weekly, but per release milestone it has paid for itself every time.
+
+## Gotchas
+
+- **Self-review grades the essay, not the facts.** Trap: letting the skill's quality check compare the draft to a rubric while every factual claim goes unchecked. Symptom: a confident, well-structured output with a false premise sails through. Escape: give reviewers (and the skill itself) repo access and require a verification command per claim; a claim you cannot verify gets removed, not softened.
+- **Honor-system steps are skipped exactly when they matter.** Trap: writing "run the code blocks and check the output" as an instruction. Symptom: outputs that say "this is the real output" over code that cannot run; my audit found one whose demo files were never defined anywhere. Escape: turn the instruction into a command the agent runs (mine extracts a draft's code blocks into numbered files), so skipping it becomes visible instead of silent.
+- **Output-only reviews miss tool friction entirely.** Trap: auditing what the skill produced and never how the agent got there. Symptom: the outputs look fine while every run quietly burns calls fighting the same CLI quirk. Escape: make transcripts a first-class evidence stream; the silent no-op I fixed in 0.11.0 appeared in zero outputs and two transcripts.
+- **The review has a real price and nobody is tracking it.** Trap: running fan-out reviews on a schedule without measuring them. Symptom: a surprise in the usage dashboard. Escape: pull the token counts from your platform's subagent usage reports while the round is fresh, and set the cadence from the number instead of a hunch.
+
+## Sources
+
+- [Google SRE Book: Postmortem Culture](https://sre.google/sre-book/postmortem-culture/) — blameless postmortems, and findings becoming prioritized preventive action
+- [Anthropic: Building effective agents](https://www.anthropic.com/engineering/building-effective-agents) — measure and iterate; invest in the agent-computer interface like UI design
+- [Reflexion: Language Agents with Verbal Reinforcement Learning](https://arxiv.org/abs/2303.11366) — agents improve when reflections persist in durable memory across episodes
+
+## Changelog
+
+- feat(devlog): 0.11.0 — tombstones, ground-truth gate, and the six-run audit fixes (#86) ([bd8fa5d](https://github.com/natejswenson/claude-skills/commit/bd8fa5d88a8c33dbf6f13ed64394b0682b4b8f4b))