@theronap/cortex-mcp 0.9.149 → 0.9.151

package/lib/capture_queue.mjs

@@ -0,0 +1,876 @@
+// The local capture queue — D9 + the client half of D8 (eng review 2026-09-11).
+//
+// 🔴 WHY THIS EXISTS. Two ways a session capture used to vanish:
+//
+//   1. A failed /api/ingest was logged to ~/.cortex/capture.log and dropped. A transient 503, a timeout,
+//      a rate-limit 429 — each one lost that turn's capture, and if it was the session's LAST turn, the
+//      session. Nothing retried, and nothing anyone reads said so.
+//
+//   2. When the person's own AI could not summarize the session locally (`claude -p` missing, its login
+//      expired, a subscription limit), the client shipped the raw transcript tail in `transcript` and
+//      the SERVER summarized it with Agnoclast's Anthropic key. D8 ends that: the transcript stays on the
+//      machine. So a turn that cannot be summarized now has to WAIT somewhere — here — until it can.
+//
+// This repo's most-repeated failure is a green status over work that did not happen. So the queue has
+// one rule above the others: NOTHING LEAVES IT SILENTLY. An entry leaves because it was recorded,
+// because the server made a final decision about it, or because a bound forced it out — and the last
+// two of those are written to a drop ledger that the next session start reports, alongside every entry
+// that has failed FLAG_AFTER_ATTEMPTS times.
+//
+// Layout — ~/.cortex/capture-queue/ (dir 0700, files 0600, every write tmp+rename so a concurrent
+// reader never sees half a file):
+//   <session key>.json   one entry per SESSION. The Stop hook fires every turn and the server keeps one
+//                        record per session, so a newer turn REPLACES the older entry (keeping its
+//                        attempt count), and a recorded turn removes it.
+//   _dropped.json        the drop ledger: what a bound or a server rejection forced out, for 7 days.
+//
+// Two kinds of entry:
+//   summarize  the local summarizer was unavailable. Holds the redacted transcript TAIL (≤ ~7 KB, the
+//              same text `claude -p` would have been shown) plus the ingest envelope. LOCAL ONLY — the
+//              tail is never sent anywhere; it is summarized here, later, and only the digest leaves.
+//   ingest     summarized fine, the POST failed. Holds the finished body, re-posted later. A summarize
+//              entry becomes an ingest entry the moment its summary exists, so a failed POST never
+//              costs a second `claude -p` run — and the transcript is deleted from the entry right then.
+//
+// Bounds: QUEUE_MAX_ENTRIES entries, QUEUE_MAX_AGE_DAYS days, QUEUE_MAX_ENTRY_BYTES per entry. Drained
+// opportunistically by later Stop-hook runs (the detached capture worker, which blocks nobody), with
+// equal-jittered exponential backoff per entry and a Retry-After floor.
+
+import { mkdirSync, readdirSync, readFileSync, writeFileSync, renameSync, unlinkSync, statSync, openSync, closeSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+import { createHash, randomBytes } from 'node:crypto'
+import { fetchCortex, classify, responseRetryClass, parseRetryAfter } from './diagnose.mjs'
+
+export const QUEUE_MAX_ENTRIES = 50
+export const QUEUE_MAX_AGE_DAYS = 14
+export const QUEUE_MAX_ENTRY_BYTES = 256 * 1024
+/** After this many failed attempts an entry is FLAGGED: kept, retried, and reported at session start. */
+export const FLAG_AFTER_ATTEMPTS = 3
+export const QUEUE_FIRST_RETRY_MS = 2 * 60_000
+export const QUEUE_MAX_RETRY_MS = 6 * 60 * 60_000
+/** How long a drop stays in the ledger (and so in the session-start line). */
+export const DROPPED_WINDOW_DAYS = 7
+const DROPPED_FILE = '_dropped.json'
+const DROPPED_MAX = 100
+const STALE_TMP_MS = 60 * 60_000
+
+/** One drain, per Stop-hook run: a few cheap POSTs, at most one `claude -p` retry, and a time cap.
+ *
+ * 🔴 THE BUDGET BOUNDS EVERY BLOCKING STEP, NOT JUST THE GAPS BETWEEN THEM. It first only wrapped the
+ * loop, so a due summarize entry could block for the full `summaryTimeoutMs()` (300 s by default, a
+ * SYNCHRONOUS spawnSync) and then still post — about five times the stated cap, and under
+ * CORTEX_CAPTURE_SYNC that is the hook itself waiting. Now a summarize is attempted only when the
+ * remaining budget can actually cover one, runs with `timeoutMs` set to what is left minus a reserve
+ * for its POST, and every POST is bounded by what remains after that.
+ *
+ * Sized so ONE summarization fits: a Stop-hook tail is ≤ ~7 KB (far smaller than the 50–147 KB lecture
+ * transcripts whose 56–150 s timings edge_extract.mjs records), and a session that cannot be summarized
+ * inside DRAIN_MIN_SUMMARY_MS is left for a later run rather than half-done. */
+export const DRAIN_MAX_POSTS = 5
+export const DRAIN_MAX_SUMMARIES = 1
+export const DRAIN_BUDGET_MS = 180_000
+/** Don't start a summarize unless this much budget remains (plus the post reserve). */
+export const DRAIN_MIN_SUMMARY_MS = 90_000
+/** Held back from a summarize's timeout so the digest it produces can still be posted. */
+export const DRAIN_POST_RESERVE_MS = 15_000
+/** Below this, don't start a POST at all — leave the entry for the next run. */
+export const DRAIN_MIN_POST_MS = 2_000
+/** Per-POST ceiling (also the live capture's). A detached worker self-terminates in ~25 s. */
+export const INGEST_BUDGET_MS = 25_000
+
+// ── Ordering: a per-session lease, and a generation per capture ───────────────────────────────────
+//
+// 🔴 WHY (adversarial review, 2026-09-11). The Stop hook fires per TURN and each turn runs in its own
+// detached worker, so two workers for one session overlap routinely — extraction alone takes tens of
+// seconds. They finished in any order, and nothing ordered their effects:
+//
+//   · turn N+1 queues after a transient 503; turn N finishes later, posts fine, and its
+//     `removeEntry(session)` deletes N+1's queued capture. The newer turn is silently lost.
+//   · turn N, failing later, overwrites N+1's queued entry with its older content.
+//   · the drain posts a digest it read minutes earlier while a live capture posts a newer one;
+//     /api/ingest dedupes by sessionId and takes the last write, so the stale replay wins.
+//
+// Two mechanisms, because the problem has two halves. A GENERATION orders the captures of one session;
+// a LEASE stops two of them from racing at the server.
+//
+// GENERATION = [transcript characters read, capture time]. The transcript is append-only per session, so
+// a later turn always reads at least as many characters; the timestamp only breaks ties (the Stop hook
+// firing twice with nothing new in between). Carried in the queue entry AND in the posted payload, so a
+// later server PR can reject an older generation for a session it already holds — see the follow-up note
+// in the PR. Every queue write is compare-and-set (never replace a newer generation) and every removal
+// is compare-and-delete (only delete what the caller superseded).
+//
+// LEASE = one file per session under `_locks/`, held across the check-then-POST-then-update critical
+// section by whoever is posting — live capture or drain. It makes a session's uploads serial, so a
+// poster can check "has a newer generation already been recorded?" (`_posted/`) and act on the answer
+// without another poster slipping in between. Never held across a summarize: that is minutes, and the
+// lease must stay short. Stale by pid-death or TTL, because a worker can be killed mid-post.
+export const LEASE_TTL_MS = 60_000
+/** A live capture will wait this long for another worker's POST of the same session to finish. */
+export const LEASE_WAIT_MS = 30_000
+/** A drain that has spent a `claude -p` run waits at most this long for the lease before giving up. */
+export const DRAIN_LEASE_WAIT_MS = 5_000
+/** A claim is abandoned after this, or as soon as the claiming process is gone (crash mid-claim). */
+export const CLAIM_TTL_MS = 10 * 60_000
+/** A summarize entry touched this recently belongs to a session that is probably still running — its
+ *  own next turn will capture it. A drain `claude -p` run holds the machine-wide summarizer lock for
+ *  ~1–2 min (edge_extract.mjs), which turns every concurrent session's capture into `busy`; spending
+ *  that on a session about to capture itself is pure cost. */
+export const SUMMARIZE_SETTLE_MS = 5 * 60_000
+
+const DAY_MS = 24 * 60 * 60_000
+
+export function queueDir(home = homedir()) {
+  return join(home, '.cortex', 'capture-queue')
+}
+
+/** A filesystem-safe, stable key per session. Claude and Codex session ids are UUIDs and pass as-is. */
+export function entryKey(sessionId) {
+  if (typeof sessionId === 'string' && /^[A-Za-z0-9_-]{1,100}$/.test(sessionId)) return sessionId
+  if (typeof sessionId === 'string' && sessionId) return `h-${createHash('sha1').update(sessionId).digest('hex').slice(0, 24)}`
+  return `nosession-${Date.now()}-${randomBytes(4).toString('hex')}`
+}
+
+/**
+ * PURE. The generation of one capture: how much transcript it read, and when it read it.
+ * `chars` is the ordering key; `at` only breaks ties.
+ */
+export function makeGen(chars, at) {
+  return { chars: Number.isFinite(chars) ? Math.max(0, Math.trunc(chars)) : 0, at: at ?? 0 }
+}
+
+/** PURE. Order two generations. A missing generation is older than any real one. */
+export function cmpGen(a, b) {
+  const ac = a ? a.chars ?? 0 : -1
+  const bc = b ? b.chars ?? 0 : -1
+  if (ac !== bc) return ac < bc ? -1 : 1
+  const aa = a?.at ?? 0
+  const ba = b?.at ?? 0
+  return aa === ba ? 0 : aa < ba ? -1 : 1
+}
+
+/** The generation as it travels in the posted payload (snake_case, like its neighbours there). */
+export function genPayload(gen) {
+  return { transcript_chars: gen?.chars ?? 0, captured_at: new Date(gen?.at ?? Date.now()).toISOString() }
+}
+
+/** PURE. Per-entry retry delay: 2 min doubling to 6 h, EQUAL jitter (half fixed, half random) so a
+ *  fleet that failed together does not come back together, while no entry retries sooner than half. */
+export function queueBackoffMs(attempts, random = Math.random) {
+  const ceiling = Math.min(QUEUE_FIRST_RETRY_MS * 2 ** Math.max(0, attempts - 1), QUEUE_MAX_RETRY_MS)
+  return Math.floor(ceiling / 2 + random() * (ceiling / 2))
+}
+
+export const isFlagged = (entry) => (entry?.attempts ?? 0) >= FLAG_AFTER_ATTEMPTS
+
+// ── storage ──────────────────────────────────────────────────────────────────────────────────────
+
+function ensureDir(dir) {
+  mkdirSync(dir, { recursive: true, mode: 0o700 })
+}
+
+function atomicWrite(dir, name, text) {
+  ensureDir(dir)
+  const file = join(dir, name)
+  const tmp = `${file}.${process.pid}.${randomBytes(3).toString('hex')}.tmp`
+  writeFileSync(tmp, text, { mode: 0o600 })
+  renameSync(tmp, file) // atomic: a concurrent reader never sees half a file
+}
+
+export function readEntry(dir, key) {
+  try { return JSON.parse(readFileSync(join(dir, `${key}.json`), 'utf8')) } catch { return null }
+}
+
+/** Every readable entry. An unreadable one is removed AND recorded — never silently skipped forever. */
+export function readEntries(dir, { log = () => {}, now = Date.now } = {}) {
+  let names
+  try { names = readdirSync(dir) } catch { return [] }
+  const out = []
+  for (const name of names) {
+    if (!name.endsWith('.json') || name.startsWith('_')) continue
+    try {
+      const entry = JSON.parse(readFileSync(join(dir, name), 'utf8'))
+      if (entry && typeof entry === 'object' && entry.key) out.push(entry)
+      else throw new Error('not an entry')
+    } catch {
+      try { unlinkSync(join(dir, name)) } catch { /* raced with another reader */ }
+      recordDropped(dir, { sessionId: name.replace(/\.json$/, ''), reason: 'unreadable queue file' }, now())
+      log(`DROPPED unreadable queue file ${name}`)
+    }
+  }
+  return out
+}
+
+function writeEntry(dir, entry) {
+  const text = JSON.stringify(entry)
+  if (Buffer.byteLength(text) > QUEUE_MAX_ENTRY_BYTES) return false
+  atomicWrite(dir, `${entry.key}.json`, text)
+  return true
+}
+
+/** Remove `key` — but only if it is still the revision we hold (a newer turn may have replaced it). */
+export function removeEntry(dir, key, rev = null) {
+  if (rev) {
+    const cur = readEntry(dir, key)
+    if (cur && cur.rev !== rev) return false
+  }
+  try { unlinkSync(join(dir, `${key}.json`)); return true } catch { return false }
+}
+
+/**
+ * Compare-and-delete: drop this session's entry only if it is NOT newer than the generation the caller
+ * just recorded. This is what a successful post calls, and it is the whole fix for "turn N's success
+ * deleted turn N+1's queued capture" — N+1 is newer, so it stays.
+ */
+export function removeIfNotNewer(dir, key, gen) {
+  const cur = readEntry(dir, key)
+  if (!cur) return false
+  if (cmpGen(cur.gen, gen) > 0) return false
+  return removeEntry(dir, key, cur.rev)
+}
+
+const newRev = () => randomBytes(6).toString('hex')
+
+// ── the session lease ────────────────────────────────────────────────────────────────────────────
+
+const leaseDir = (dir) => join(dir, '_locks')
+
+function pidAlive(pid) {
+  const n = Number(pid)
+  if (!Number.isInteger(n) || n <= 0) return false
+  try { process.kill(n, 0); return true } catch (e) { return e?.code === 'EPERM' } // EPERM = alive, not ours
+}
+
+/** Is the lock file abandoned? Its holder is gone, or it has outlived the TTL. */
+function leaseIsStale(file) {
+  let text
+  try { text = readFileSync(file, 'utf8') } catch (e) { return e?.code === 'ENOENT' }
+  const [pid, at] = text.trim().split(/\s+/)
+  // An EMPTY file is a lock caught between create and write — young, and not ours to break. Judge those
+  // by mtime only, or we would steal a lease microseconds after a neighbour created it.
+  if (!pid || !Number.isFinite(Number(at))) {
+    try { return Date.now() - statSync(file).mtimeMs > LEASE_TTL_MS } catch { return false }
+  }
+  return !pidAlive(pid) || Date.now() - Number(at) > LEASE_TTL_MS
+}
+
+/**
+ * Take this session's lease. Returns a release function, or null when someone else holds it and
+ * `waitMs` ran out. Timestamps are REAL time (a lock file outlives any injected clock).
+ *
+ * Crash recovery: a lease whose holder is dead, or older than LEASE_TTL_MS, is broken and retaken. The
+ * release only unlinks a file that still carries OUR token, so a lease broken underneath us is never
+ * deleted from under its new holder.
+ */
+export async function acquireLease(dir, key, { waitMs = 0, pollMs = 25 } = {}) {
+  const file = join(leaseDir(dir), `${key}.lock`)
+  try { mkdirSync(leaseDir(dir), { recursive: true, mode: 0o700 }) } catch { return null }
+  const token = `${process.pid} ${Date.now()} ${randomBytes(4).toString('hex')}`
+  const deadline = Date.now() + waitMs
+  let breaks = 0
+  for (;;) {
+    try {
+      const fd = openSync(file, 'wx', 0o600)
+      try { writeFileSync(fd, token) } finally { closeSync(fd) }
+      return () => { try { if (readFileSync(file, 'utf8') === token) unlinkSync(file) } catch { /* broken or gone */ } }
+    } catch (e) {
+      if (e?.code !== 'EEXIST') return null
+    }
+    if (leaseIsStale(file) && breaks < 3) {
+      breaks += 1
+      try { unlinkSync(file) } catch { /* raced with another breaker */ }
+      continue
+    }
+    if (Date.now() >= deadline) return null
+    await new Promise((r) => setTimeout(r, pollMs))
+  }
+}
+
+/** Run `fn` under the session lease. `fn` receives true if the lease was actually held. */
+export async function withLease(dir, key, opts, fn) {
+  const release = await acquireLease(dir, key, opts)
+  try { return await fn(Boolean(release)) } finally { release?.() }
+}
+
+// ── what has already been recorded for a session ─────────────────────────────────────────────────
+//
+// The high-water mark of generations this machine has successfully posted for a session, kept after the
+// queue entry is gone. It is how a poster knows its own capture is already superseded, and it is checked
+// under the lease so the answer cannot go stale between the check and the POST.
+
+const postedDir = (dir) => join(dir, '_posted')
+
+export function readPostedGen(dir, key) {
+  try { return JSON.parse(readFileSync(join(postedDir(dir), `${key}.json`), 'utf8')).gen ?? null } catch { return null }
+}
+
+/** Raise the mark. Monotonic: an older generation never lowers it. Call under the lease. */
+export function raisePostedGen(dir, key, gen) {
+  if (!key || !gen) return
+  const cur = readPostedGen(dir, key)
+  if (cur && cmpGen(cur, gen) >= 0) return
+  try { atomicWrite(postedDir(dir), `${key}.json`, JSON.stringify({ gen, at: Date.now() })) } catch { /* best-effort */ }
+}
+
+/** Has a NEWER capture of this session already been recorded? Then this entry is finished, not pending. */
+export function supersededByPosted(dir, entry) {
+  const posted = readPostedGen(dir, entry.key)
+  if (!posted || cmpGen(posted, entry.gen) <= 0) return false
+  removeEntry(dir, entry.key, entry.rev)
+  return true
+}
+
+/** A claim is live while the process that made it is alive and it has not outlived CLAIM_TTL_MS. */
+export function claimIsLive(claim) {
+  if (!claim) return false
+  if (!pidAlive(claim.pid)) return false
+  return Date.now() - (claim.at ?? 0) < CLAIM_TTL_MS
+}
+
+// ── the drop ledger ──────────────────────────────────────────────────────────────────────────────
+
+export function readDropped(dir, now = Date.now()) {
+  try {
+    const list = JSON.parse(readFileSync(join(dir, DROPPED_FILE), 'utf8'))
+    return Array.isArray(list) ? list.filter((d) => now - (d?.at ?? 0) <= DROPPED_WINDOW_DAYS * DAY_MS) : []
+  } catch {
+    return []
+  }
+}
+
+export function recordDropped(dir, { sessionId, reason }, now = Date.now()) {
+  try {
+    const list = [...readDropped(dir, now), { at: now, sessionId: sessionId ?? null, reason: String(reason ?? 'unknown') }]
+    atomicWrite(dir, DROPPED_FILE, JSON.stringify(list.slice(-DROPPED_MAX)))
+  } catch { /* a ledger write must never cost a capture */ }
+}
+
+// ── enqueue + bounds ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Queue (or re-queue) a capture that did not land. Returns the entry written, or null if it could not
+ * be stored — in which case it is in the drop ledger, so it is still reported.
+ *
+ * COMPARE-AND-SET on the generation: if a NEWER turn of this session is already queued, this older one
+ * is not written over it, and the newer entry is returned carrying `superseded: true`. That is the fix
+ * for an out-of-order worker replacing a newer queued capture with its own staler content. Nothing is
+ * lost by refusing: the newer turn read a superset of this transcript.
+ *
+ * `countsAsFailure: false` is for a summarizer that was merely BUSY (another session held the lock):
+ * that resolves itself and is not evidence of anything wrong, so it must not march an entry toward
+ * being flagged.
+ *
+ * Call under the session lease where one is available; the read-compare-write is re-checked here, so a
+ * best-effort call without the lease is still ordered against whatever is on disk at this moment.
+ */
+export function enqueueCapture(dir, item, { now = Date.now(), random = Math.random, log = () => {} } = {}) {
+  const { sessionId, kind, reason, detail = null, common, transcript, body, retryAfterMs = null, countsAsFailure = true } = item
+  const key = entryKey(sessionId)
+  const gen = item.gen ?? makeGen(0, now)
+  const prev = readEntry(dir, key)
+  if (prev && cmpGen(prev.gen, gen) > 0) {
+    log(`${sessionId ?? key}: a NEWER turn of this session is already queued (${prev.kind}) — this older capture is not written over it`)
+    return { ...prev, superseded: true }
+  }
+  const attempts = (prev?.attempts ?? 0) + (countsAsFailure ? 1 : 0)
+  const entry = {
+    v: 1,
+    key,
+    rev: newRev(),
+    gen,
+    sessionId: sessionId ?? null,
+    kind,
+    reason: String(reason ?? 'unknown'),
+    detail: detail ? String(detail).slice(0, 300) : null,
+    attempts,
+    firstQueuedAt: prev?.firstQueuedAt ?? now,
+    capturedAt: now,
+    updatedAt: now,
+    nextAttemptAt: now + Math.max(queueBackoffMs(Math.max(1, attempts), random), retryAfterMs ?? 0),
+    ...(kind === 'summarize' ? { common, transcript } : { body }),
+  }
+  try {
+    if (!writeEntry(dir, entry)) {
+      recordDropped(dir, { sessionId, reason: 'too large to queue' }, now)
+      log(`DROPPED ${sessionId ?? key} — the capture is larger than ${QUEUE_MAX_ENTRY_BYTES} bytes and cannot be queued`)
+      return null
+    }
+  } catch (e) {
+    recordDropped(dir, { sessionId, reason: 'queue write failed' }, now)
+    log(`DROPPED ${sessionId ?? key} — could not write the local queue: ${e instanceof Error ? e.message : String(e)}`)
+    return null
+  }
+  if (isFlagged(entry) && !isFlagged(prev)) {
+    log(`FLAGGED ${sessionId ?? key} after ${attempts} failed attempts [${entry.reason}] — kept, retried, and reported at the next session start`)
+  }
+  enforceBounds(dir, { now: () => now, log, keep: key })
+  return entry
+}
+
+/** Re-schedule an entry after another failed attempt. No-op if a newer turn replaced it meanwhile.
+ *  Also clears any claim: the attempt that held it is over. */
+function bumpEntry(dir, entry, { reason, detail, retryAfterMs = null, patch = {} }, { now, random, log }) {
+  const cur = readEntry(dir, entry.key)
+  if (!cur || cur.rev !== entry.rev) return null
+  const attempts = (cur.attempts ?? 0) + 1
+  const next = {
+    ...cur,
+    ...patch,
+    claim: undefined,
+    rev: newRev(),
+    reason: String(reason ?? cur.reason),
+    detail: detail ? String(detail).slice(0, 300) : cur.detail,
+    attempts,
+    updatedAt: now,
+    nextAttemptAt: now + Math.max(queueBackoffMs(attempts, random), retryAfterMs ?? 0),
+  }
+  try { writeEntry(dir, next) } catch { return null }
+  if (isFlagged(next) && !isFlagged(cur)) {
+    log(`FLAGGED ${cur.sessionId ?? cur.key} after ${attempts} failed attempts [${next.reason}] — kept, retried, and reported at the next session start`)
+  }
+  return next
+}
+
+/**
+ * Enforce age and count. Everything forced out is RECORDED in the drop ledger and logged — a bound is
+ * the one place this queue loses anything, so it is exactly where it must say so.
+ */
+export function enforceBounds(dir, { now = Date.now, log = () => {}, keep = null } = {}) {
+  const t = now()
+  // Stale temp files from a process killed mid-write.
+  try {
+    for (const name of readdirSync(dir)) {
+      if (!name.endsWith('.tmp')) continue
+      try { if (t - statSync(join(dir, name)).mtimeMs > STALE_TMP_MS) unlinkSync(join(dir, name)) } catch { /* raced */ }
+    }
+  } catch { return }
+  pruneSidecars(dir)
+  let entries = readEntries(dir, { log, now })
+  for (const e of entries) {
+    if (t - (e.capturedAt ?? e.firstQueuedAt ?? t) > QUEUE_MAX_AGE_DAYS * DAY_MS) {
+      // rev-guarded: if a live capture rewrote this entry since we listed it, leave the newer one alone.
+      if (!removeEntry(dir, e.key, e.rev)) continue
+      recordDropped(dir, { sessionId: e.sessionId, reason: `older than ${QUEUE_MAX_AGE_DAYS} days (last: ${e.reason})` }, t)
+      log(`DROPPED ${e.sessionId ?? e.key} — queued ${QUEUE_MAX_AGE_DAYS}+ days without being recorded (last reason: ${e.reason})`)
+    }
+  }
+  entries = readEntries(dir, { log, now })
+  if (entries.length <= QUEUE_MAX_ENTRIES) return
+  const evictable = entries.filter((e) => e.key !== keep).sort((a, b) => (a.firstQueuedAt ?? 0) - (b.firstQueuedAt ?? 0))
+  for (const e of evictable.slice(0, entries.length - QUEUE_MAX_ENTRIES)) {
+    if (!removeEntry(dir, e.key, e.rev)) continue
+    recordDropped(dir, { sessionId: e.sessionId, reason: `queue full (${QUEUE_MAX_ENTRIES} entries; last: ${e.reason})` }, t)
+    log(`DROPPED ${e.sessionId ?? e.key} — the local queue is full (${QUEUE_MAX_ENTRIES}); oldest evicted (last reason: ${e.reason})`)
+  }
+}
+
+/**
+ * The two sidecar directories, pruned on REAL time (they are filesystem artifacts, not scheduled work):
+ *   _posted/  one tiny mark per session, kept as long as an entry could still be queued for it
+ *   _locks/   released on exit, so anything this old is a lease whose process died
+ */
+function pruneSidecars(dir) {
+  const real = Date.now()
+  for (const [sub, maxAge] of [[postedDir(dir), QUEUE_MAX_AGE_DAYS * DAY_MS], [leaseDir(dir), 10 * LEASE_TTL_MS]]) {
+    try {
+      for (const name of readdirSync(sub)) {
+        try { if (real - statSync(join(sub, name)).mtimeMs > maxAge) unlinkSync(join(sub, name)) } catch { /* raced */ }
+      }
+    } catch { /* no sidecar dir yet */ }
+  }
+}
+
+// ── /api/ingest: what an answer means ────────────────────────────────────────────────────────────
+//
+// IDEMPOTENCY — CONCLUSION (read against web/app/api/ingest/route.ts, 2026-09-11):
+// /api/ingest DEDUPES a session capture on its sessionId, on every path it can take:
+//   · records     `insert … on conflict (org_id, dedupe_key) do update`, dedupe_key = `${source}:${sessionId}`.
+//                 A replay rewrites the same row. The post-commit project→actor edge strengthens only
+//                 when `inserted`, so a replay does not double-count it either.
+//   · private     the intake unit is keyed by intakeUnitKey(), which is the sessionId when present;
+//     intake      a re-push answers `duplicate` / `updated` against the same unit.
+//   · staged      unrouted captures stage under the same `${source}:${sessionId}` key.
+// And the Stop hook has ALWAYS re-posted the same session on every turn, relying on exactly this.
+// So a capture WITH a sessionId is idempotent: it may be retried after a 5xx, a broken connection or
+// one timeout, in-process or from this queue.
+//
+// ⚠ WITHOUT a sessionId it is NOT: the records key falls back to occurred_at (stable only if sent) and
+// intakeUnitKey falls back to a key containing Date.now() — a replay mints a second unit. Such a capture
+// is replayed only after failures where the request provably never reached the server (a refused
+// connection, 429/503, an infrastructure 403, a 401). Claude Code and Codex always send a session id,
+// so this is the rare path, and it fails toward a REPORTED drop rather than a silent duplicate.
+
+/** The policy for every capture POST. Worst case ~25 s in a DETACHED worker that nobody waits on —
+ *  less when the caller has less left (the drain passes its remaining budget). */
+export function ingestPolicy(body, budgetMs = INGEST_BUDGET_MS) {
+  const idempotent = typeof body?.sessionId === 'string' && body.sessionId.length > 0
+  const budget = Math.max(1, Math.min(INGEST_BUDGET_MS, Number.isFinite(budgetMs) ? budgetMs : INGEST_BUDGET_MS))
+  return { retries: 1, idempotent, budgetMs: budget }
+}
+
+/**
+ * PURE. Classify an /api/ingest HTTP answer:
+ *   recorded  a session record exists (the only success)
+ *   final     the server decided — staged, skipped, discarded, filed as an intake unit. Retrying cannot
+ *             change it; any older queued turn of the same session is superseded.
+ *   retry     not recorded, and a later replay is safe and may succeed
+ *   rejected  not recorded, and a replay is unsafe or pointless (a 4xx verdict). Reported as a drop.
+ */
+export function classifyIngestResponse(status, contentType, raw, { repo = 'general', requestId = null, idempotent = false } = {}) {
+  if (status >= 200 && status < 300) {
+    let j = null
+    try { j = JSON.parse(raw) } catch { j = null }
+    // A 2xx FROM /api/ingest DOES NOT MEAN A RECORD EXISTS. The route answers `ok: true` on at least
+    // six outcomes and only one of them writes a session record:
+    //
+    //   { ok: true, id, inserted, title }            -> RECORDED (the only success)
+    //   { ok: true, staged: true, id, reason }       -> held in staged_records, NOT recorded
+    //   { ok: true, skipped: '<why>' }               -> no-op session / past the ingest horizon
+    //   { ok: false, skipped: '<why>' }              -> connector excluded from this brain
+    //   { ok: true, discarded: true }                -> tombstoned by private intake
+    //   { ok: true, queued: true }                   -> accepted for later work
+    //   { ok: true, via: 'private_intake', intakeItemId } -> an intake unit, not a session record
+    //
+    // The old line read `j.inserted ? 'captured' : 'updated'`, so EVERY one of the six non-writing
+    // outcomes printed "updated" — the word for a successful upsert. Worse, `.json().catch(() => ({}))`
+    // means an unparseable body also yields `{}` and therefore also printed "updated". Verified against
+    // prod 2026-08-19: a capture printed `cortex: updated "general" → general` for a session that has no
+    // row in `records` and none in `staged_records` either.
+    //
+    // NOTE `staged` CARRIES AN `id`, so testing for an id alone is not enough — that id is the
+    // staged_records row, not a record. This is the same false-success defect already fixed once in
+    // log_session ("`inserted` is merely falsy when nothing is recorded — an agent reported a session as
+    // saved when it was not"); the fix was applied there and not here. The server route already knew:
+    // its own comment at the no_route_for_source branch says "`{ok: true}` reads as success to
+    // everything that is not looking closely" and notes 84 rows accumulating behind that wording.
+    // (Moved here from capture.mjs with the classifier, 2026-09-11.)
+    if (j && j.id && !j.staged) return { outcome: 'recorded', line: `${j.inserted ? 'captured' : 'updated'} "${j.title ?? repo}" → ${repo}` }
+    if (j && j.staged) return { outcome: 'final', line: `NOT RECORDED — staged (${j.reason ?? 'no reason given'}); staged session logs are not drainable by /api/staged/promote` }
+    if (j && j.skipped) return { outcome: 'final', line: `NOT RECORDED — server skipped: ${j.skipped}` }
+    if (j && j.discarded) return { outcome: 'final', line: 'NOT RECORDED — discarded by private intake' }
+    // Accepted before anything was written, with no record id to prove it ever was. Replaying the
+    // same session is harmless (idempotent) and is the only way to find out.
+    if (j && j.queued) {
+      return idempotent
+        ? { outcome: 'retry', reason: 'accepted without a record (queued)', line: 'NOT RECORDED — queued for later processing; kept in the local queue until a record exists' }
+        : { outcome: 'final', line: 'NOT RECORDED — queued for later processing' }
+    }
+    if (j && j.intakeItemId) return { outcome: 'final', line: `not a session record — filed as private intake unit ${j.intakeItemId}` }
+    // Unparseable or unrecognised 2xx. Deliberately NOT success — that is what the old code laundered
+    // into "updated".
+    return {
+      outcome: idempotent ? 'retry' : 'rejected',
+      reason: 'unrecognised response',
+      line: `NOT RECORDED — unrecognised 2xx response: ${String(raw).slice(0, 200)}`,
+    }
+  }
+  const d = classify(status, contentType, raw, requestId)
+  const klass = responseRetryClass(status, contentType)
+  const line = `NOT RECORDED — ingest failed: ${d.message}`
+  // Refused (429/503/infra 403) and 401 were never processed: always safe to replay. A 401 is kept
+  // rather than dropped because an expired token is fixed by logging in again — and the capture
+  // should still be here when that happens (the same call obligations_worker makes).
+  if (klass === 'refused') return { outcome: 'retry', reason: `HTTP ${status}`, line }
+  if (status === 401) return { outcome: 'retry', reason: 'token rejected (HTTP 401)', line }
+  if (klass === 'server') return idempotent ? { outcome: 'retry', reason: `HTTP ${status}`, line } : { outcome: 'rejected', reason: `HTTP ${status}, no session id to replay safely`, line }
+  return { outcome: 'rejected', reason: `rejected (HTTP ${status})`, line }
+}
+
+/**
+ * POST one capture. Never throws. The single exit through which every capture body leaves the machine,
+ * which is why D8's rule is enforced HERE and not only at the call sites: whatever a caller passes, a
+ * `transcript` field is stripped before it is serialized.
+ */
+export async function postIngest({ base, token, body, repo = 'general', fetchImpl = fetchCortex, budgetMs = INGEST_BUDGET_MS }) {
+  const { transcript: _neverSent, ...safe } = body ?? {} // D8: the transcript never leaves this machine
+  const policy = ingestPolicy(safe, budgetMs)
+  let res
+  try {
+    res = await fetchImpl(`${base}/api/ingest`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify(safe),
+      timeoutMs: Math.min(10_000, policy.budgetMs),
+    }, policy)
+  } catch (e) {
+    // No response at all. Safe to replay if nothing reached the server, or if the body dedupes.
+    const neverSent = e?.retryClass === 'connect'
+    const reason = e?.retryClass === 'timeout' ? 'timed out' : neverSent ? 'unreachable' : 'connection failed'
+    const line = `NOT RECORDED — ${e instanceof Error ? e.message : String(e)}`
+    return neverSent || policy.idempotent
+      ? { outcome: 'retry', reason, line, retryAfterMs: null }
+      : { outcome: 'rejected', reason: `${reason}, no session id to replay safely`, line }
+  }
+  const raw = await res.text().catch(() => '')
+  const c = classifyIngestResponse(res.status, res.headers.get('content-type'), raw, {
+    repo, requestId: res.headers.get('x-vercel-id'), idempotent: policy.idempotent,
+  })
+  return c.outcome === 'retry' ? { ...c, retryAfterMs: parseRetryAfter(res.headers.get('retry-after')) } : c
+}
+
+// ── drain ────────────────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Retry what is due. Called after every detached capture, so it must be cheap when the queue is empty
+ * (one readdir) and must never throw. Health signals from the capture that just ran decide what is
+ * worth trying: if the server just failed, no POSTs; if the summarizer just failed, no `claude -p`.
+ * A summarizer that just WORKED is also why a summarize entry may skip its schedule — the thing it was
+ * waiting for is demonstrably back.
+ *
+ * 🔴 NOTHING IS POSTED FROM THE LISTING. The listing is a snapshot, and a summarize can take minutes, so
+ * by upload time a live capture of that session may have posted something newer and rewritten or removed
+ * the entry. Every upload therefore happens under this session's LEASE, against a FRESH read, and only
+ * after checking `_posted/` — so a digest read minutes ago can never overwrite a newer record. A
+ * summarize is additionally CLAIMED first, so two drains don't spend two `claude -p` runs on one entry.
+ *
+ * deps: { dir, now, random, log, post(body, entry, {budgetMs}) → outcome,
+ *         summarize(transcript, {timeoutMs}) → {extracted}|{noop}|{unavailable},
+ *         buildBody(common, extracted, transcript) → body, skipKey, serverHealthy, summarizerHealthy,
+ *         allowSummarize, maxPosts, maxSummaries, budgetMs }
+ */
+export async function drainCaptureQueue(deps) {
+  const {
+    dir, now = Date.now, random = Math.random, log = () => {},
+    post, summarize, buildBody,
+    skipKey = null, serverHealthy = null, summarizerHealthy = null, allowSummarize = false,
+    maxPosts = DRAIN_MAX_POSTS, maxSummaries = DRAIN_MAX_SUMMARIES, budgetMs = DRAIN_BUDGET_MS,
+    leaseWaitMs = DRAIN_LEASE_WAIT_MS,
+  } = deps
+  const result = { recorded: 0, final: 0, dropped: 0, failed: 0, summarized: 0, superseded: 0, deferred: 0 }
+  try {
+    enforceBounds(dir, { now, log })
+    const startedAt = now()
+    const remaining = () => budgetMs - (now() - startedAt)
+    const entries = readEntries(dir, { log, now })
+      .filter((e) => e.key !== skipKey)
+      .sort((a, b) => (a.firstQueuedAt ?? 0) - (b.firstQueuedAt ?? 0))
+    let serverOk = serverHealthy !== false
+    let summarizerOk = summarizerHealthy !== false
+    let posts = 0
+    let summaries = 0
+    const ctx = () => ({ now: now(), random, log })
+
+    /** Act on a POST outcome. Runs under the lease, against the entry that was actually posted. */
+    const settle = (out, entry, who) => {
+      if (out.outcome === 'recorded' || out.outcome === 'final') {
+        raisePostedGen(dir, entry.key, entry.gen)
+        removeIfNotNewer(dir, entry.key, entry.gen)
+        if (out.outcome === 'recorded') {
+          result.recorded += 1
+          log(`queued ${who} recorded on attempt ${(entry.attempts ?? 0) + 1}: ${out.line}`)
+        } else {
+          result.final += 1
+          log(`queued ${who} ${out.line}`)
+        }
+      } else if (out.outcome === 'rejected') {
+        removeEntry(dir, entry.key, entry.rev)
+        recordDropped(dir, { sessionId: entry.sessionId, reason: out.reason }, now())
+        result.dropped += 1
+        log(`DROPPED queued ${who} — ${out.line}`)
+      } else {
+        serverOk = false // the server just failed: leave everything else for a later run
+        const next = bumpEntry(dir, entry, { reason: out.reason, retryAfterMs: out.retryAfterMs }, ctx())
+        result.failed += 1
+        log(`queued ${who} still ${out.line} (attempt ${next?.attempts ?? '?'})`)
+      }
+    }
+
+    for (const listed of entries) {
+      if (remaining() < DRAIN_MIN_POST_MS) break
+      if (!serverOk || posts >= maxPosts) break // every remaining path needs a POST
+
+      const who = listed.sessionId ?? listed.key
+
+      if (listed.kind === 'summarize') {
+        if (!allowSummarize || !summarizerOk || summaries >= maxSummaries) continue
+        if (now() - (listed.updatedAt ?? 0) < SUMMARIZE_SETTLE_MS) continue // session likely still live
+        if (summarizerHealthy !== true && (listed.nextAttemptAt ?? 0) > now()) continue
+        // A summarize is the one step this loop cannot interrupt, so it is only STARTED when the budget
+        // can pay for it AND for the upload that follows. Otherwise it waits for the next run.
+        if (remaining() < DRAIN_MIN_SUMMARY_MS + DRAIN_POST_RESERVE_MS) {
+          result.deferred += 1
+          log(`queued ${who}: ${Math.round(remaining() / 1000)}s of drain budget left, not enough to summarize — left for a later run`)
+          continue
+        }
+        // CLAIM it (under the lease) so a second drain does not summarize the same entry.
+        const claimed = await claimForSummary(dir, listed, { log, now })
+        if (!claimed) continue
+        summaries += 1
+        const timeoutMs = Math.max(0, remaining() - DRAIN_POST_RESERVE_MS)
+        const s = await summarize(claimed.transcript, { timeoutMs })
+        // The summary (if any) is built OUTSIDE the lease: obligation review can spawn `claude` too.
+        const body = s?.extracted
+          ? { ...(await buildBody(claimed.common, s.extracted, claimed.transcript)), occurredAt: new Date(claimed.capturedAt ?? now()).toISOString() }
+          : null
+        await withLease(dir, listed.key, { waitMs: Math.min(leaseWaitMs, Math.max(0, remaining())) }, async (held) => {
+          if (!held) { log(`queued ${who}: summarized, but this session's lease stayed busy — left for a later run`); return }
+          // REVALIDATE against a fresh read: anything that touched this entry while we summarized wins.
+          const cur = readEntry(dir, listed.key)
+          if (!cur || cur.rev !== claimed.rev) {
+            result.superseded += 1
+            log(`queued ${who}: replaced by a newer capture while it was being summarized — the stale digest was NOT posted`)
+            return
+          }
+          if (s?.unavailable) {
+            if (s.unavailable.reason === 'busy') {
+              writeEntry(dir, { ...cur, claim: undefined, rev: newRev() }) // release the claim, no attempt spent
+              log(`queued ${who}: summarizer busy, will retry`)
+              return
+            }
+            summarizerOk = false
+            const next = bumpEntry(dir, cur, { reason: s.unavailable.reason, detail: s.unavailable.detail }, ctx())
+            result.failed += 1
+            log(`queued ${who} still NOT RECORDED — local summarization unavailable [${s.unavailable.reason}] (attempt ${next?.attempts ?? '?'})`)
+            return
+          }
+          if (s?.noop) {
+            removeEntry(dir, cur.key, cur.rev)
+            result.final += 1
+            log(`queued ${who} NOT RECORDED — summarizer classified it a no-op session`)
+            return
+          }
+          if (supersededByPosted(dir, cur)) {
+            result.superseded += 1
+            log(`queued ${who}: a newer capture of this session was recorded while it was being summarized — dropped without posting`)
+            return
+          }
+          result.summarized += 1
+          // Become an ingest entry BEFORE posting: the summary exists now, so a failed POST must never
+          // cost another `claude -p` run — and the transcript tail is no longer needed, so it goes.
+          const converted = { ...cur, kind: 'ingest', body, transcript: undefined, common: undefined, claim: undefined, rev: newRev(), updatedAt: now() }
+          writeEntry(dir, converted)
+          if (remaining() < DRAIN_MIN_POST_MS) {
+            result.deferred += 1
+            log(`queued ${who}: summarized, but the drain budget is spent — the upload is left for a later run`)
+            return
+          }
+          posts += 1
+          settle(await post(body, converted, { budgetMs: Math.min(INGEST_BUDGET_MS, remaining()) }), converted, who)
+        })
+        continue
+      }
+
+      if (listed.kind !== 'ingest') continue
+      if ((listed.nextAttemptAt ?? 0) > now()) continue
+      // try-only: a held lease means a live capture of this session is mid-upload. It will handle its
+      // own entry, and whatever it records is newer than what we hold.
+      await withLease(dir, listed.key, { waitMs: 0 }, async (held) => {
+        if (!held) return
+        const cur = readEntry(dir, listed.key)
+        if (!cur || cur.kind !== 'ingest' || (cur.nextAttemptAt ?? 0) > now()) return
+        if (claimIsLive(cur.claim)) return
+        if (supersededByPosted(dir, cur)) {
+          result.superseded += 1
+          log(`queued ${who}: a newer capture of this session is already recorded — dropped without posting`)
+          return
+        }
+        if (remaining() < DRAIN_MIN_POST_MS) return
+        posts += 1
+        settle(await post(cur.body, cur, { budgetMs: Math.min(INGEST_BUDGET_MS, remaining()) }), cur, who)
+      })
+    }
+  } catch (e) {
+    log(`queue drain stopped: ${e instanceof Error ? e.message : String(e)}`)
+  }
+  return result
+}
+
+/**
+ * Take an entry for summarizing: under the lease, confirm it is still the one we listed, unclaimed, and
+ * not already superseded, then stamp our claim on it. Returns the claimed entry (whose `rev` the drain
+ * revalidates after summarizing) or null to skip it.
+ *
+ * Crash mid-claim recovers by itself: `claimIsLive` is false as soon as the claiming process is gone,
+ * and in any case after CLAIM_TTL_MS, so the entry becomes claimable again with nothing to clean up.
+ */
+async function claimForSummary(dir, listed, { log, now }) {
+  return withLease(dir, listed.key, { waitMs: 0 }, async (held) => {
+    if (!held) return null
+    const cur = readEntry(dir, listed.key)
+    if (!cur || cur.kind !== 'summarize' || cur.rev !== listed.rev) return null
+    if (claimIsLive(cur.claim)) return null
+    if (supersededByPosted(dir, cur)) {
+      log(`queued ${cur.sessionId ?? cur.key}: a newer capture of this session is already recorded — dropped without summarizing`)
+      return null
+    }
+    const claimed = { ...cur, claim: { pid: process.pid, at: Date.now() }, rev: newRev() }
+    return writeEntry(dir, claimed) ? claimed : null
+  })
+}
+
+// ── what the next session start says ─────────────────────────────────────────────────────────────
+
+const SUMMARIZE_REASONS = {
+  'auth-expired': 'its Claude login expired — run `claude setup-token`',
+  'no-claude': '`claude` is not on PATH for the capture hook',
+  timeout: 'summarizing timed out',
+  busy: 'another session was summarizing',
+  exit: 'the `claude` summarizer exited with an error',
+  unparseable: 'the summary came back unreadable',
+  'no-output': 'the summarizer returned nothing',
+  disabled: 'CORTEX_SUMMARIZE_DISABLED is set',
+  'spawn-failed': 'the summarizer could not start',
+  threw: 'the summarizer crashed',
+  'no-result': 'the summarizer could not start',
+}
+
+/** The most common reason among `items`, as a human label, noting if others exist too. */
+function dominantReason(items, labels = {}) {
+  const counts = new Map()
+  for (const it of items) counts.set(it.reason, (counts.get(it.reason) ?? 0) + 1)
+  const [top] = [...counts.entries()].sort((a, b) => b[1] - a[1])
+  const label = labels[top?.[0]] ?? top?.[0] ?? 'unknown'
+  return counts.size > 1 ? `${label}; and ${counts.size - 1} other reason${counts.size - 1 === 1 ? '' : 's'}` : label
+}
+
+/** Read-only: what the queue holds, for the session-start line and `doctor`. Never throws. */
+export function queueStatus(dir = queueDir(), now = Date.now()) {
+  let entries = []
+  try {
+    entries = readdirSync(dir)
+      .filter((n) => n.endsWith('.json') && !n.startsWith('_'))
+      .map((n) => { try { return JSON.parse(readFileSync(join(dir, n), 'utf8')) } catch { return null } })
+      .filter((e) => e && e.key)
+  } catch { /* no queue yet */ }
+  const flagged = entries.filter(isFlagged)
+  return {
+    pending: entries.length,
+    flaggedSummarize: flagged.filter((e) => e.kind === 'summarize'),
+    flaggedIngest: flagged.filter((e) => e.kind === 'ingest'),
+    dropped: readDropped(dir, now),
+  }
+}
+
+/**
+ * PURE. The session-start lines — empty when there is nothing to report. Only FLAGGED entries and
+ * DROPS are reported: an entry on its first or second try is a retry in flight, not news.
+ */
+export function renderQueueNotice(status) {
+  const lines = []
+  const plural = (n, one, many) => (n === 1 ? one : many)
+  const s = status?.flaggedSummarize ?? []
+  if (s.length) {
+    lines.push(
+      `Agnoclast: ⚠ ${s.length} ${plural(s.length, 'session', 'sessions')} not captured yet: your AI couldn't summarize ` +
+      `${plural(s.length, 'it', 'them')} (${dominantReason(s, SUMMARIZE_REASONS)}). ` +
+      `${plural(s.length, "It'll", "They'll")} retry automatically.`,
+    )
+  }
+  const i = status?.flaggedIngest ?? []
+  if (i.length) {
+    lines.push(
+      `Agnoclast: ⚠ ${i.length} ${plural(i.length, 'session', 'sessions')} not captured yet: the upload to Agnoclast kept failing ` +
+      `(${dominantReason(i)}). ${plural(i.length, "It'll", "They'll")} retry automatically.`,
+    )
+  }
+  const d = status?.dropped ?? []
+  if (d.length) {
+    lines.push(
+      `Agnoclast: ⚠ ${d.length} session ${plural(d.length, 'capture', 'captures')} could not be saved and ` +
+      `${plural(d.length, 'was', 'were')} dropped in the last ${DROPPED_WINDOW_DAYS} days (${dominantReason(d)}). ` +
+      `Details: ~/.cortex/capture.log`,
+    )
+  }
+  return lines
+}