@try-works/dsh-memory 0.1.19 → 0.1.21

package/src/queue.js

@@ -1,602 +1,602 @@
-// The memory queue: an append-only journal of memory CANDIDATES and how far each got.
-//
-// This is NOT the memory record. Markdown under the layers is the record, and it stays
-// human-readable, git-diffable and hand-editable. This is the operational queue that feeds it:
-// every session that might become memory gets one entry, and that entry carries an explicit
-// STATE rather than a bag of flags a reader has to interpret.
-//
-// Why explicit states. The previous design stored a settled timestamp, a summary source, an
-// attempt counter, a review flag and a watermark in one record, and every question about it was
-// answered by DERIVING semantics: pending meant the timestamp was null, gave-up meant the
-// timestamp was set AND a review flag was set, failed meant an attempt counter crossed a
-// threshold, deferred meant a source string had a particular value. Each derivation is a place
-// to get it wrong, and several were: a failed summary was once indistinguishable from a
-// deliberate conclusion, and empty digests were consumed as if they were finished work.
-//
-// Why a journal rather than a map of current states. A map answers where is this now, and
-// nothing else. The journal also answers how it got there, which is what makes a stuck queue
-// visible: an entry that has been summarizing for a day is a different problem from one that
-// failed, and orphaned - a candidate whose material is gone - is invisible in a map that keeps
-// only terminal flags.
-import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'
-import { join } from 'node:path'
-
-/** Bump when the entry envelope changes shape; readers refuse unknown versions. */
-export const QUEUE_FORMAT = 1
-
-/**
- * The lifecycle of one memory candidate.
- *
- * pending and summarized are waiting states. summarizing and consolidating are in flight.
- * added and skipped are successful-terminal: the work is done, whether or not anything was
- * written. failed and orphaned are problem-terminal: work stopped for a reason a reader must
- * see. held is parking, used to stage a batch deliberately.
- */
-export const STATES = {
-  PENDING: 'pending',
-  SUMMARIZING: 'summarizing',
-  SUMMARIZED: 'summarized',
-  CONSOLIDATING: 'consolidating',
-  ADDED: 'added',
-  SKIPPED: 'skipped',
-  FAILED: 'failed',
-  ORPHANED: 'orphaned',
-  HELD: 'held',
-  ABANDONED: 'abandoned',
-}
-
-/** States from which nothing further happens without an explicit operator action. */
-export const TERMINAL = [STATES.ADDED, STATES.SKIPPED, STATES.ABANDONED]
-
-/** The only transitions the queue records. An illegal one is a bug, not a silent state. */
-const TRANSITIONS = {
-  [STATES.PENDING]: [STATES.SUMMARIZING, STATES.HELD, STATES.ORPHANED, STATES.SKIPPED],
-  [STATES.SUMMARIZING]: [STATES.SUMMARIZED, STATES.SKIPPED, STATES.FAILED, STATES.PENDING, STATES.ABANDONED],
-  // A candidate is a SESSION, and a session keeps producing material. So every resting state has
-  // exactly one exit back into the summarizing stage, meaning NEW MATERIAL ARRIVED. That is an
-  // external event rather than the queue driving itself, which is why these states still count as
-  // resting: nothing here reopens on its own.
-  [STATES.SUMMARIZED]: [STATES.CONSOLIDATING, STATES.SUMMARIZING, STATES.HELD, STATES.ORPHANED],
-  [STATES.CONSOLIDATING]: [STATES.ADDED, STATES.SKIPPED, STATES.FAILED, STATES.SUMMARIZED, STATES.ABANDONED],
-  [STATES.HELD]: [STATES.PENDING, STATES.SUMMARIZED, STATES.SUMMARIZING],
-  [STATES.ADDED]: [STATES.SUMMARIZING],
-  [STATES.SKIPPED]: [STATES.SUMMARIZING],
-  // A retry re-enters the stage that failed, which the entry records as its `from`. Requeueing
-  // (pending/summarized) is the heavier option for when the material itself must be rebuilt.
-  [STATES.FAILED]: [STATES.SUMMARIZING, STATES.CONSOLIDATING, STATES.PENDING, STATES.SUMMARIZED, STATES.ABANDONED, STATES.HELD],
-  [STATES.ORPHANED]: [STATES.SUMMARIZING, STATES.CONSOLIDATING, STATES.PENDING, STATES.SUMMARIZED, STATES.ABANDONED, STATES.HELD],
-  [STATES.ABANDONED]: [STATES.PENDING, STATES.SUMMARIZED, STATES.SUMMARIZING],
-}
-
-/** True when a transition from one state to another is allowed. */
-export function canTransition(from, to) {
-  // A candidate can be BORN waiting or BORN PARKED. Parked matters: staging a batch - the
-  // backfill hold - creates entries that must not be worked until an operator releases them,
-  // and refusing that would force the hold to live outside the queue it belongs to.
-  if (from === undefined) return to === STATES.PENDING || to === STATES.SUMMARIZED || to === STATES.HELD
-  return (TRANSITIONS[from] ?? []).includes(to)
-}
-
-/** Is this state finished? */
-export function isTerminal(state) { return TERMINAL.includes(state) }
-
-/**
- * Reason codes for a state that stopped. A state alone says work stopped; the reason says why,
- * which is the difference between a correct decision and a defect.
- */
-export const REASONS = {
-  THIN: 'thin',                   // delta below the signal floor: not worth a model call
-  TRIVIAL: 'trivial',             // a ping, or maintenance chatter
-  NO_DURABLE: 'no-durable',       // the model read it and concluded nothing was durable
-  EMPTY_REPLY: 'empty-reply',     // the model returned no text: a defect
-  UNPARSEABLE: 'unparseable',     // the reply did not follow the protocol
-  NO_MODEL: 'no-model',           // no routable model produced text
-  ATTEMPTS: 'attempts-exhausted', // retried to the cap
-  DIGEST_GONE: 'digest-missing',  // the derived material was pruned before it was used
-  NO_ENTRIES: 'no-entries',       // consolidated, and the model added nothing
-  OPERATOR: 'operator',           // parked deliberately
-  NEEDS_APPROVAL: 'needs-approval', // parked because acting would cost more than a guess is worth
-  UNCLASSIFIED: 'unclassified',   // a legacy empty summary whose cause was never recorded
-  STALLED: 'stalled',             // a stage stopped reporting progress: presumed dead
-}
-
-export function queuePaths(root) {
-  return { journal: join(root, 'state', 'queue.jsonl'), snapshot: join(root, 'state', 'queue.json') }
-}
-
-/** One ISO stamp, taken once so an entry is internally consistent. */
-function stamp(now) { return (now ?? new Date()).toISOString() }
-
-/**
- * Read the journal and fold it into the current state of every candidate.
- * The last entry for an id wins; a repeat of the same state is a refresh, not a transition.
- * @returns a Map of id -> entry.
- */
-export function readQueue(root) {
-  const { journal } = queuePaths(root)
-  const current = new Map()
-  let raw = ''
-  try { raw = readFileSync(journal, 'utf8') } catch { return current }
-  for (const line of raw.split('\n')) {
-    if (line === '') continue
-    let entry
-    try { entry = JSON.parse(line) } catch { continue }
-    // Refuse a foreign envelope rather than misreading it as this format.
-    if (entry.v !== QUEUE_FORMAT || typeof entry.id !== 'string') continue
-    current.set(entry.id, entry)
-  }
-  return current
-}
-
-/** The journal's next position: monotonic, so order never depends on clocks. */
-export function nextSeq(root) {
-  const { journal } = queuePaths(root)
-  let last = 0
-  try {
-    for (const line of readFileSync(journal, 'utf8').split('\n')) {
-      if (line === '') continue
-      try { const e = JSON.parse(line); if (typeof e.seq === 'number' && e.seq > last) last = e.seq } catch {}
-    }
-  } catch {}
-  return last + 1
-}
-
-/**
- * Append one transition.
- *
- * The journal is APPEND-ONLY: a state change never rewrites history, so the queue's past stays
- * auditable. A snapshot beside it is rewritten on every change, because re-reading a long
- * journal to answer what is pending is wasteful. The snapshot is derived and never trusted as
- * the source of truth.
- *
- * @param root - store root.
- * @param id - the candidate's stable id (a session id, or a caller-chosen key).
- * @param state - one of STATES.
- * @param options.reason - required when work stops, so a decision can be told from a defect.
- * @param options.attempts - attempt counter for the stage that just ran.
- * @param options.detail - anything else worth keeping (tier, counts, model used).
- * @throws when the transition is illegal, rather than recording an unreachable state.
- */
-export function transition(root, id, state, options = {}) {
-  if (!Object.values(STATES).includes(state)) throw new Error('queue: unknown state ' + state)
-  if (typeof id !== 'string' || id === '') throw new Error('queue: an entry needs an id')
-  const current = readQueue(root).get(id)
-  const from = current?.state
-  if (from !== state && !canTransition(from, state)) {
-    throw new Error('queue: illegal transition ' + (from ?? '(new)') + ' -> ' + state + ' for ' + id)
-  }
-  const stops = state === STATES.FAILED || state === STATES.SKIPPED || state === STATES.ORPHANED
-  // The reason requirement is about ENTERING a stopping state, not restating one. A refresh -
-  // the same state recorded again to carry new detail - keeps the reason already on the entry,
-  // so demanding it afresh would make a legitimate refresh impossible.
-  if (stops && from !== state && !options.reason) {
-    throw new Error('queue: ' + state + ' needs a reason so a reader can tell a decision from a defect')
-  }
-  // The projection is LAST-ENTRY-WINS over standalone entries, so a transition that does not
-  // restate a field DROPS it. Carrying counters and detail forward is therefore not a
-  // convenience: without it an attempt counter resets on every hop and a retry cap can never be
-  // reached, which is precisely the bug this line fixes.
-  const attempts = options.attempts ?? current?.attempts
-  const detail = { ...(current?.detail ?? {}), ...(options.detail ?? {}) }
-  const retryable = state === STATES.FAILED || state === STATES.ORPHANED
-  const nextAttemptAt = options.nextAttemptAt ?? (retryable ? current?.nextAttemptAt : undefined)
-  const entry = {
-    v: QUEUE_FORMAT,
-    seq: nextSeq(root),
-    at: stamp(options.now),
-    id,
-    state,
-    ...(from === undefined || from === state ? {} : { from }),
-    ...((options.reason ?? (from === state ? current?.reason : undefined)) ? { reason: options.reason ?? current.reason } : {}),
-    ...(attempts === undefined ? {} : { attempts }),
-    ...(nextAttemptAt ? { nextAttemptAt } : {}),
-    ...(Object.keys(detail).length ? { detail } : {}),
-  }
-  mkdirSync(join(root, 'state'), { recursive: true })
-  const { journal } = queuePaths(root)
-  try { if (existsSync(journal) && statSync(journal).size > 4 * 1024 * 1024) renameSync(journal, journal + '.1') } catch {}
-  appendFileSync(journal, JSON.stringify(entry) + '\n')
-  writeSnapshot(root, readQueue(root))
-  return entry
-}
-
-/** Rewrite the derived snapshot. Cheap to regenerate, so it is never authoritative. */
-function writeSnapshot(root, current) {
-  try {
-    const { snapshot } = queuePaths(root)
-    const plain = {}
-    for (const [id, entry] of current) plain[id] = entry
-    writeFileSync(snapshot, JSON.stringify(plain, null, 2) + '\n', 'utf8')
-  } catch {}
-}
-
-/** Candidates currently in one of the given states. */
-export function byState(root, states) {
-  const want = new Set(Array.isArray(states) ? states : [states])
-  return [...readQueue(root).values()].filter((e) => want.has(e.state))
-}
-
-/** Work waiting to be done, in journal order. */
-export function pending(root) {
-  return byState(root, [STATES.PENDING, STATES.SUMMARIZED])
-}
-
-/** Entries that stopped for a reason a reader must see. */
-export function problems(root) {
-  // ABANDONED belongs here most of all. It is the end of the retry loop, so if it is not listed
-  // as a problem the work that gave up is the work nobody sees - which is exactly the silent
-  // loss this queue exists to prevent.
-  return byState(root, [STATES.FAILED, STATES.ORPHANED, STATES.ABANDONED])
-}
-
-/** A count per state, for status output. */
-export function summarize(root) {
-  const counts = {}
-  for (const entry of readQueue(root).values()) counts[entry.state] = (counts[entry.state] ?? 0) + 1
-  return counts
-}
-
-/**
- * Retry policy for work that failed.
- *
- * A failure is a statement about the ATTEMPT, not about the material, so nothing may rest at
- * failed: it is retried until it succeeds or until the attempts are spent. Backoff keeps a
- * broken candidate from spending a model call every tick while it is broken, and it grows
- * geometrically with a ceiling so a long outage does not push the next attempt days away.
- *
- * The cap exists because retrying forever is its own failure: it burns calls on work that has
- * demonstrably not worked. Reaching it produces ABANDONED, which stays visible and can be
- * revived - unlike the old design, where an exhausted candidate was marked done and vanished.
- */
-export const RETRY = {
-  maxAttempts: 5,
-  baseMs: 60 * 1000,
-  maxMs: 6 * 60 * 60 * 1000,
-}
-
-/**
- * Delay before the next attempt: base, then 4x each time, capped. Attempt 1 waits base,
- * attempt 2 waits 4x base, attempt 3 waits 16x base, and so on up to the ceiling.
- */
-export function backoffMs(attempts, { baseMs = RETRY.baseMs, maxMs = RETRY.maxMs } = {}) {
-  const n = Math.max(1, Number(attempts) || 1)
-  return Math.min(baseMs * Math.pow(4, n - 1), maxMs)
-}
-
-/**
- * What to do with one entry that stopped.
- * @returns {action, waitMs, attempts, reason} where action is retry, abandon or wait.
- */
-export function retryDecision(entry, now = Date.now(), options = {}) {
-  if (!entry) return { action: 'wait', attempts: 0, reason: 'unknown-entry' }
-  // ORPHANED is not retryable, and treating it as such is not a harmless extra attempt: the
-  // material is GONE, so a retry is guaranteed to fail while still costing a model call. What it
-  // needs is RECOVERY - rebuilding the digest from the source session - which is a different
-  // action that only the tool performs, and for a large session only with approval.
-  if (entry.state === STATES.ORPHANED) {
-    return { action: 'recover', attempts: Number(entry.attempts ?? 0), reason: entry.reason ?? 'digest-missing' }
-  }
-  if (entry.state !== STATES.FAILED) {
-    return { action: 'wait', attempts: Number(entry.attempts ?? 0), reason: 'not-retryable' }
-  }
-  const attempts = Number(entry.attempts ?? 1)
-  const max = options.maxAttempts ?? RETRY.maxAttempts
-  if (attempts >= max) return { action: 'abandon', attempts, reason: entry.reason ?? 'attempts' }
-  const due = entry.nextAttemptAt ? Date.parse(entry.nextAttemptAt) : 0
-  if (Number.isFinite(due) && due > 0 && now < due) return { action: 'wait', waitMs: due - now, attempts, reason: 'backoff' }
-  return { action: 'retry', attempts, reason: entry.reason ?? '' }
-}
-
-/**
- * Everything the cadence should act on now: waiting work, plus failures whose backoff has
- * expired. The retry loop lives here rather than in the caller, so a new caller cannot forget
- * to bring failed work back into the queue.
- */
-export function due(root, now = Date.now(), options = {}) {
-  const out = []
-  for (const entry of readQueue(root).values()) {
-    if (entry.state === STATES.PENDING || entry.state === STATES.SUMMARIZED) {
-      out.push({ entry, action: 'work' })
-      continue
-    }
-    const decision = retryDecision(entry, now, options)
-    // 'recover' is reported so a caller can SEE it, but it is not work: the cadence cannot perform
-    // it and must not mistake it for a digest to consolidate.
-    if (decision.action !== 'wait') out.push({ entry, action: decision.action, decision })
-  }
-  // PRIORITY, and it is not cosmetic. Callers take a BOUNDED batch, so queue order decides what
-  // is ever reached. A failure whose backoff expired is time-sensitive: the retry window is open
-  // NOW, and it opened because time passed rather than because new work arrived. Waiting work has
-  // no such deadline. Returning them in sequence order therefore starves retries completely -
-  // measured on a real store, the single retry sat at position 101 of 111 due items behind 34
-  // waiting ones, so with a batch of 3 it would never have run and 'we cannot let things stay
-  // failed' would have been true in the model and false in the system.
-  const RANK = { retry: 0, abandon: 1, work: 2, recover: 3 }
-  out.sort((a, b) => (RANK[a.action] ?? 9) - (RANK[b.action] ?? 9))
-  return out
-}
-
-/**
- * Record a failure and schedule the next attempt.
- *
- * The cap is enforced HERE so no caller can leave a candidate failed forever by forgetting to
- * check it: the last permitted failure becomes an abandonment automatically.
- */
-export function fail(root, id, options = {}) {
-  const current = readQueue(root).get(id)
-  const attempts = Number(options.attempts ?? ((current?.attempts ?? 0) + 1))
-  const max = options.maxAttempts ?? RETRY.maxAttempts
-  const state = attempts >= max ? STATES.ABANDONED : STATES.FAILED
-  const at = options.now ?? new Date()
-  return transition(root, id, state, {
-    reason: options.reason,
-    attempts,
-    now: options.now,
-    nextAttemptAt: new Date(at.getTime() + backoffMs(attempts)).toISOString(),
-    detail: { ...(options.detail ?? {}), ...(options.error ? { lastError: String(options.error).slice(0, 200) } : {}) },
-  })
-}
-
-/** Bring a candidate back into the queue. Works for failed, orphaned and abandoned work. */
-export function requeue(root, id, options = {}) {
-  return transition(root, id, options.state ?? STATES.PENDING, {
-    attempts: options.attempts ?? 0,
-    detail: { ...(options.detail ?? {}), requeuedAt: new Date().toISOString(), ...(options.why ? { why: options.why } : {}) },
-  })
-}
-
-// ---------------------------------------------------------------------------
-// Migration from the flag-based state file
-// ---------------------------------------------------------------------------
-
-/**
- * Classify one legacy record into a state. PURE: the caller supplies what is on disk.
- *
- * The guiding rule is CONSERVATIVE: where the old flags are ambiguous, prefer a state that
- * causes the work to be looked at again over one that declares it finished. The old design
- * could not distinguish a model concluding there was nothing durable from a model returning
- * nothing at all, so a record with no recorded reason must not be assumed to be a conclusion.
- *
- * @param record - one entry from sessions.json.
- * @param facts - { digestExists, digestHasSummary } describing the derived material.
- * @returns { state, reason, detail } with reason present whenever work stopped.
- */
-export function classifyLegacyRecord(record = {}, facts = {}) {
-  const detail = {
-    ...(record.tier ? { tier: record.tier } : {}),
-    ...(record.title ? { title: record.title } : {}),
-    ...(record.capturedAt ? { capturedAt: record.capturedAt } : {}),
-    ...(record.lastSeq === undefined ? {} : { lastSeq: record.lastSeq }),
-    ...(record.digestHash ? { digestHash: record.digestHash } : {}),
-    ...(record.digestFormat === undefined ? {} : { digestFormat: record.digestFormat }),
-    ...(record.eventCount === undefined ? {} : { eventCount: record.eventCount }),
-    ...(record.toolCallCount === undefined ? {} : { toolCallCount: record.toolCallCount }),
-    // The original settled time, kept because the legacy reader still asks WHEN, and a migration
-    // timestamp would silently rewrite history.
-    ...(record.consolidatedAt ? { legacyConsolidatedAt: record.consolidatedAt } : {}),
-    // ABSENT is not the same as null in the old shape: absent means no writer ever set the key,
-    // null means one explicitly wrote 'not yet'. The distinction is load-bearing for anything
-    // that asks whether a record was ever touched, so it is carried rather than flattened.
-    ...(('consolidatedAt' in record) ? {} : { noConsolidatedAtKey: true }),
-    legacy: true,
-  }
-  const attempts = record.consolidateAttempts ?? record.summaryAttempts;
-  const settled = Boolean(record.consolidatedAt);
-  const source = record.summarySource;
-  const reason = record.summaryReason;
-
-  // A candidate explicitly flagged for review is a known problem, whatever else it says.
-  if (record.needsReview) return { state: STATES.FAILED, reason: REASONS.ATTEMPTS, attempts, detail };
-
-  if (!settled) {
-    // Waiting work. What it is waiting FOR depends on whether material exists.
-    if (source === 'backfill') return { state: STATES.HELD, reason: REASONS.OPERATOR, attempts, detail };
-    if (!facts.digestExists) return { state: STATES.ORPHANED, reason: REASONS.DIGEST_GONE, attempts, detail };
-    if (facts.digestHasSummary) return { state: STATES.SUMMARIZED, attempts, detail };
-    return { state: STATES.PENDING, attempts, detail };
-  }
-
-  // Settled. The honest question is what it settled AS.
-  if (source === 'llm' || source === 'fallback' || source === 'backfill') {
-    return { state: STATES.ADDED, attempts, detail };
-  }
-  if (source === 'thin') return { state: STATES.SKIPPED, reason: REASONS.THIN, attempts, detail };
-  if (source === 'skipped') return { state: STATES.SKIPPED, reason: REASONS.TRIVIAL, attempts, detail };
-  if (source === 'none' || source === undefined) {
-    // Only a recorded no-durable conclusion is trusted as a decision. Anything else - including
-    // a legacy record that recorded no reason at all - becomes retryable work, because assuming
-    // it was a conclusion is exactly how content was lost before.
-    if (reason === REASONS.NO_DURABLE) return { state: STATES.SKIPPED, reason: REASONS.NO_DURABLE, attempts, detail };
-    // A KNOWN defect is retryable: retrying is the right response to it.
-    if (reason) return { state: STATES.FAILED, reason, attempts, detail };
-    // A missing digest is the operative fact, and it outranks the old flag. A digest is DERIVED,
-    // so its absence only matters when nothing came of the session: an `added` record whose digest
-    // was pruned is fine, but one that never produced entries has lost its material and is
-    // recoverable from the source session. Checking this only in the NOT-settled branch left 78
-    // recoverable candidates parked as ordinary waiting work, where nothing would ever look.
-    if (!facts.digestExists) return { state: STATES.ORPHANED, reason: REASONS.DIGEST_GONE, attempts, detail };
-    // Settled, no source, and the digest HAS content: consolidation marked it done. It stamps a
-    // timestamp without writing a source, so this is what a successful consolidation looks like.
-    if (facts.digestHasSummary) return { state: STATES.ADDED, attempts, detail };
-    // Material present but empty, cause never recorded. An UNKNOWN cause is parked, not retried.
-    // Marking these failed would put historical sessions back through summarization on a guess.
-    // sessions back through summarization on the strength of a guess about what went wrong -
-    // which is a backfill by another name, and a spend nobody asked for. The old writer recorded
-    // no reason, so a decision and a defect are genuinely indistinguishable here. Parked keeps
-    // them counted and visible and lets an operator decide, which is the conservative answer.
-    return { state: STATES.HELD, reason: REASONS.UNCLASSIFIED, attempts, detail };
-  }
-  // An unrecognised source is not silently trusted either.
-  return { state: STATES.FAILED, reason: REASONS.UNCLASSIFIED, attempts, detail };
-}
-
-export function legacyFacts(root, id, readDigest) {
-  const text = typeof readDigest === 'function' ? String(readDigest(root, id) ?? '') : '';
-  return { digestExists: text !== '', digestHasSummary: text.includes('summary (') };
-}
-
-/**
- * Build the entries the migration WOULD write, without writing anything.
- * @returns an array of { id, state, reason, attempts, detail }.
- */
-export function planMigration(root, sessions, readDigest) {
-  const existing = readQueue(root);
-  const plan = [];
-  for (const [id, record] of Object.entries(sessions ?? {})) {
-    if (existing.has(id)) continue;   // idempotent: never re-migrate what is already queued
-    const facts = legacyFacts(root, id, readDigest);
-    plan.push({ id, ...classifyLegacyRecord(record, facts) });
-  }
-  return plan;
-}
-
-/**
- * Apply a migration plan. Re-runnable: entries already present are skipped by planMigration,
- * and a state that is already current is recorded as a refresh rather than a transition.
- * @returns { written, counts } - how many entries were appended, and the state tally.
- */
-export function applyMigration(root, plan) {
-  let written = 0;
-  const counts = {};
-  for (const item of plan) {
-    counts[item.state] = (counts[item.state] ?? 0) + 1;
-    // seed, not transition: the state was reached in the OLD store, and the journal must not
-    // pretend it watched that happen.
-    seed(root, item.id, item.state, { reason: item.reason, attempts: item.attempts, detail: item.detail });
-    written++;
-  }
-  return { written, counts };
-}
-
-/**
- * Seed an entry that did not originate here.
- *
- * A migration imports candidates that already reached some state in the old store. Walking
- * each one through the lifecycle would FABRICATE a history it never had - a transition table
- * exists to describe what happened, not to be satisfied. So an imported entry is recorded
- * directly and marked `imported`, which keeps the journal honest: a reader can tell an entry
- * that was born here from one that arrived with its state already decided.
- *
- * The reason requirement still applies, so an import cannot smuggle in an unexplained stop.
- * @throws when the id already exists - seeding is not a way to overwrite live state.
- */
-export function seed(root, id, state, options = {}) {
-  if (!Object.values(STATES).includes(state)) throw new Error('queue: unknown state ' + state)
-  if (typeof id !== 'string' || id === '') throw new Error('queue: an entry needs an id')
-  const existing = readQueue(root).get(id)
-  if (existing) throw new Error('queue: ' + id + ' already exists in state ' + existing.state)
-  const stops = state === STATES.FAILED || state === STATES.SKIPPED || state === STATES.ORPHANED;
-  if (stops && !options.reason) throw new Error('queue: seeding ' + state + ' needs a reason')
-  const entry = {
-    v: QUEUE_FORMAT,
-    seq: nextSeq(root),
-    at: stamp(options.now),
-    id,
-    state,
-    imported: true,
-    ...(options.reason ? { reason: options.reason } : {}),
-    ...(options.attempts === undefined ? {} : { attempts: options.attempts }),
-    ...(options.detail ? { detail: options.detail } : {}),
-  }
-  mkdirSync(join(root, 'state'), { recursive: true })
-  appendFileSync(queuePaths(root).journal, JSON.stringify(entry) + '\n')
-  writeSnapshot(root, readQueue(root));
-  return entry;
-}
-
-// ---------------------------------------------------------------------------
-// Reading the queue through the old shape (a temporary compatibility layer)
-// ---------------------------------------------------------------------------
-
-/**
- * Project one queue entry into the legacy record shape.
- *
- * This is a SHIM and it is deliberately lossy in one direction only: it does not remove the
- * derivation this whole change exists to delete, it CONCENTRATES it in one place. The old
- * readers asked questions like is-this-settled, which used to be answered by inspecting a
- * timestamp; here that question is answered once, from the state, and every legacy caller keeps
- * working unchanged. Phase 8 deletes it.
- *
- * `consolidatedAt` is genuinely three-valued in the old shape and the difference is load-bearing:
- * a timestamp means settled, null means explicitly pending, and ABSENT means no write ever set
- * it. Collapsing null and absent would change what pendingDigests returns.
- */
-/**
- * Has this candidate's work been DEALT WITH?
- *
- * This was a projection artefact for too long: the shim derived it from a state so that legacy
- * readers could keep asking an old question. It is a real question with a real answer, so it lives
- * here and both the queue-native readers and the projection use it - one rule, not two that agree
- * by accident.
- *
- * Note what is NOT settled: PENDING and SUMMARIZED are waiting, ORPHANED needs recovery, and a
- * plain FAILED is retryable. Treating any of those as done is how work disappeared.
- */
-export function isSettled(entry) {
-  if (!entry) return false;
-  if (entry.state === STATES.ADDED || entry.state === STATES.SKIPPED) return true;
-  // Giving up SETTLES: the old code stamped a timestamp when the retry cap was reached.
-  if (entry.state === STATES.FAILED && entry.reason === REASONS.ATTEMPTS) return true;
-  // A migrated record that was settled in the OLD file stays settled for the old reader's
-  // purposes, even though its state now says it is parked.
-  if (entry.state === STATES.HELD && entry.reason === REASONS.UNCLASSIFIED) return true;
-  if (entry.imported && entry.detail?.legacyConsolidatedAt) return true;
-  return false;
-}
-
-export function advance(root, id, target, options = {}) {
-  const current = readQueue(root).get(id);
-  if (current?.state === target) return 0;
-  const path = shortestPath(current?.state, target);
-  if (!path) throw new Error('queue: no legal path from ' + (current?.state ?? '(new)') + ' to ' + target + ' for ' + id);
-  let steps = 0;
-  for (const step of path) {
-    const last = step === path[path.length - 1];
-    transition(root, id, step, last ? options : {});
-    steps++;
-  }
-  return steps;
-}
-
-/** Breadth-first search over the transition table. The graph is tiny, so this stays cheap. */
-function shortestPath(from, to) {
-  if (from === to) return [];
-  const queue = [[from, []]];
-  const seen = new Set([from]);
-  while (queue.length) {
-    const [at, path] = queue.shift();
-    for (const next of (at === undefined ? [STATES.PENDING, STATES.SUMMARIZED, STATES.HELD] : (TRANSITIONS[at] ?? []))) {
-      if (seen.has(next)) continue;
-      if (next === to) return [...path, next];
-      seen.add(next);
-      queue.push([next, [...path, next]]);
-    }
-  }
-  return undefined;
-}
-
-export const STAGE_TIMEOUT_MS = 30 * 60 * 1000
-
-/**
- * Return entries stranded in an in-progress state to the failure path.
- *
- * Called before every batch. Without it a crash during summarization or consolidation parks a
- * candidate forever in a state that reads as 'busy' and is, in fact, abandoned.
- * @returns the ids that were recovered.
- */
-export function reapStalled(root, now = Date.now(), timeoutMs = STAGE_TIMEOUT_MS) {
-  const recovered = [];
-  for (const entry of readQueue(root).values()) {
-    const inProgress = entry.state === STATES.SUMMARIZING || entry.state === STATES.CONSOLIDATING;
-    if (!inProgress) continue;
-    const started = Date.parse(entry.at);
-    if (!Number.isFinite(started) || now - started < timeoutMs) continue;
-    fail(root, entry.id, {
-      reason: REASONS.STALLED,
-      attempts: Number(entry.detail?.consolidateAttempts ?? entry.attempts ?? 0) + 1,
-      error: 'no progress for ' + Math.round((now - started) / 60000) + ' minutes',
-    });
-    recovered.push(entry.id);
-  }
-  return recovered;
-}
+// The memory queue: an append-only journal of memory CANDIDATES and how far each got.
+//
+// This is NOT the memory record. Markdown under the layers is the record, and it stays
+// human-readable, git-diffable and hand-editable. This is the operational queue that feeds it:
+// every session that might become memory gets one entry, and that entry carries an explicit
+// STATE rather than a bag of flags a reader has to interpret.
+//
+// Why explicit states. The previous design stored a settled timestamp, a summary source, an
+// attempt counter, a review flag and a watermark in one record, and every question about it was
+// answered by DERIVING semantics: pending meant the timestamp was null, gave-up meant the
+// timestamp was set AND a review flag was set, failed meant an attempt counter crossed a
+// threshold, deferred meant a source string had a particular value. Each derivation is a place
+// to get it wrong, and several were: a failed summary was once indistinguishable from a
+// deliberate conclusion, and empty digests were consumed as if they were finished work.
+//
+// Why a journal rather than a map of current states. A map answers where is this now, and
+// nothing else. The journal also answers how it got there, which is what makes a stuck queue
+// visible: an entry that has been summarizing for a day is a different problem from one that
+// failed, and orphaned - a candidate whose material is gone - is invisible in a map that keeps
+// only terminal flags.
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+/** Bump when the entry envelope changes shape; readers refuse unknown versions. */
+export const QUEUE_FORMAT = 1
+
+/**
+ * The lifecycle of one memory candidate.
+ *
+ * pending and summarized are waiting states. summarizing and consolidating are in flight.
+ * added and skipped are successful-terminal: the work is done, whether or not anything was
+ * written. failed and orphaned are problem-terminal: work stopped for a reason a reader must
+ * see. held is parking, used to stage a batch deliberately.
+ */
+export const STATES = {
+  PENDING: 'pending',
+  SUMMARIZING: 'summarizing',
+  SUMMARIZED: 'summarized',
+  CONSOLIDATING: 'consolidating',
+  ADDED: 'added',
+  SKIPPED: 'skipped',
+  FAILED: 'failed',
+  ORPHANED: 'orphaned',
+  HELD: 'held',
+  ABANDONED: 'abandoned',
+}
+
+/** States from which nothing further happens without an explicit operator action. */
+export const TERMINAL = [STATES.ADDED, STATES.SKIPPED, STATES.ABANDONED]
+
+/** The only transitions the queue records. An illegal one is a bug, not a silent state. */
+const TRANSITIONS = {
+  [STATES.PENDING]: [STATES.SUMMARIZING, STATES.HELD, STATES.ORPHANED, STATES.SKIPPED],
+  [STATES.SUMMARIZING]: [STATES.SUMMARIZED, STATES.SKIPPED, STATES.FAILED, STATES.PENDING, STATES.ABANDONED],
+  // A candidate is a SESSION, and a session keeps producing material. So every resting state has
+  // exactly one exit back into the summarizing stage, meaning NEW MATERIAL ARRIVED. That is an
+  // external event rather than the queue driving itself, which is why these states still count as
+  // resting: nothing here reopens on its own.
+  [STATES.SUMMARIZED]: [STATES.CONSOLIDATING, STATES.SUMMARIZING, STATES.HELD, STATES.ORPHANED],
+  [STATES.CONSOLIDATING]: [STATES.ADDED, STATES.SKIPPED, STATES.FAILED, STATES.SUMMARIZED, STATES.ABANDONED],
+  [STATES.HELD]: [STATES.PENDING, STATES.SUMMARIZED, STATES.SUMMARIZING],
+  [STATES.ADDED]: [STATES.SUMMARIZING],
+  [STATES.SKIPPED]: [STATES.SUMMARIZING],
+  // A retry re-enters the stage that failed, which the entry records as its `from`. Requeueing
+  // (pending/summarized) is the heavier option for when the material itself must be rebuilt.
+  [STATES.FAILED]: [STATES.SUMMARIZING, STATES.CONSOLIDATING, STATES.PENDING, STATES.SUMMARIZED, STATES.ABANDONED, STATES.HELD],
+  [STATES.ORPHANED]: [STATES.SUMMARIZING, STATES.CONSOLIDATING, STATES.PENDING, STATES.SUMMARIZED, STATES.ABANDONED, STATES.HELD],
+  [STATES.ABANDONED]: [STATES.PENDING, STATES.SUMMARIZED, STATES.SUMMARIZING],
+}
+
+/** True when a transition from one state to another is allowed. */
+export function canTransition(from, to) {
+  // A candidate can be BORN waiting or BORN PARKED. Parked matters: staging a batch - the
+  // backfill hold - creates entries that must not be worked until an operator releases them,
+  // and refusing that would force the hold to live outside the queue it belongs to.
+  if (from === undefined) return to === STATES.PENDING || to === STATES.SUMMARIZED || to === STATES.HELD
+  return (TRANSITIONS[from] ?? []).includes(to)
+}
+
+/** Is this state finished? */
+export function isTerminal(state) { return TERMINAL.includes(state) }
+
+/**
+ * Reason codes for a state that stopped. A state alone says work stopped; the reason says why,
+ * which is the difference between a correct decision and a defect.
+ */
+export const REASONS = {
+  THIN: 'thin',                   // delta below the signal floor: not worth a model call
+  TRIVIAL: 'trivial',             // a ping, or maintenance chatter
+  NO_DURABLE: 'no-durable',       // the model read it and concluded nothing was durable
+  EMPTY_REPLY: 'empty-reply',     // the model returned no text: a defect
+  UNPARSEABLE: 'unparseable',     // the reply did not follow the protocol
+  NO_MODEL: 'no-model',           // no routable model produced text
+  ATTEMPTS: 'attempts-exhausted', // retried to the cap
+  DIGEST_GONE: 'digest-missing',  // the derived material was pruned before it was used
+  NO_ENTRIES: 'no-entries',       // consolidated, and the model added nothing
+  OPERATOR: 'operator',           // parked deliberately
+  NEEDS_APPROVAL: 'needs-approval', // parked because acting would cost more than a guess is worth
+  UNCLASSIFIED: 'unclassified',   // a legacy empty summary whose cause was never recorded
+  STALLED: 'stalled',             // a stage stopped reporting progress: presumed dead
+}
+
+export function queuePaths(root) {
+  return { journal: join(root, 'state', 'queue.jsonl'), snapshot: join(root, 'state', 'queue.json') }
+}
+
+/** One ISO stamp, taken once so an entry is internally consistent. */
+function stamp(now) { return (now ?? new Date()).toISOString() }
+
+/**
+ * Read the journal and fold it into the current state of every candidate.
+ * The last entry for an id wins; a repeat of the same state is a refresh, not a transition.
+ * @returns a Map of id -> entry.
+ */
+export function readQueue(root) {
+  const { journal } = queuePaths(root)
+  const current = new Map()
+  let raw = ''
+  try { raw = readFileSync(journal, 'utf8') } catch { return current }
+  for (const line of raw.split('\n')) {
+    if (line === '') continue
+    let entry
+    try { entry = JSON.parse(line) } catch { continue }
+    // Refuse a foreign envelope rather than misreading it as this format.
+    if (entry.v !== QUEUE_FORMAT || typeof entry.id !== 'string') continue
+    current.set(entry.id, entry)
+  }
+  return current
+}
+
+/** The journal's next position: monotonic, so order never depends on clocks. */
+export function nextSeq(root) {
+  const { journal } = queuePaths(root)
+  let last = 0
+  try {
+    for (const line of readFileSync(journal, 'utf8').split('\n')) {
+      if (line === '') continue
+      try { const e = JSON.parse(line); if (typeof e.seq === 'number' && e.seq > last) last = e.seq } catch {}
+    }
+  } catch {}
+  return last + 1
+}
+
+/**
+ * Append one transition.
+ *
+ * The journal is APPEND-ONLY: a state change never rewrites history, so the queue's past stays
+ * auditable. A snapshot beside it is rewritten on every change, because re-reading a long
+ * journal to answer what is pending is wasteful. The snapshot is derived and never trusted as
+ * the source of truth.
+ *
+ * @param root - store root.
+ * @param id - the candidate's stable id (a session id, or a caller-chosen key).
+ * @param state - one of STATES.
+ * @param options.reason - required when work stops, so a decision can be told from a defect.
+ * @param options.attempts - attempt counter for the stage that just ran.
+ * @param options.detail - anything else worth keeping (tier, counts, model used).
+ * @throws when the transition is illegal, rather than recording an unreachable state.
+ */
+export function transition(root, id, state, options = {}) {
+  if (!Object.values(STATES).includes(state)) throw new Error('queue: unknown state ' + state)
+  if (typeof id !== 'string' || id === '') throw new Error('queue: an entry needs an id')
+  const current = readQueue(root).get(id)
+  const from = current?.state
+  if (from !== state && !canTransition(from, state)) {
+    throw new Error('queue: illegal transition ' + (from ?? '(new)') + ' -> ' + state + ' for ' + id)
+  }
+  const stops = state === STATES.FAILED || state === STATES.SKIPPED || state === STATES.ORPHANED
+  // The reason requirement is about ENTERING a stopping state, not restating one. A refresh -
+  // the same state recorded again to carry new detail - keeps the reason already on the entry,
+  // so demanding it afresh would make a legitimate refresh impossible.
+  if (stops && from !== state && !options.reason) {
+    throw new Error('queue: ' + state + ' needs a reason so a reader can tell a decision from a defect')
+  }
+  // The projection is LAST-ENTRY-WINS over standalone entries, so a transition that does not
+  // restate a field DROPS it. Carrying counters and detail forward is therefore not a
+  // convenience: without it an attempt counter resets on every hop and a retry cap can never be
+  // reached, which is precisely the bug this line fixes.
+  const attempts = options.attempts ?? current?.attempts
+  const detail = { ...(current?.detail ?? {}), ...(options.detail ?? {}) }
+  const retryable = state === STATES.FAILED || state === STATES.ORPHANED
+  const nextAttemptAt = options.nextAttemptAt ?? (retryable ? current?.nextAttemptAt : undefined)
+  const entry = {
+    v: QUEUE_FORMAT,
+    seq: nextSeq(root),
+    at: stamp(options.now),
+    id,
+    state,
+    ...(from === undefined || from === state ? {} : { from }),
+    ...((options.reason ?? (from === state ? current?.reason : undefined)) ? { reason: options.reason ?? current.reason } : {}),
+    ...(attempts === undefined ? {} : { attempts }),
+    ...(nextAttemptAt ? { nextAttemptAt } : {}),
+    ...(Object.keys(detail).length ? { detail } : {}),
+  }
+  mkdirSync(join(root, 'state'), { recursive: true })
+  const { journal } = queuePaths(root)
+  try { if (existsSync(journal) && statSync(journal).size > 4 * 1024 * 1024) renameSync(journal, journal + '.1') } catch {}
+  appendFileSync(journal, JSON.stringify(entry) + '\n')
+  writeSnapshot(root, readQueue(root))
+  return entry
+}
+
+/** Rewrite the derived snapshot. Cheap to regenerate, so it is never authoritative. */
+function writeSnapshot(root, current) {
+  try {
+    const { snapshot } = queuePaths(root)
+    const plain = {}
+    for (const [id, entry] of current) plain[id] = entry
+    writeFileSync(snapshot, JSON.stringify(plain, null, 2) + '\n', 'utf8')
+  } catch {}
+}
+
+/** Candidates currently in one of the given states. */
+export function byState(root, states) {
+  const want = new Set(Array.isArray(states) ? states : [states])
+  return [...readQueue(root).values()].filter((e) => want.has(e.state))
+}
+
+/** Work waiting to be done, in journal order. */
+export function pending(root) {
+  return byState(root, [STATES.PENDING, STATES.SUMMARIZED])
+}
+
+/** Entries that stopped for a reason a reader must see. */
+export function problems(root) {
+  // ABANDONED belongs here most of all. It is the end of the retry loop, so if it is not listed
+  // as a problem the work that gave up is the work nobody sees - which is exactly the silent
+  // loss this queue exists to prevent.
+  return byState(root, [STATES.FAILED, STATES.ORPHANED, STATES.ABANDONED])
+}
+
+/** A count per state, for status output. */
+export function summarize(root) {
+  const counts = {}
+  for (const entry of readQueue(root).values()) counts[entry.state] = (counts[entry.state] ?? 0) + 1
+  return counts
+}
+
+/**
+ * Retry policy for work that failed.
+ *
+ * A failure is a statement about the ATTEMPT, not about the material, so nothing may rest at
+ * failed: it is retried until it succeeds or until the attempts are spent. Backoff keeps a
+ * broken candidate from spending a model call every tick while it is broken, and it grows
+ * geometrically with a ceiling so a long outage does not push the next attempt days away.
+ *
+ * The cap exists because retrying forever is its own failure: it burns calls on work that has
+ * demonstrably not worked. Reaching it produces ABANDONED, which stays visible and can be
+ * revived - unlike the old design, where an exhausted candidate was marked done and vanished.
+ */
+export const RETRY = {
+  maxAttempts: 5,
+  baseMs: 60 * 1000,
+  maxMs: 6 * 60 * 60 * 1000,
+}
+
+/**
+ * Delay before the next attempt: base, then 4x each time, capped. Attempt 1 waits base,
+ * attempt 2 waits 4x base, attempt 3 waits 16x base, and so on up to the ceiling.
+ */
+export function backoffMs(attempts, { baseMs = RETRY.baseMs, maxMs = RETRY.maxMs } = {}) {
+  const n = Math.max(1, Number(attempts) || 1)
+  return Math.min(baseMs * Math.pow(4, n - 1), maxMs)
+}
+
+/**
+ * What to do with one entry that stopped.
+ * @returns {action, waitMs, attempts, reason} where action is retry, abandon or wait.
+ */
+export function retryDecision(entry, now = Date.now(), options = {}) {
+  if (!entry) return { action: 'wait', attempts: 0, reason: 'unknown-entry' }
+  // ORPHANED is not retryable, and treating it as such is not a harmless extra attempt: the
+  // material is GONE, so a retry is guaranteed to fail while still costing a model call. What it
+  // needs is RECOVERY - rebuilding the digest from the source session - which is a different
+  // action that only the tool performs, and for a large session only with approval.
+  if (entry.state === STATES.ORPHANED) {
+    return { action: 'recover', attempts: Number(entry.attempts ?? 0), reason: entry.reason ?? 'digest-missing' }
+  }
+  if (entry.state !== STATES.FAILED) {
+    return { action: 'wait', attempts: Number(entry.attempts ?? 0), reason: 'not-retryable' }
+  }
+  const attempts = Number(entry.attempts ?? 1)
+  const max = options.maxAttempts ?? RETRY.maxAttempts
+  if (attempts >= max) return { action: 'abandon', attempts, reason: entry.reason ?? 'attempts' }
+  const due = entry.nextAttemptAt ? Date.parse(entry.nextAttemptAt) : 0
+  if (Number.isFinite(due) && due > 0 && now < due) return { action: 'wait', waitMs: due - now, attempts, reason: 'backoff' }
+  return { action: 'retry', attempts, reason: entry.reason ?? '' }
+}
+
+/**
+ * Everything the cadence should act on now: waiting work, plus failures whose backoff has
+ * expired. The retry loop lives here rather than in the caller, so a new caller cannot forget
+ * to bring failed work back into the queue.
+ */
+export function due(root, now = Date.now(), options = {}) {
+  const out = []
+  for (const entry of readQueue(root).values()) {
+    if (entry.state === STATES.PENDING || entry.state === STATES.SUMMARIZED) {
+      out.push({ entry, action: 'work' })
+      continue
+    }
+    const decision = retryDecision(entry, now, options)
+    // 'recover' is reported so a caller can SEE it, but it is not work: the cadence cannot perform
+    // it and must not mistake it for a digest to consolidate.
+    if (decision.action !== 'wait') out.push({ entry, action: decision.action, decision })
+  }
+  // PRIORITY, and it is not cosmetic. Callers take a BOUNDED batch, so queue order decides what
+  // is ever reached. A failure whose backoff expired is time-sensitive: the retry window is open
+  // NOW, and it opened because time passed rather than because new work arrived. Waiting work has
+  // no such deadline. Returning them in sequence order therefore starves retries completely -
+  // measured on a real store, the single retry sat at position 101 of 111 due items behind 34
+  // waiting ones, so with a batch of 3 it would never have run and 'we cannot let things stay
+  // failed' would have been true in the model and false in the system.
+  const RANK = { retry: 0, abandon: 1, work: 2, recover: 3 }
+  out.sort((a, b) => (RANK[a.action] ?? 9) - (RANK[b.action] ?? 9))
+  return out
+}
+
+/**
+ * Record a failure and schedule the next attempt.
+ *
+ * The cap is enforced HERE so no caller can leave a candidate failed forever by forgetting to
+ * check it: the last permitted failure becomes an abandonment automatically.
+ */
+export function fail(root, id, options = {}) {
+  const current = readQueue(root).get(id)
+  const attempts = Number(options.attempts ?? ((current?.attempts ?? 0) + 1))
+  const max = options.maxAttempts ?? RETRY.maxAttempts
+  const state = attempts >= max ? STATES.ABANDONED : STATES.FAILED
+  const at = options.now ?? new Date()
+  return transition(root, id, state, {
+    reason: options.reason,
+    attempts,
+    now: options.now,
+    nextAttemptAt: new Date(at.getTime() + backoffMs(attempts)).toISOString(),
+    detail: { ...(options.detail ?? {}), ...(options.error ? { lastError: String(options.error).slice(0, 200) } : {}) },
+  })
+}
+
+/** Bring a candidate back into the queue. Works for failed, orphaned and abandoned work. */
+export function requeue(root, id, options = {}) {
+  return transition(root, id, options.state ?? STATES.PENDING, {
+    attempts: options.attempts ?? 0,
+    detail: { ...(options.detail ?? {}), requeuedAt: new Date().toISOString(), ...(options.why ? { why: options.why } : {}) },
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Migration from the flag-based state file
+// ---------------------------------------------------------------------------
+
+/**
+ * Classify one legacy record into a state. PURE: the caller supplies what is on disk.
+ *
+ * The guiding rule is CONSERVATIVE: where the old flags are ambiguous, prefer a state that
+ * causes the work to be looked at again over one that declares it finished. The old design
+ * could not distinguish a model concluding there was nothing durable from a model returning
+ * nothing at all, so a record with no recorded reason must not be assumed to be a conclusion.
+ *
+ * @param record - one entry from sessions.json.
+ * @param facts - { digestExists, digestHasSummary } describing the derived material.
+ * @returns { state, reason, detail } with reason present whenever work stopped.
+ */
+export function classifyLegacyRecord(record = {}, facts = {}) {
+  const detail = {
+    ...(record.tier ? { tier: record.tier } : {}),
+    ...(record.title ? { title: record.title } : {}),
+    ...(record.capturedAt ? { capturedAt: record.capturedAt } : {}),
+    ...(record.lastSeq === undefined ? {} : { lastSeq: record.lastSeq }),
+    ...(record.digestHash ? { digestHash: record.digestHash } : {}),
+    ...(record.digestFormat === undefined ? {} : { digestFormat: record.digestFormat }),
+    ...(record.eventCount === undefined ? {} : { eventCount: record.eventCount }),
+    ...(record.toolCallCount === undefined ? {} : { toolCallCount: record.toolCallCount }),
+    // The original settled time, kept because the legacy reader still asks WHEN, and a migration
+    // timestamp would silently rewrite history.
+    ...(record.consolidatedAt ? { legacyConsolidatedAt: record.consolidatedAt } : {}),
+    // ABSENT is not the same as null in the old shape: absent means no writer ever set the key,
+    // null means one explicitly wrote 'not yet'. The distinction is load-bearing for anything
+    // that asks whether a record was ever touched, so it is carried rather than flattened.
+    ...(('consolidatedAt' in record) ? {} : { noConsolidatedAtKey: true }),
+    legacy: true,
+  }
+  const attempts = record.consolidateAttempts ?? record.summaryAttempts;
+  const settled = Boolean(record.consolidatedAt);
+  const source = record.summarySource;
+  const reason = record.summaryReason;
+
+  // A candidate explicitly flagged for review is a known problem, whatever else it says.
+  if (record.needsReview) return { state: STATES.FAILED, reason: REASONS.ATTEMPTS, attempts, detail };
+
+  if (!settled) {
+    // Waiting work. What it is waiting FOR depends on whether material exists.
+    if (source === 'backfill') return { state: STATES.HELD, reason: REASONS.OPERATOR, attempts, detail };
+    if (!facts.digestExists) return { state: STATES.ORPHANED, reason: REASONS.DIGEST_GONE, attempts, detail };
+    if (facts.digestHasSummary) return { state: STATES.SUMMARIZED, attempts, detail };
+    return { state: STATES.PENDING, attempts, detail };
+  }
+
+  // Settled. The honest question is what it settled AS.
+  if (source === 'llm' || source === 'fallback' || source === 'backfill') {
+    return { state: STATES.ADDED, attempts, detail };
+  }
+  if (source === 'thin') return { state: STATES.SKIPPED, reason: REASONS.THIN, attempts, detail };
+  if (source === 'skipped') return { state: STATES.SKIPPED, reason: REASONS.TRIVIAL, attempts, detail };
+  if (source === 'none' || source === undefined) {
+    // Only a recorded no-durable conclusion is trusted as a decision. Anything else - including
+    // a legacy record that recorded no reason at all - becomes retryable work, because assuming
+    // it was a conclusion is exactly how content was lost before.
+    if (reason === REASONS.NO_DURABLE) return { state: STATES.SKIPPED, reason: REASONS.NO_DURABLE, attempts, detail };
+    // A KNOWN defect is retryable: retrying is the right response to it.
+    if (reason) return { state: STATES.FAILED, reason, attempts, detail };
+    // A missing digest is the operative fact, and it outranks the old flag. A digest is DERIVED,
+    // so its absence only matters when nothing came of the session: an `added` record whose digest
+    // was pruned is fine, but one that never produced entries has lost its material and is
+    // recoverable from the source session. Checking this only in the NOT-settled branch left 78
+    // recoverable candidates parked as ordinary waiting work, where nothing would ever look.
+    if (!facts.digestExists) return { state: STATES.ORPHANED, reason: REASONS.DIGEST_GONE, attempts, detail };
+    // Settled, no source, and the digest HAS content: consolidation marked it done. It stamps a
+    // timestamp without writing a source, so this is what a successful consolidation looks like.
+    if (facts.digestHasSummary) return { state: STATES.ADDED, attempts, detail };
+    // Material present but empty, cause never recorded. An UNKNOWN cause is parked, not retried.
+    // Marking these failed would put historical sessions back through summarization on a guess.
+    // sessions back through summarization on the strength of a guess about what went wrong -
+    // which is a backfill by another name, and a spend nobody asked for. The old writer recorded
+    // no reason, so a decision and a defect are genuinely indistinguishable here. Parked keeps
+    // them counted and visible and lets an operator decide, which is the conservative answer.
+    return { state: STATES.HELD, reason: REASONS.UNCLASSIFIED, attempts, detail };
+  }
+  // An unrecognised source is not silently trusted either.
+  return { state: STATES.FAILED, reason: REASONS.UNCLASSIFIED, attempts, detail };
+}
+
+export function legacyFacts(root, id, readDigest) {
+  const text = typeof readDigest === 'function' ? String(readDigest(root, id) ?? '') : '';
+  return { digestExists: text !== '', digestHasSummary: text.includes('summary (') };
+}
+
+/**
+ * Build the entries the migration WOULD write, without writing anything.
+ * @returns an array of { id, state, reason, attempts, detail }.
+ */
+export function planMigration(root, sessions, readDigest) {
+  const existing = readQueue(root);
+  const plan = [];
+  for (const [id, record] of Object.entries(sessions ?? {})) {
+    if (existing.has(id)) continue;   // idempotent: never re-migrate what is already queued
+    const facts = legacyFacts(root, id, readDigest);
+    plan.push({ id, ...classifyLegacyRecord(record, facts) });
+  }
+  return plan;
+}
+
+/**
+ * Apply a migration plan. Re-runnable: entries already present are skipped by planMigration,
+ * and a state that is already current is recorded as a refresh rather than a transition.
+ * @returns { written, counts } - how many entries were appended, and the state tally.
+ */
+export function applyMigration(root, plan) {
+  let written = 0;
+  const counts = {};
+  for (const item of plan) {
+    counts[item.state] = (counts[item.state] ?? 0) + 1;
+    // seed, not transition: the state was reached in the OLD store, and the journal must not
+    // pretend it watched that happen.
+    seed(root, item.id, item.state, { reason: item.reason, attempts: item.attempts, detail: item.detail });
+    written++;
+  }
+  return { written, counts };
+}
+
+/**
+ * Seed an entry that did not originate here.
+ *
+ * A migration imports candidates that already reached some state in the old store. Walking
+ * each one through the lifecycle would FABRICATE a history it never had - a transition table
+ * exists to describe what happened, not to be satisfied. So an imported entry is recorded
+ * directly and marked `imported`, which keeps the journal honest: a reader can tell an entry
+ * that was born here from one that arrived with its state already decided.
+ *
+ * The reason requirement still applies, so an import cannot smuggle in an unexplained stop.
+ * @throws when the id already exists - seeding is not a way to overwrite live state.
+ */
+export function seed(root, id, state, options = {}) {
+  if (!Object.values(STATES).includes(state)) throw new Error('queue: unknown state ' + state)
+  if (typeof id !== 'string' || id === '') throw new Error('queue: an entry needs an id')
+  const existing = readQueue(root).get(id)
+  if (existing) throw new Error('queue: ' + id + ' already exists in state ' + existing.state)
+  const stops = state === STATES.FAILED || state === STATES.SKIPPED || state === STATES.ORPHANED;
+  if (stops && !options.reason) throw new Error('queue: seeding ' + state + ' needs a reason')
+  const entry = {
+    v: QUEUE_FORMAT,
+    seq: nextSeq(root),
+    at: stamp(options.now),
+    id,
+    state,
+    imported: true,
+    ...(options.reason ? { reason: options.reason } : {}),
+    ...(options.attempts === undefined ? {} : { attempts: options.attempts }),
+    ...(options.detail ? { detail: options.detail } : {}),
+  }
+  mkdirSync(join(root, 'state'), { recursive: true })
+  appendFileSync(queuePaths(root).journal, JSON.stringify(entry) + '\n')
+  writeSnapshot(root, readQueue(root));
+  return entry;
+}
+
+// ---------------------------------------------------------------------------
+// Reading the queue through the old shape (a temporary compatibility layer)
+// ---------------------------------------------------------------------------
+
+/**
+ * Project one queue entry into the legacy record shape.
+ *
+ * This is a SHIM and it is deliberately lossy in one direction only: it does not remove the
+ * derivation this whole change exists to delete, it CONCENTRATES it in one place. The old
+ * readers asked questions like is-this-settled, which used to be answered by inspecting a
+ * timestamp; here that question is answered once, from the state, and every legacy caller keeps
+ * working unchanged. Phase 8 deletes it.
+ *
+ * `consolidatedAt` is genuinely three-valued in the old shape and the difference is load-bearing:
+ * a timestamp means settled, null means explicitly pending, and ABSENT means no write ever set
+ * it. Collapsing null and absent would change what pendingDigests returns.
+ */
+/**
+ * Has this candidate's work been DEALT WITH?
+ *
+ * This was a projection artefact for too long: the shim derived it from a state so that legacy
+ * readers could keep asking an old question. It is a real question with a real answer, so it lives
+ * here and both the queue-native readers and the projection use it - one rule, not two that agree
+ * by accident.
+ *
+ * Note what is NOT settled: PENDING and SUMMARIZED are waiting, ORPHANED needs recovery, and a
+ * plain FAILED is retryable. Treating any of those as done is how work disappeared.
+ */
+export function isSettled(entry) {
+  if (!entry) return false;
+  if (entry.state === STATES.ADDED || entry.state === STATES.SKIPPED) return true;
+  // Giving up SETTLES: the old code stamped a timestamp when the retry cap was reached.
+  if (entry.state === STATES.FAILED && entry.reason === REASONS.ATTEMPTS) return true;
+  // A migrated record that was settled in the OLD file stays settled for the old reader's
+  // purposes, even though its state now says it is parked.
+  if (entry.state === STATES.HELD && entry.reason === REASONS.UNCLASSIFIED) return true;
+  if (entry.imported && entry.detail?.legacyConsolidatedAt) return true;
+  return false;
+}
+
+export function advance(root, id, target, options = {}) {
+  const current = readQueue(root).get(id);
+  if (current?.state === target) return 0;
+  const path = shortestPath(current?.state, target);
+  if (!path) throw new Error('queue: no legal path from ' + (current?.state ?? '(new)') + ' to ' + target + ' for ' + id);
+  let steps = 0;
+  for (const step of path) {
+    const last = step === path[path.length - 1];
+    transition(root, id, step, last ? options : {});
+    steps++;
+  }
+  return steps;
+}
+
+/** Breadth-first search over the transition table. The graph is tiny, so this stays cheap. */
+function shortestPath(from, to) {
+  if (from === to) return [];
+  const queue = [[from, []]];
+  const seen = new Set([from]);
+  while (queue.length) {
+    const [at, path] = queue.shift();
+    for (const next of (at === undefined ? [STATES.PENDING, STATES.SUMMARIZED, STATES.HELD] : (TRANSITIONS[at] ?? []))) {
+      if (seen.has(next)) continue;
+      if (next === to) return [...path, next];
+      seen.add(next);
+      queue.push([next, [...path, next]]);
+    }
+  }
+  return undefined;
+}
+
+export const STAGE_TIMEOUT_MS = 30 * 60 * 1000
+
+/**
+ * Return entries stranded in an in-progress state to the failure path.
+ *
+ * Called before every batch. Without it a crash during summarization or consolidation parks a
+ * candidate forever in a state that reads as 'busy' and is, in fact, abandoned.
+ * @returns the ids that were recovered.
+ */
+export function reapStalled(root, now = Date.now(), timeoutMs = STAGE_TIMEOUT_MS) {
+  const recovered = [];
+  for (const entry of readQueue(root).values()) {
+    const inProgress = entry.state === STATES.SUMMARIZING || entry.state === STATES.CONSOLIDATING;
+    if (!inProgress) continue;
+    const started = Date.parse(entry.at);
+    if (!Number.isFinite(started) || now - started < timeoutMs) continue;
+    fail(root, entry.id, {
+      reason: REASONS.STALLED,
+      attempts: Number(entry.detail?.consolidateAttempts ?? entry.attempts ?? 0) + 1,
+      error: 'no progress for ' + Math.round((now - started) / 60000) + ' minutes',
+    });
+    recovered.push(entry.id);
+  }
+  return recovered;
+}