mixdog 0.9.14 → 0.9.16

package/src/runtime/channels/lib/memory-client.mjs

@@ -2,6 +2,8 @@ import http from 'node:http'
 import fs from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { createStandaloneMemoryRuntime } from '../../../standalone/memory-runtime-proxy.mjs'
 
 const RUNTIME_ROOT = process.env.MIXDOG_RUNTIME_ROOT
   ? path.resolve(process.env.MIXDOG_RUNTIME_ROOT)
@@ -10,6 +12,54 @@ const ACTIVE_INSTANCE_FILE = path.join(RUNTIME_ROOT, 'active-instance.json')
 
 let _portCache = null // { port, mtime, ts }
 
+// Shared memory-runtime proxy handle for respawn. The channels worker is the
+// one process that can find the daemon dead (no port / ECONNREFUSED) while a
+// remote session is still alive. Rather than dead-lettering forever, it asks
+// the SAME proxy the host uses (createStandaloneMemoryRuntime.start()) to
+// re-ensure the singleton daemon. Owner-claim + fork logic is NOT duplicated
+// here — start() reuses the on-disk singleton owner file, so a live daemon is
+// reused and only a genuinely dead one is respawned.
+const MEMORY_ENTRY = fileURLToPath(new URL('../../memory/index.mjs', import.meta.url))
+const MEMORY_DATA_DIR = process.env.MIXDOG_DATA_DIR
+  ? path.resolve(process.env.MIXDOG_DATA_DIR)
+  : RUNTIME_ROOT
+let _memoryProxy = null
+let _ensuringDaemon = null
+function getMemoryProxy() {
+  if (!_memoryProxy) {
+    _memoryProxy = createStandaloneMemoryRuntime({ entry: MEMORY_ENTRY, dataDir: MEMORY_DATA_DIR })
+  }
+  return _memoryProxy
+}
+// Ensure the daemon exists via the shared proxy. Reentrancy-guarded so a burst
+// of failed appends/drains coalesces into a single start(). On success the
+// port cache is invalidated so the next getMemoryPort() re-reads the freshly
+// published active-instance.json. Never throws — callers keep buffering on
+// failure, so a respawn miss degrades to the existing retry path, not a crash.
+async function ensureMemoryDaemon() {
+  if (_ensuringDaemon) return _ensuringDaemon
+  _ensuringDaemon = (async () => {
+    try {
+      const res = await getMemoryProxy().start()
+      _portCache = null
+      return res?.port ?? null
+    } catch (e) {
+      process.stderr.write(`[memory-client] ensureMemoryDaemon failed (${e.message})\n`)
+      return null
+    } finally {
+      _ensuringDaemon = null
+    }
+  })()
+  return _ensuringDaemon
+}
+
+function isConnRefusedLike(err) {
+  const code = String(err?.code || '')
+  const msg = String(err?.message || err || '')
+  return code === 'ECONNREFUSED'
+    || /ECONNREFUSED|missing memory_port|memory-service timeout/i.test(msg)
+}
+
 async function getMemoryPort() {
   const now = Date.now()
   if (_portCache && (now - _portCache.ts) < 5_000) return _portCache.port
@@ -103,6 +153,11 @@ export async function appendEntry(data) {
     return await memoryFetch('POST', '/entry', payload, 3_000)
   } catch (e) {
     process.stderr.write(`[memory-client] appendEntry failed (${e.message}) — buffering\n`)
+    // No-port / connection-refused means the daemon is likely dead. Ask the
+    // shared proxy to re-ensure the singleton (reentrancy-guarded, non-throwing)
+    // so buffered entries have a live target on the next periodic drain instead
+    // of being retried against nothing until quarantine.
+    if (isConnRefusedLike(e)) void ensureMemoryDaemon()
     const bufferPath = bufferToDisk('entry', payload)
     return bufferPath ? { ok: false, buffered: true, path: bufferPath } : { ok: false }
   }
@@ -221,7 +276,13 @@ function retryName(name, n) {
 export async function drainBuffer() {
   if (_draining) return { ok: true, skipped: 'in-progress' }
   const port = await getMemoryPort()
-  if (!port) return { ok: false, reason: 'no-port' }
+  if (!port) {
+    // No published port: the daemon is down while buffered work is waiting.
+    // Respawn it via the shared proxy (reused singleton — no fork dup) and
+    // let the next tick drain against the freshly published port.
+    void ensureMemoryDaemon()
+    return { ok: false, reason: 'no-port' }
+  }
   _draining = true
   let drained = 0
   let failed = 0
@@ -261,6 +322,10 @@ export async function drainBuffer() {
       } catch (e) {
         const attempts = parseRetry(name) + 1
         failed++
+        // Connection-refused mid-drain: the daemon died between the port read
+        // and this POST. Re-ensure it via the shared proxy so the next tick
+        // has a live target (the file is aged/kept below, never dropped here).
+        if (isConnRefusedLike(e)) void ensureMemoryDaemon()
         if (attempts >= MAX_DRAIN_ATTEMPTS) {
           // Quarantine, don't drop: move to dead/ so a poison record neither
           // wedges the queue nor silently vanishes (recoverable for triage).

package/src/runtime/channels/lib/webhook/deliveries.mjs

@@ -99,6 +99,18 @@ function loadEndpointConfig(name) {
 // reference the same `id` and are merged latest-wins at read time.
 const DELIVERY_INDEX_MAX_IDS = 2000;
 const DELIVERY_LOG_MAX_LINES = 10_000;
+// A "pending" claim normally resolves to a terminal status (done/failed) or
+// "processing" within seconds. If a row is stuck at "pending" past this TTL
+// (crash between the pending write and the terminal write, with no restart
+// warm to clear it), treat it as abandoned rather than always-kept —
+// otherwise an unbounded number of stuck-pending ids could accumulate and
+// defeat DELIVERY_INDEX_MAX_IDS entirely.
+const DELIVERY_PENDING_TTL_MS = 10 * 60 * 1000;
+function _isFreshPending(entry) {
+  const ts = Date.parse(entry?.ts ?? "");
+  if (Number.isNaN(ts)) return true; // no timestamp to judge staleness — keep (fail safe toward dedup).
+  return Date.now() - ts < DELIVERY_PENDING_TTL_MS;
+}
 /** @type {Map<string, Map<string, object>>} */
 const _deliveryIndexByEndpoint = new Map();
 /** @type {Set<string>} */
@@ -139,7 +151,16 @@ function _retainedDeliveryIds(entries) {
   const done = [];
   const other = [];
   for (const e of entries) {
-    if (e.status === "received" || e.status === "processing") inflight.push(e);
+    // "pending" is a non-terminal claim too (see _isBlockingDeliveryStatus)
+    // — it must stay in the always-kept inflight set during pruning, or a
+    // concurrent retry of the same id can bypass dedup while the original
+    // delivery is still pending. Bounded by DELIVERY_PENDING_TTL_MS so a
+    // stuck/abandoned pending row (crash, never resolved) doesn't pin
+    // itself in the always-kept set forever and grow the index unbounded;
+    // once stale it falls through to the capped `other` pool below.
+    if (e.status === "received" || e.status === "processing" || (e.status === "pending" && _isFreshPending(e))) {
+      inflight.push(e);
+    }
     else if (e.status === "done") done.push(e);
     else other.push(e);
   }

package/src/runtime/memory/index.mjs

@@ -250,6 +250,39 @@ let _bootTimestamp = null
 // flush pass (the old boolean guard silently SKIPPED bursts, leaving rows
 // embedding-less until the ~60s tick — a recall (no results) window).
 let _rawEmbedFlushChain = Promise.resolve()
+// Per-session ingest serialization. Concurrent ingest_session calls for the
+// SAME session raced on MAX(source_turn) → duplicate turn allocation. Chain
+// same-session ingests so the MAX read + insert loop for one session never
+// overlaps another for the same session. Different sessions stay parallel.
+const _ingestSessionChains = new Map()
+// Cache of (role, cleaned content) per session message OBJECT, keyed by
+// identity (WeakMap — entries vanish once the message is GC'd, e.g. after
+// compaction drops the array). The subset-reingest ordinal fix below must
+// replay cleanMemoryText/shouldExcludeIngestMessage over messages[0, start)
+// on every call; without caching that turned an O(limit)-bounded ingest into
+// an O(full transcript) regex pass on every hydrate. Message objects are
+// immutable transcript entries reused by reference across calls (recall-
+// fasttrack re-ingests the same in-memory array as the session grows), so
+// once a message's identity fields are computed they never change — caching
+// makes every call after the first pay only for genuinely NEW messages,
+// restoring the limit-bounded cost in steady state.
+const _ingestIdentityFieldsCache = new WeakMap()
+// Return { role, content } for a session message's dedup identity, or null if
+// the message would be skipped by ingest (no role / excluded / empty content
+// after cleaning). Cached per message object so repeated calls over the same
+// (unchanged) transcript prefix never re-run the expensive clean/normalize
+// regex pipeline.
+function _ingestIdentityFields(m) {
+  if (_ingestIdentityFieldsCache.has(m)) return _ingestIdentityFieldsCache.get(m)
+  let fields = null
+  const role = normalizeIngestRole(m.role)
+  if (role && !shouldExcludeIngestMessage(m)) {
+    const content = cleanMemoryText(sessionMessageContentForIngest(m))
+    if (content && content.trim()) fields = { role, content }
+  }
+  _ingestIdentityFieldsCache.set(m, fields)
+  return fields
+}
 // Ingest now WAITS for its embedding flush (bounded) so freshly ingested rows
 // are dense-searchable the moment ingest_session returns. On a cold ONNX
 // model the flush can stall on warmup; the race below caps the wait and lets
@@ -441,6 +474,23 @@ async function setCycleLastRun(kind, ts) {
 
 async function ingestSessionMessages(args = {}) {
   const sessionId = String(args.sessionId || args.session_id || `session-${Date.now()}`).trim()
+  // Serialize per-session so the MAX(source_turn) read + insert loop for one
+  // session never overlaps a concurrent ingest for the SAME session (they
+  // otherwise race the turn allocator and double-allocate). Distinct sessions
+  // run in parallel.
+  const prev = _ingestSessionChains.get(sessionId) ?? Promise.resolve()
+  const run = prev.catch(() => {}).then(() => _ingestSessionMessagesImpl(sessionId, args))
+  _ingestSessionChains.set(sessionId, run.catch(() => {}))
+  const tail = _ingestSessionChains.get(sessionId)
+  try {
+    return await run
+  } finally {
+    // Best-effort GC: drop the map entry only if no later call chained after us.
+    if (_ingestSessionChains.get(sessionId) === tail) _ingestSessionChains.delete(sessionId)
+  }
+}
+
+async function _ingestSessionMessagesImpl(sessionId, args = {}) {
   const messages = Array.isArray(args.messages) ? args.messages : []
   // Recall fast-track hydrates the current session before compaction; allow
   // callers to ingest the full in-memory transcript instead of silently
@@ -468,6 +518,30 @@ async function ingestSessionMessages(args = {}) {
     prevMaxTurn = Number(maxRow.rows?.[0]?.max_turn) || 0
   } catch { prevMaxTurn = 0 }
   const turnAllocator = createIngestTurnAllocator(prevMaxTurn)
+  const _identityOccurrence = new Map()
+  // Per-session-stable ordinal for untimestamped, textually-identical repeats.
+  // Must NOT use the global next source_turn (differs every run → re-ingest
+  // duplicates). The Nth identical (role,content) untimestamped turn gets
+  // ordinal N based on its POSITION IN THE FULL messages ARRAY (0..i), not
+  // merely within this call's [start, len) window. Seed the counter by
+  // replaying the SAME identity/exclude logic over messages[0, start) below
+  // (no DB writes, no query — the full array is already in memory) so a
+  // SUBSET re-ingest (e.g. only a session's later half) reproduces the exact
+  // ordinal a full re-ingest would assign at that array position. Appended
+  // genuine repeats therefore never collide with earlier persisted rows of
+  // the same identity, while a full identical re-ingest still reproduces the
+  // same ordinals/source_refs and ON CONFLICT dedupes exactly.
+  for (let i = 0; i < start; i += 1) {
+    const m = messages[i]
+    if (!m || typeof m !== 'object') continue
+    // Cached (WeakMap-keyed) so a repeated hydrate over the SAME prefix
+    // objects only pays the clean/normalize cost once per message, keeping
+    // this seed pass effectively free after the first ingest call.
+    const fields = _ingestIdentityFields(m)
+    if (!fields) continue
+    const occKey = `${fields.role}\u0000${fields.content}`
+    _identityOccurrence.set(occKey, (_identityOccurrence.get(occKey) ?? 0) + 1)
+  }
   for (let i = start; i < messages.length; i += 1) {
     const m = messages[i]
     if (!m || typeof m !== 'object') continue
@@ -490,16 +564,25 @@ async function ingestSessionMessages(args = {}) {
     if (!content || !content.trim()) continue
     considered += 1
     const tsMs = parseTsToMs(m.ts ?? m.timestamp ?? (Date.now() - (messages.length - i)))
+    // Assign the next monotonic turn BEFORE building the source_ref so identical
+    // untimestamped repeats get distinct identities (peekNext is stable until a
+    // row is actually inserted → next()).
+    const assignedTurn = turnAllocator.peekNext()
+    // Stable occurrence index for this identity (pre-hash; role+content is a
+    // sufficient discriminator for the duplicate case the ordinal disambiguates),
+    // seeded above from messages[0, start) so it reflects full-array position
+    // regardless of where this call's window starts.
+    const occKey = `${role}\u0000${content}`
+    const occurrence = (_identityOccurrence.get(occKey) ?? 0)
+    _identityOccurrence.set(occKey, occurrence + 1)
     // Stable per-message identity. The previous `session:${id}#${i+1}` key was
     // positional, so after compaction shrinks/reindexes session.messages a
     // later turn could reuse an old index and be silently skipped by
     // ON CONFLICT DO NOTHING. stableSessionSourceRef hashes only durable
     // fields (role, original ts if present, tool ids, content) — never the
-    // synthesized tsMs fallback or the loop index.
-    const sourceRef = stableSessionSourceRef(sessionId, m, role, content)
-    // Assign the next monotonic turn; only consume it when the row is actually
-    // inserted (a conflicting re-ingest keeps its original source_turn).
-    const assignedTurn = turnAllocator.peekNext()
+    // synthesized tsMs fallback or the loop index. For untimestamped turns the
+    // monotonic ordinal is folded in so genuine repeats persist (not collapsed).
+    const sourceRef = stableSessionSourceRef(sessionId, m, role, content, occurrence)
     const result = await db.query(`
       INSERT INTO entries(ts, role, content, source_ref, session_id, source_turn, project_id)
       VALUES ($1, $2, $3, $4, $5, $6, $7)
@@ -1310,7 +1393,7 @@ async function handleMemoryAction(args, signal) {
     const RETRO_BATCH = 50
     const cycle2Config = config?.cycle2 || {}
     const allActive = (await db.query(
-      `SELECT id, element, category, summary, score, last_seen_at, project_id, status
+      `SELECT id, element, category, summary, score, last_seen_at, project_id, status, reviewed_at
        FROM entries WHERE is_root = 1 AND status = 'active'
        ORDER BY reviewed_at ASC, id ASC`
     )).rows
@@ -1370,19 +1453,23 @@ async function handleMemoryAction(args, signal) {
       }
       if (!primaryActions.length) { kept += batch.filter(r => successIds.has(Number(r.id))).length; continue }
       const acted = new Set()
+      const rowById = new Map(batch.map(r => [Number(r.id), r]))
       for (const act of primaryActions) {
         try {
           const eid = Number(act?.entry_id)
           if (!Number.isFinite(eid) || !allowed.has(eid)) continue
           acted.add(eid)
+          // Optimistic guard: skip if the row moved since this batch was read.
+          const snap = rowById.get(eid)
+          const expected = snap ? { status: snap.status, reviewedAt: snap.reviewed_at } : null
           if (act.action === 'archived') {
-            if (await applySimpleStatus(db, eid, 'archived')) archived += 1
+            if (await applySimpleStatus(db, eid, 'archived', expected)) archived += 1
           } else if (act.action === 'active') {
             // active → active is a keep verdict from the gate.
             kept += 1
             await setCoreSummary(eid, coreSummaryById.get(eid))
           } else if (act.action === 'update') {
-            if (await applyUpdate(db, eid, act.element, act.summary)) updated += 1
+            if (await applyUpdate(db, eid, act.element, act.summary, { expected })) updated += 1
             await setCoreSummary(eid, coreSummaryById.get(eid))
           } else if (act.action === 'merge') {
             const targetId = Number(act?.target_id)

package/src/runtime/memory/lib/cycle-scheduler.mjs

@@ -27,11 +27,19 @@ import fs from 'node:fs'
 
 const CYCLE1_HEALTH_OVERDUE_MS = 5 * 60_000
 const CYCLE1_AUTO_RESTART_COOLDOWN_MS = 5 * 60_000
+const CYCLE1_OMITTED_COOLDOWN_MS = 60 * 60 * 1000
 const BACKLOG_WARN_COOLDOWN_MS = 10 * 60_000
 const BACKLOG_WARN_PENDING = 500
 const BACKLOG_WARN_FAILURES = 5
 // Max back-to-back cycle2 passes per scheduled slot (config: cycle2.catchup_passes).
 const CYCLE2_CATCHUP_PASSES = 4
+const CYCLE2_CATCHUP_PASSES_MAX = 10
+
+function resolveCycle2CatchupPasses(config) {
+  const raw = Number(config?.catchup_passes ?? CYCLE2_CATCHUP_PASSES)
+  if (!Number.isFinite(raw)) return CYCLE2_CATCHUP_PASSES
+  return Math.min(CYCLE2_CATCHUP_PASSES_MAX, Math.max(1, Math.floor(raw)))
+}
 
 export function createCycleScheduler(deps) {
   const {
@@ -260,10 +268,11 @@ export function createCycleScheduler(deps) {
         // Catch-up drain: one scheduled slot may run several back-to-back
         // passes while pending roots remain, so a backlog above the batch
         // size drains in one interval instead of one batch per hour.
-        const drainPasses = Math.max(1, Number(config?.catchup_passes ?? CYCLE2_CATCHUP_PASSES))
+        const drainPasses = resolveCycle2CatchupPasses(config)
         for (let pass = 0; pass < drainPasses; pass++) {
           let c2Options = {
             coalescedRetry: true,
+            catchUpDrainPass: pass > 0,
             onCoalescedSuccess: _finalizeCycle2Run,
           }
           if (typeof c2Options?.callLlm !== 'function') {
@@ -285,13 +294,16 @@ export function createCycleScheduler(deps) {
           // A gate parse/coverage failure marks the run failed even with
           // ok=true; never chain passes on a failing gate.
           if (result?.gate_failed === true) break
-          if (pass + 1 >= drainPasses) break
           const pendingRes = await getDb().query(
             `SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`,
           )
           const pendingLeft = Number(pendingRes?.rows?.[0]?.c ?? 0)
+          log(
+            `[cycle2] catch-up pass ${pass + 1}/${drainPasses}: `
+            + `promoted=${Number(result?.promoted ?? 0)} pending=${pendingLeft}\n`,
+          )
           if (pendingLeft <= 0) break
-          log(`[cycle2] catch-up pass ${pass + 2}/${drainPasses}: pending=${pendingLeft}\n`)
+          if (pass + 1 >= drainPasses) break
         }
       } catch (err) {
         log(`[cycle2] scheduled queue failed: ${err?.message || err}\n`)
@@ -418,13 +430,20 @@ export function createCycleScheduler(deps) {
       const unchunked = Number((await db.query(
         `SELECT COUNT(*) c FROM entries WHERE chunk_root IS NULL AND NULLIF(btrim(session_id), '') IS NOT NULL`,
       )).rows[0]?.c ?? 0)
+      const unchunkedEligible = Number((await db.query(
+        `SELECT COUNT(*) c FROM entries
+         WHERE chunk_root IS NULL
+           AND NULLIF(btrim(session_id), '') IS NOT NULL
+           AND (reviewed_at IS NULL OR reviewed_at < $1)`,
+        [now - CYCLE1_OMITTED_COOLDOWN_MS],
+      )).rows[0]?.c ?? 0)
       const cycle2Pending = Number((await db.query(
         `SELECT COUNT(*) c FROM entries WHERE is_root = 1 AND status = 'pending'`,
       )).rows[0]?.c ?? 0)
-      _cycleBacklogSnapshot = { unchunked, cycle2_pending: cycle2Pending, at: now }
+      _cycleBacklogSnapshot = { unchunked, unchunked_eligible: unchunkedEligible, cycle2_pending: cycle2Pending, at: now }
       _writeCycleStateFile()
       if (unchunked > BACKLOG_WARN_PENDING || cycle2Pending > BACKLOG_WARN_PENDING) {
-        _warnCycleHealth(`backlog unchunked=${unchunked} cycle2_pending=${cycle2Pending}`)
+        _warnCycleHealth(`backlog unchunked=${unchunked} eligible=${unchunkedEligible} cycle2_pending=${cycle2Pending}`)
       }
       if (unchunked > 0 && !_rawEmbedFlushInFlight) {
         _rawEmbedFlushInFlight = true

package/src/runtime/memory/lib/memory-cycle1.mjs

@@ -17,6 +17,11 @@ const VALID_CATEGORIES = new Set([
 ])
 const CYCLE1_OMITTED_RETRY_LIMIT = 2
 const CYCLE1_OMITTED_COOLDOWN_MS = 60 * 60 * 1000
+// LLM/transport outages are transient and must NOT archive healthy raw rows at
+// the tight parse-failure limit. Cooldown reviewed_at (so they sort to the back
+// and stop hot-looping) but bump error_count under a much higher ceiling so a
+// sustained provider outage doesn't shed the backlog.
+const CYCLE1_TRANSIENT_RETRY_LIMIT = 50
 
 // Structural validation only. Conceptual omission belongs to the LLM; code
 // should not encode language-specific phrases such as acknowledgements.
@@ -59,7 +64,7 @@ async function markTerminalRows(db, rowIds, label = 'terminal') {
   }
 }
 
-async function markOmittedRows(db, rowIds) {
+async function markOmittedRows(db, rowIds, retryLimit = CYCLE1_OMITTED_RETRY_LIMIT) {
   const ids = [...new Set((Array.isArray(rowIds) ? rowIds : [])
     .map(id => Number(id))
     .filter(id => Number.isFinite(id) && id > 0))]
@@ -81,7 +86,7 @@ async function markOmittedRows(db, rowIds) {
        FROM candidates c
        WHERE e.id = c.id
        RETURNING e.id, (c.next_error_count >= $3) AS marked_terminal`,
-      [ids, Date.now(), CYCLE1_OMITTED_RETRY_LIMIT],
+      [ids, Date.now(), retryLimit],
     )
     const rows = Array.isArray(result?.rows) ? result.rows : []
     const marked = rows.filter(r => r.marked_terminal === true).length
@@ -309,6 +314,20 @@ async function countPendingRows(db) {
   }
 }
 
+async function countRawUnchunkedRows(db) {
+  try {
+    const result = await db.query(
+      `SELECT COUNT(*) AS c
+       FROM entries
+       WHERE chunk_root IS NULL
+         AND NULLIF(btrim(session_id), '') IS NOT NULL`,
+    )
+    return Number(result.rows[0]?.c ?? 0)
+  } catch {
+    return null
+  }
+}
+
 function uniqueNumbers(values) {
   return [...new Set((Array.isArray(values) ? values : [])
     .map(v => Number(v))
@@ -464,6 +483,7 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
   const signal = options?.signal
   throwIfAborted(signal)
   const pendingRowsAtStart = await countPendingRows(db)
+  const rawUnchunkedAtStart = await countRawUnchunkedRows(db)
   throwIfAborted(signal)
   const batchSize = Math.max(1, Number(config.batch_size ?? 100))
   const windowSize = Math.max(1, Number(config.window_size ?? config.windowSize ?? batchSize))
@@ -520,7 +540,14 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
   throwIfAborted(signal)
   const rowsDesc = fetchResult.rows
 
-  if (Number.isFinite(pendingRowsAtStart) && pendingRowsAtStart < minBatch) {
+  const bypassMinBatchForCooldown = Number.isFinite(rawUnchunkedAtStart)
+    && rawUnchunkedAtStart >= minBatch
+    && Number.isFinite(pendingRowsAtStart)
+    && pendingRowsAtStart < minBatch
+  if (Number.isFinite(pendingRowsAtStart) && pendingRowsAtStart < minBatch && !bypassMinBatchForCooldown) {
+    const pendingLog = Number.isFinite(rawUnchunkedAtStart) ? rawUnchunkedAtStart : 'na'
+    const eligibleLog = Number.isFinite(pendingRowsAtStart) ? pendingRowsAtStart : 'na'
+    __mixdogMemoryLog(`[cycle1] quick-exit pending=${pendingLog} eligible=${eligibleLog} min_batch=${minBatch}\n`)
     throwIfAborted(signal)
     flushEmbeddingDirty(db, { signal }).catch((err) =>
       __mixdogMemoryLog(`[cycle1] quick-exit embedding flush failed: ${err.message}\n`)
@@ -629,6 +656,12 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
       const first = await callCycle1Grouping(buildCycle1ChunkPrompt(rows))
       if (!first.chunks) {
         __mixdogMemoryLog(`[cycle1] unparseable response (window=${windowIdx}) (${String(first.raw).slice(0, 200)})\n`)
+        // Cooldown the rows via the omitted-retry counter so a permanently
+        // unparseable/LLM-erroring window stops hot-looping. markOmittedRows
+        // bumps error_count and archives once CYCLE1_OMITTED_RETRY_LIMIT is hit
+        // (bounded backoff), instead of returning them failed to be re-picked
+        // on the very next tick forever.
+        const cooldown = await markOmittedRows(db, rows.map(r => Number(r.id)))
         return {
           committedChunks: 0, committedMembers: 0, skippedChunks: rows.length, rowsConsidered: originalRows.length,
           invalidChunks: [{ reason: 'unparseable_response', member_ids: rows.map(r => Number(r.id)) }],
@@ -637,6 +670,8 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
           prefilteredRowIds,
           prefilterMarked,
           prefilterMarkFailed,
+          cooldownMarked: cooldown.marked,
+          cooldownDeferred: cooldown.deferred,
         }
       }
       firstChunkList = first.chunks
@@ -671,6 +706,11 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
     } catch (err) {
       if (signal?.aborted) throw err.reason ?? err
       __mixdogMemoryLog(`[cycle1] LLM error (window=${windowIdx}): ${err.message}\n`)
+      // Transient LLM/transport outage — NOT a data-quality parse failure.
+      // Cooldown reviewed_at so the window stops hot-looping, but under a much
+      // higher retry ceiling so a sustained provider outage does not archive
+      // healthy raw rows at the tight parse-failure limit.
+      const cooldown = await markOmittedRows(db, rows.map(r => Number(r.id)), CYCLE1_TRANSIENT_RETRY_LIMIT)
       return {
         committedChunks: 0, committedMembers: 0, skippedChunks: rows.length, rowsConsidered: originalRows.length,
         invalidChunks: [{ reason: 'llm_error', member_ids: rows.map(r => Number(r.id)) }],
@@ -679,6 +719,8 @@ async function _runCycle1Impl(db, config = {}, options = {}, _dataDir = null) {
         prefilteredRowIds,
         prefilterMarked,
         prefilterMarkFailed,
+        cooldownMarked: cooldown.marked,
+        cooldownDeferred: cooldown.deferred,
       }
     }
 

package/src/runtime/memory/lib/memory-cycle2-gate.mjs

@@ -353,11 +353,15 @@ export async function runUnifiedGate(db, rows, activeContext, config = {}, optio
   // numbered Entries block uses). The gate may echo either the real batch id
   // or the row ordinal 1..N; the parser resolves both when no integer token is
   // ambiguous (id K present in batch but row K points at a different entry).
-  const ordinalToId = new Map(rows.map((r, i) => [i + 1, Number(r.id)]))
+  // On collision, DISABLE ordinal fallback (exact ids only) instead of
+  // skipping: a skipped batch is re-selected with the same rows next run and
+  // collides again — a permanent stall (observed: same "batch id 3 collides"
+  // skip every cycle while cycle2_pending climbed past 500).
+  let ordinalToId = new Map(rows.map((r, i) => [i + 1, Number(r.id)]))
   const collideToken = batchOrdinalIdCollides(statusById, ordinalToId, rows.length)
   if (collideToken != null) {
-    __mixdogMemoryLog(`[cycle2] batch id ${collideToken} collides with row ordinal ${collideToken} (id ${ordinalToId.get(collideToken)}) — skipping batch (no safe id resolution)\n`)
-    return { actions: null, rejected: new Set(), parseOk: false }
+    __mixdogMemoryLog(`[cycle2] batch id ${collideToken} collides with row ordinal ${collideToken} (id ${ordinalToId.get(collideToken)}) — ordinal fallback disabled, exact ids only\n`)
+    ordinalToId = null
   }
 
   __mixdogMemoryLog(`[cycle2-diag] unified prompt=${prompt.length} bytes; rows=${rows.length}\n`)

package/src/runtime/memory/lib/memory-cycle2-mutations.mjs

@@ -49,18 +49,25 @@ export function clampPendingPromotions(statusBatch, rowsById, activeCount, activ
 // Trigger handles score recompute automatically — no app-side score writes.
 export async function applyBatchStatusVerdicts(db, batch, nowMs) {
   if (!batch || batch.length === 0) return { promoted: 0, archived: 0 }
+  // Optimistic guard: each item may carry expected {status, reviewedAt} — the
+  // snapshot the verdict was computed against. NULL expected_* means "no guard"
+  // (legacy blind update). The UPDATE only fires when the row still matches, so
+  // a concurrent write skips the stale verdict.
   const valueRows = batch.map((item, i) => {
-    const base = i * 3
-    return `($${base + 1}::bigint, $${base + 2}::text, $${base + 3}::boolean)`
+    const base = i * 5
+    return `($${base + 1}::bigint, $${base + 2}::text, $${base + 3}::boolean, $${base + 4}::entry_status, $${base + 5}::bigint)`
   })
   const params = []
   for (const item of batch) {
-    params.push(item.entry_id, item.new_status, item.was_pending)
+    const exp = item.expected ?? null
+    const expStatus = exp && typeof exp.status === 'string' ? exp.status : null
+    const expReviewed = exp && Number.isFinite(Number(exp.reviewedAt)) ? Number(exp.reviewedAt) : null
+    params.push(item.entry_id, item.new_status, item.was_pending, expStatus, expReviewed)
   }
   params.push(nowMs)
   const lastParam = `$${params.length}`
   const res = await db.query(
-    `WITH actions(entry_id, new_status, was_pending) AS (
+    `WITH actions(entry_id, new_status, was_pending, exp_status, exp_reviewed) AS (
        VALUES ${valueRows.join(', ')}
      )
      UPDATE entries
@@ -72,6 +79,8 @@ export async function applyBatchStatusVerdicts(db, batch, nowMs) {
          END
      FROM actions a
      WHERE entries.id = a.entry_id AND entries.is_root = 1
+       AND (a.exp_status   IS NULL OR entries.status      IS NOT DISTINCT FROM a.exp_status)
+       AND (a.exp_reviewed IS NULL OR entries.reviewed_at IS NOT DISTINCT FROM a.exp_reviewed)
      RETURNING entries.id, entries.status, a.was_pending, a.new_status`,
     params,
   )
@@ -85,10 +94,27 @@ export async function applyBatchStatusVerdicts(db, batch, nowMs) {
 }
 
 // Generic status update for archived/active terminal transitions.
-export async function applySimpleStatus(db, entryId, nextStatus) {
+// Optimistic concurrency: when the caller passes `expected` (the status and/or
+// reviewed_at it observed when it made the verdict), the UPDATE only fires if
+// the row still matches. A concurrent cycle/recall write that changed status or
+// bumped reviewed_at makes the guard fail (0 rows) so a stale LLM verdict does
+// not overwrite the newer state. Omitting `expected` preserves legacy blind
+// update-by-id behavior for callers that don't track a baseline.
+export async function applySimpleStatus(db, entryId, nextStatus, expected = null) {
+  const params = [nextStatus, entryId]
+  const guards = []
+  if (expected && typeof expected.status === 'string') {
+    guards.push(`status IS NOT DISTINCT FROM $${params.length + 1}::entry_status`)
+    params.push(expected.status)
+  }
+  if (expected && Number.isFinite(Number(expected.reviewedAt))) {
+    guards.push(`reviewed_at IS NOT DISTINCT FROM $${params.length + 1}::bigint`)
+    params.push(Number(expected.reviewedAt))
+  }
+  const guardSql = guards.length ? ` AND ${guards.join(' AND ')}` : ''
   const res = await db.query(
-    `UPDATE entries SET status = $1 WHERE id = $2 AND is_root = 1`,
-    [nextStatus, entryId],
+    `UPDATE entries SET status = $1 WHERE id = $2 AND is_root = 1${guardSql}`,
+    params,
   )
   return Number(res.rowCount ?? res.affectedRows ?? 0) > 0
 }
@@ -108,8 +134,22 @@ export async function applyUpdate(db, entryId, element, summary, options = {}) {
   }
   if (setClauses.length === 0) return false
   params.push(entryId)
+  const idParam = paramIdx++
+  // Optimistic guard (see applySimpleStatus): skip the write if the row moved
+  // on since the verdict was computed. options.expected = { status, reviewedAt }.
+  const guards = []
+  const expected = options?.expected
+  if (expected && typeof expected.status === 'string') {
+    guards.push(`status IS NOT DISTINCT FROM $${paramIdx++}::entry_status`)
+    params.push(expected.status)
+  }
+  if (expected && Number.isFinite(Number(expected.reviewedAt))) {
+    guards.push(`reviewed_at IS NOT DISTINCT FROM $${paramIdx++}::bigint`)
+    params.push(Number(expected.reviewedAt))
+  }
+  const guardSql = guards.length ? ` AND ${guards.join(' AND ')}` : ''
   const res = await db.query(
-    `UPDATE entries SET ${setClauses.join(', ')} WHERE id = $${paramIdx} AND is_root = 1`,
+    `UPDATE entries SET ${setClauses.join(', ')} WHERE id = $${idParam} AND is_root = 1${guardSql}`,
     params,
   )
   if (Number(res.rowCount ?? res.affectedRows ?? 0) === 0) return false
@@ -278,7 +318,11 @@ export async function runPhaseMerge(db, options = {}) {
       String(pair.a.summary ?? ''),
       String(pair.b.summary ?? ''),
       tier2Pairs,
-      { signal },
+      // Thread the in-process LLM adapter through — omitting it falls back to
+      // callAgentDispatch (IPC), which is unavailable in the standalone
+      // memory service and failed every judge call with
+      // `agent-ipc: IPC channel unavailable`.
+      { signal, callLlm: options?.callLlm },
     )
     throwIfAborted(signal)
     if (shouldMerge) await doMerge(pair.a, pair.b, pair.sim)
@@ -324,7 +368,7 @@ export async function runPhaseMerge(db, options = {}) {
       String(row.entry_summary ?? ''),
       String(row.core_summary ?? ''),
       [],
-      { signal },
+      { signal, callLlm: options?.callLlm },
     )
     throwIfAborted(signal)
     if (!verdictMerge) continue