mio-agent-runtime 0.11.0 → 0.13.0

package/server/subscription-store.js

@@ -9,10 +9,56 @@
 // would mean one store with two different notions of "where the data lives".
 //
 // The digest is cursor-based: digest_state.json records, per subscription, the
-// timestamp of the newest event already delivered. Running it again therefore
+// position of the newest event already delivered. Running it again therefore
 // returns only what is new -- which is the point, but also why a digest is not
 // idempotent and the cursor must be written even when nothing matched (otherwise
 // a subscription created just now would replay the whole trace history).
+//
+// A cursor is `{ time, id }` and is an *exclusive* bound: an event is delivered
+// iff it sorts strictly after it, with `id` breaking ties inside a millisecond.
+// The tie-break is not cosmetic. Both timestamps in play come from the same wall
+// clock -- task-store.js stamps the event, digest() stamps the cursor -- and the
+// gap between them is one small writeFileSync, so comparing time alone silently
+// swallowed every event that shared a millisecond with the cursor, permanently,
+// because the cursor only moves forward. Measured 2026-09-22: that gap is ~2ms
+// on a Defender-scanned Windows box but sub-millisecond on a CI runner, which is
+// why `digest respects topic filter` passed 30/30 locally and flaked about half
+// the time on CI. See __tests__/observer-digest-cursor.test.js.
+//
+// The `id` field has three states, and the difference matters:
+//   ''     a wall-clock cursor: digest() found nothing to deliver, so nothing at
+//          that instant has been consumed. The empty string sorts below every
+//          real id, so same-millisecond events are still delivered.
+//   null   a legacy cursor, written before the tie-break existed as a bare ISO
+//          string. `null` sorts above every real id, reproducing the old
+//          inclusive time comparison exactly -- so an upgrade never re-delivers.
+//   '<id>' an event cursor: the event with that id has been delivered.
+const WALL_CURSOR_ID = ''
+
+function readCursor(raw) {
+  if (typeof raw === 'string') {
+    const time = Date.parse(raw)
+    return Number.isFinite(time) ? { time, id: null } : null
+  }
+  if (raw && typeof raw === 'object' && typeof raw.time === 'string') {
+    const time = Date.parse(raw.time)
+    if (!Number.isFinite(time)) return null
+    return { time, id: typeof raw.id === 'string' ? raw.id : null }
+  }
+  return null
+}
+
+// Exclusive: true when `event` sorts strictly after `cursor`.
+function isNewEvent(eventTime, eventId, cursor) {
+  if (!cursor || cursor.time === null || eventTime === null) return true
+  if (eventTime !== cursor.time) return eventTime > cursor.time
+  // Same millisecond -- fall back to the id. Ids are `trace_<ms>_<hex>`, so the
+  // order inside a millisecond is by the random suffix: arbitrary, but stable,
+  // which is all "do not deliver twice" needs.
+  if (cursor.id === null) return false
+  if (typeof eventId !== 'string' || eventId === '') return true
+  return eventId > cursor.id
+}
 
 const fs = require('fs')
 const path = require('path')
@@ -148,10 +194,10 @@ function createSubscriptionStore(options = {}) {
     const matchedSubscriptionIds = new Set()
 
     for (const subscription of subscriptions) {
-      const cursor = state[subscription.id] || null
-      const cursorTime = cursor ? Date.parse(cursor) : null
+      const cursor = readCursor(state[subscription.id])
       const subscriptionEventTypes = subscription.eventTypes || []
       let lastEventTime = null
+      let lastEventId = WALL_CURSOR_ID
       for (const event of traces) {
         if (subscription.project && event.project && event.project !== subscription.project) {
           continue
@@ -168,9 +214,17 @@ function createSubscriptionStore(options = {}) {
           if (!haystack.includes(subscription.topic.toLowerCase())) continue
         }
         const eventTime = event.timestamp ? Date.parse(event.timestamp) : null
-        if (cursorTime !== null && eventTime !== null && eventTime <= cursorTime) continue
-        if (eventTime !== null && (lastEventTime === null || eventTime > lastEventTime)) {
-          lastEventTime = eventTime
+        if (!isNewEvent(eventTime, event.id, cursor)) continue
+        if (eventTime !== null) {
+          if (lastEventTime === null || eventTime > lastEventTime) {
+            lastEventTime = eventTime
+            lastEventId = typeof event.id === 'string' ? event.id : WALL_CURSOR_ID
+          } else if (eventTime === lastEventTime && typeof event.id === 'string' && event.id > lastEventId) {
+            // Several delivered events share the newest millisecond. The cursor
+            // has to sit on the highest id among them, otherwise the next digest
+            // would hand the lower ones out again.
+            lastEventId = event.id
+          }
         }
         if (seenIds.has(event.id)) continue
         seenIds.add(event.id)
@@ -189,9 +243,9 @@ function createSubscriptionStore(options = {}) {
       // Advance the cursor even when nothing matched: a brand-new subscription
       // has no cursor, and without this it would replay the entire history.
       if (lastEventTime !== null) {
-        state[subscription.id] = new Date(lastEventTime).toISOString()
+        state[subscription.id] = { time: new Date(lastEventTime).toISOString(), id: lastEventId }
       } else if (cursor === null) {
-        state[subscription.id] = digestTime
+        state[subscription.id] = { time: digestTime, id: WALL_CURSOR_ID }
       }
       if (events.length >= limit) break
     }

package/server/task-store.js

@@ -18,12 +18,32 @@
 
 const fs = require('fs')
 const path = require('path')
-const { createId, readJsonl, appendJsonl, writeJsonl } = require('./memory-store.js')
+const { createId, readJsonlCached, appendJsonl, writeJsonl } = require('./memory-store.js')
 const { REUSE_STATUS_FILTERS } = require('./experience-store.js')
 const { createQueryLog } = require('./query-log.js')
 
 const AGENT_HEALTH_FRESH_MS = 48 * 3600000
 
+// Token overlap helpers for the routing-gate diagnostic below. Kept local and
+// deliberately stricter than memory-store's scoreRecord: that scorer also
+// counts prefix matches, which are right for fuzzy recall but would make this
+// diagnostic fire on unrelated records. These require exact token equality.
+function tokenizeForOverlap(text) {
+  const normalized = String(text || '').toLowerCase()
+  const latin = normalized.match(/[a-z0-9]+/g) || []
+  const cjk = (normalized.match(/[\u4e00-\u9fff]/g) || []).map((char) => `cjk:${char}`)
+  return [...latin, ...cjk]
+}
+
+function cjkBigramsForOverlap(text) {
+  const chars = String(text || '').toLowerCase().match(/[\u4e00-\u9fff]/g) || []
+  const bigrams = new Set()
+  for (let i = 0; i + 1 < chars.length; i += 1) {
+    bigrams.add(`${chars[i]}${chars[i + 1]}`)
+  }
+  return bigrams
+}
+
 function createTaskStore(options) {
   const dataDir = options.dataDir
   const projectName = options.projectName || (() => null)
@@ -92,13 +112,28 @@ function createTaskStore(options) {
     const scope = normalizeScope(args.scope)
     const limit = Math.min(Math.max(Number(args.limit) || 5, 1), 10)
 
-    const memories = readJsonl(memoryPath).filter((record) => record.archived !== true)
+    const memories = readJsonlCached(memoryPath).filter((record) => record.archived !== true)
     const memoryById = new Map(memories.map((record) => [record.id, record]))
     const evidence = loadEvidenceWeights()
 
     // Match verified reuse experiences whose source memory is relevant to the task.
+    //
+    // ⚠️ The `verified` predicate (experience-store.js) is
+    // `reuse && behaviorChanged && outcomeImproved` -- note it does NOT include
+    // `confirmed`. It has a failure mode that looks exactly like "no match": an
+    // auto-claim is written with behaviorChanged=false (task-store.js
+    // buildAutoClaim), and nothing flips it to true except
+    // `mio.experience.confirm` (experience-store.js:146). So if nothing was ever
+    // confirmed, verified is empty and routing returns nothing for every task,
+    // forever. Measured on the real dataset 2026-09-20: 21 reuse records,
+    // behaviorChanged=false in all of them, verified == 0, and `task.route`
+    // could not return a single hit on a task that plainly had matching
+    // experience. Confirming three records made the same route return 1. So
+    // `count` is not a routing-quality signal until confirmations exist --
+    // `gatedBy` below reports that distinction.
+    const reuseRecords = readJsonlCached(experienceReusePath)
     const routeByMemory = new Map()
-    for (const record of readJsonl(experienceReusePath)) {
+    for (const record of reuseRecords) {
       if (!REUSE_STATUS_FILTERS.verified(record)) continue
       const memory = memoryById.get(record.experienceId)
       if (!memory || !matchesProjectScope(memory, project, scope)) continue
@@ -145,18 +180,65 @@ function createTaskStore(options) {
         },
       }))
 
+    // Why routing came back empty, when it did. `no-match` means the strict
+    // filter was satisfied by nothing relevant; `unconfirmed` means relevant
+    // experience exists but was never confirmed, so the filter excluded it. The
+    // two look identical in `count` but need opposite responses -- the first is
+    // a recall problem, the second is a one-command fix.
+    //
+    // "Relevant" here deliberately does NOT reuse scoreRecord: that scorer's
+    // prefix rule is symmetric (`token.startsWith(item) || item.startsWith(token)`),
+    // so a query token merely starting with any short haystack token counts --
+    // "nonsense" matches a record containing "no read path". Fine for fuzzy
+    // recall, useless as a diagnostic: measured on the real dataset, a nonsense
+    // task scored 10 unrelated records as "relevant". Require a real shared
+    // token (or CJK bigram) instead, so the count only moves for genuine overlap.
+    let gatedBy = null
+    if (routes.length === 0) {
+      const queryTokens = new Set(tokenizeForOverlap(task))
+      const bigrams = cjkBigramsForOverlap(task)
+      const overlaps = (memory) => {
+        const content = `${memory.content || ''} ${(memory.tags || []).join(' ')} ${memory.kind || ''}`
+        if (tokenizeForOverlap(content).some((token) => queryTokens.has(token))) return true
+        if (bigrams.size === 0) return false
+        const haystack = cjkBigramsForOverlap(content)
+        for (const bigram of bigrams) if (haystack.has(bigram)) return true
+        return false
+      }
+      const relevantUnconfirmed = reuseRecords.filter((record) => {
+        if (!REUSE_STATUS_FILTERS.pending(record)) return false
+        const memory = memoryById.get(record.experienceId)
+        if (!memory || !matchesProjectScope(memory, project, scope)) return false
+        return overlaps(memory)
+      }).length
+      const totalVerified = reuseRecords.filter(REUSE_STATUS_FILTERS.verified).length
+      gatedBy = {
+        reason: relevantUnconfirmed > 0 ? 'unconfirmed' : 'no-match',
+        verifiedTotal: totalVerified,
+        relevantUnconfirmed,
+      }
+    }
+
     // Broader context: top related memories not already routed.
     const routedIds = new Set(routes.map((route) => route.experienceId))
     const scoreRelated = (record) =>
       scoreRecord(record, task, project, evidence) +
       (scope === 'all' && isGlobalRecord(record) ? 0.5 : 0)
-    const relatedMemories = memories
-      .filter((record) => !routedIds.has(record.id))
-      .filter((record) => matchesProjectScope(record, project, scope))
-      .filter((record) => scoreRelated(record) > 0)
-      .sort((a, b) => scoreRelated(b) - scoreRelated(a))
+    // Same shape as memory-store.queryMemory: score once, then sort the scored
+    // pairs. Scoring inside the comparator re-tokenized every candidate on every
+    // comparison, which is what made mio.task.route the slowest tool call
+    // (~100 ms median on a 1094-record store).
+    const scoredRelated = []
+    for (const record of memories) {
+      if (routedIds.has(record.id)) continue
+      if (!matchesProjectScope(record, project, scope)) continue
+      const score = scoreRelated(record)
+      if (score > 0) scoredRelated.push({ record, score })
+    }
+    const relatedMemories = scoredRelated
+      .sort((a, b) => b.score - a.score)
       .slice(0, Math.max(limit, 5))
-      .map((record) => {
+      .map(({ record }) => {
         const ev = evidence.get(record.id)
         return {
           id: record.id,
@@ -196,6 +278,13 @@ function createTaskStore(options) {
     if (routes.length > 0) {
       suggestion =
         `Route ${routes.length} verified experience(s) to this task; apply the top match first and record the outcome with mio.observer.ingest / mio.experience.reuse.`
+    } else if (gatedBy && gatedBy.reason === 'unconfirmed') {
+      // Distinct from "no match": the experience is there and relevant, it just
+      // was never confirmed. `verified` requires behaviorChanged, which only
+      // `experience.confirm` sets on an auto-claim, so an unconfirmed backlog
+      // silently empties the routing set. Point at the fix, not at recall.
+      suggestion =
+        `${gatedBy.relevantUnconfirmed} relevant experience(s) match this task but are unconfirmed, so routing skipped them (only verified experience routes, and confirming is what marks behaviorChanged). Run "mio status" to see the backlog, then "mio experience confirm --ids <id,...>".`
     } else if (relatedMemories.length > 0) {
       suggestion =
         'No verified experience matches yet. Consult the related memories, then record the outcome so this task can seed future routes.'
@@ -232,6 +321,7 @@ function createTaskStore(options) {
       routes,
       relatedMemories,
       agentHealth,
+      gatedBy,
       summary: {
         verifiedRoutes: routes.length,
         relatedMemories: relatedMemories.length,
@@ -331,7 +421,7 @@ function createTaskStore(options) {
 
     // 3. Update agent registry
     const now = new Date().toISOString()
-    const agents = readJsonl(agentsPath)
+    const agents = readJsonlCached(agentsPath)
     const agent = agents.find((a) => a.agentId === agentId && a.project === project)
     if (agent) {
       agent.lastSeenAt = now