mio-agent-runtime 0.10.0 → 0.12.0

package/server/task-store.js

@@ -18,12 +18,32 @@
 
 const fs = require('fs')
 const path = require('path')
-const { createId, readJsonl, appendJsonl, writeJsonl } = require('./memory-store.js')
+const { createId, readJsonlCached, appendJsonl, writeJsonl } = require('./memory-store.js')
 const { REUSE_STATUS_FILTERS } = require('./experience-store.js')
 const { createQueryLog } = require('./query-log.js')
 
 const AGENT_HEALTH_FRESH_MS = 48 * 3600000
 
+// Token overlap helpers for the routing-gate diagnostic below. Kept local and
+// deliberately stricter than memory-store's scoreRecord: that scorer also
+// counts prefix matches, which are right for fuzzy recall but would make this
+// diagnostic fire on unrelated records. These require exact token equality.
+function tokenizeForOverlap(text) {
+  const normalized = String(text || '').toLowerCase()
+  const latin = normalized.match(/[a-z0-9]+/g) || []
+  const cjk = (normalized.match(/[\u4e00-\u9fff]/g) || []).map((char) => `cjk:${char}`)
+  return [...latin, ...cjk]
+}
+
+function cjkBigramsForOverlap(text) {
+  const chars = String(text || '').toLowerCase().match(/[\u4e00-\u9fff]/g) || []
+  const bigrams = new Set()
+  for (let i = 0; i + 1 < chars.length; i += 1) {
+    bigrams.add(`${chars[i]}${chars[i + 1]}`)
+  }
+  return bigrams
+}
+
 function createTaskStore(options) {
   const dataDir = options.dataDir
   const projectName = options.projectName || (() => null)
@@ -92,13 +112,28 @@ function createTaskStore(options) {
     const scope = normalizeScope(args.scope)
     const limit = Math.min(Math.max(Number(args.limit) || 5, 1), 10)
 
-    const memories = readJsonl(memoryPath).filter((record) => record.archived !== true)
+    const memories = readJsonlCached(memoryPath).filter((record) => record.archived !== true)
     const memoryById = new Map(memories.map((record) => [record.id, record]))
     const evidence = loadEvidenceWeights()
 
     // Match verified reuse experiences whose source memory is relevant to the task.
+    //
+    // ⚠️ The `verified` predicate (experience-store.js) is
+    // `reuse && behaviorChanged && outcomeImproved` -- note it does NOT include
+    // `confirmed`. It has a failure mode that looks exactly like "no match": an
+    // auto-claim is written with behaviorChanged=false (task-store.js
+    // buildAutoClaim), and nothing flips it to true except
+    // `mio.experience.confirm` (experience-store.js:146). So if nothing was ever
+    // confirmed, verified is empty and routing returns nothing for every task,
+    // forever. Measured on the real dataset 2026-09-20: 21 reuse records,
+    // behaviorChanged=false in all of them, verified == 0, and `task.route`
+    // could not return a single hit on a task that plainly had matching
+    // experience. Confirming three records made the same route return 1. So
+    // `count` is not a routing-quality signal until confirmations exist --
+    // `gatedBy` below reports that distinction.
+    const reuseRecords = readJsonlCached(experienceReusePath)
     const routeByMemory = new Map()
-    for (const record of readJsonl(experienceReusePath)) {
+    for (const record of reuseRecords) {
       if (!REUSE_STATUS_FILTERS.verified(record)) continue
       const memory = memoryById.get(record.experienceId)
       if (!memory || !matchesProjectScope(memory, project, scope)) continue
@@ -145,18 +180,65 @@ function createTaskStore(options) {
         },
       }))
 
+    // Why routing came back empty, when it did. `no-match` means the strict
+    // filter was satisfied by nothing relevant; `unconfirmed` means relevant
+    // experience exists but was never confirmed, so the filter excluded it. The
+    // two look identical in `count` but need opposite responses -- the first is
+    // a recall problem, the second is a one-command fix.
+    //
+    // "Relevant" here deliberately does NOT reuse scoreRecord: that scorer's
+    // prefix rule is symmetric (`token.startsWith(item) || item.startsWith(token)`),
+    // so a query token merely starting with any short haystack token counts --
+    // "nonsense" matches a record containing "no read path". Fine for fuzzy
+    // recall, useless as a diagnostic: measured on the real dataset, a nonsense
+    // task scored 10 unrelated records as "relevant". Require a real shared
+    // token (or CJK bigram) instead, so the count only moves for genuine overlap.
+    let gatedBy = null
+    if (routes.length === 0) {
+      const queryTokens = new Set(tokenizeForOverlap(task))
+      const bigrams = cjkBigramsForOverlap(task)
+      const overlaps = (memory) => {
+        const content = `${memory.content || ''} ${(memory.tags || []).join(' ')} ${memory.kind || ''}`
+        if (tokenizeForOverlap(content).some((token) => queryTokens.has(token))) return true
+        if (bigrams.size === 0) return false
+        const haystack = cjkBigramsForOverlap(content)
+        for (const bigram of bigrams) if (haystack.has(bigram)) return true
+        return false
+      }
+      const relevantUnconfirmed = reuseRecords.filter((record) => {
+        if (!REUSE_STATUS_FILTERS.pending(record)) return false
+        const memory = memoryById.get(record.experienceId)
+        if (!memory || !matchesProjectScope(memory, project, scope)) return false
+        return overlaps(memory)
+      }).length
+      const totalVerified = reuseRecords.filter(REUSE_STATUS_FILTERS.verified).length
+      gatedBy = {
+        reason: relevantUnconfirmed > 0 ? 'unconfirmed' : 'no-match',
+        verifiedTotal: totalVerified,
+        relevantUnconfirmed,
+      }
+    }
+
     // Broader context: top related memories not already routed.
     const routedIds = new Set(routes.map((route) => route.experienceId))
     const scoreRelated = (record) =>
       scoreRecord(record, task, project, evidence) +
       (scope === 'all' && isGlobalRecord(record) ? 0.5 : 0)
-    const relatedMemories = memories
-      .filter((record) => !routedIds.has(record.id))
-      .filter((record) => matchesProjectScope(record, project, scope))
-      .filter((record) => scoreRelated(record) > 0)
-      .sort((a, b) => scoreRelated(b) - scoreRelated(a))
+    // Same shape as memory-store.queryMemory: score once, then sort the scored
+    // pairs. Scoring inside the comparator re-tokenized every candidate on every
+    // comparison, which is what made mio.task.route the slowest tool call
+    // (~100 ms median on a 1094-record store).
+    const scoredRelated = []
+    for (const record of memories) {
+      if (routedIds.has(record.id)) continue
+      if (!matchesProjectScope(record, project, scope)) continue
+      const score = scoreRelated(record)
+      if (score > 0) scoredRelated.push({ record, score })
+    }
+    const relatedMemories = scoredRelated
+      .sort((a, b) => b.score - a.score)
       .slice(0, Math.max(limit, 5))
-      .map((record) => {
+      .map(({ record }) => {
         const ev = evidence.get(record.id)
         return {
           id: record.id,
@@ -196,6 +278,13 @@ function createTaskStore(options) {
     if (routes.length > 0) {
       suggestion =
         `Route ${routes.length} verified experience(s) to this task; apply the top match first and record the outcome with mio.observer.ingest / mio.experience.reuse.`
+    } else if (gatedBy && gatedBy.reason === 'unconfirmed') {
+      // Distinct from "no match": the experience is there and relevant, it just
+      // was never confirmed. `verified` requires behaviorChanged, which only
+      // `experience.confirm` sets on an auto-claim, so an unconfirmed backlog
+      // silently empties the routing set. Point at the fix, not at recall.
+      suggestion =
+        `${gatedBy.relevantUnconfirmed} relevant experience(s) match this task but are unconfirmed, so routing skipped them (only verified experience routes, and confirming is what marks behaviorChanged). Run "mio status" to see the backlog, then "mio experience confirm --ids <id,...>".`
     } else if (relatedMemories.length > 0) {
       suggestion =
         'No verified experience matches yet. Consult the related memories, then record the outcome so this task can seed future routes.'
@@ -232,6 +321,7 @@ function createTaskStore(options) {
       routes,
       relatedMemories,
       agentHealth,
+      gatedBy,
       summary: {
         verifiedRoutes: routes.length,
         relatedMemories: relatedMemories.length,
@@ -331,7 +421,7 @@ function createTaskStore(options) {
 
     // 3. Update agent registry
     const now = new Date().toISOString()
-    const agents = readJsonl(agentsPath)
+    const agents = readJsonlCached(agentsPath)
     const agent = agents.find((a) => a.agentId === agentId && a.project === project)
     if (agent) {
       agent.lastSeenAt = now