mio-agent-runtime 0.5.22 → 0.6.0

package/server/llm-client.js

@@ -0,0 +1,69 @@
+'use strict'
+
+// Shared LLM client: the single implementation of chatJson, consumed by the MCP
+// server (creativity engine + insight store) and by the CLI
+// (`mio creativity generate` / `ferment`). Extracted from
+// mio-intelligence-mcp/index.js so both entry points call the same model with
+// the same configuration, and so "is an LLM configured?" has one answer.
+//
+// Configuration is entirely environment-based:
+//   LLM_API_URL       OpenAI-compatible endpoint (default: opencode zen)
+//   LLM_KEY           bearer token; empty means unauthenticated (common for a
+//                     local Ollama / llama.cpp server)
+//   LLM_CHAT_MODEL    model id, falls back to LLM_MODEL
+
+const DEFAULT_API_URL = 'https://opencode.ai/zen/go/v1/chat/completions'
+const DEFAULT_MODEL = 'deepseek-v4-flash'
+
+function llmConfig() {
+  return {
+    apiUrl: process.env.LLM_API_URL || DEFAULT_API_URL,
+    apiKey: process.env.LLM_KEY || '',
+    model: process.env.LLM_CHAT_MODEL || process.env.LLM_MODEL || DEFAULT_MODEL,
+  }
+}
+
+// True when the user has pointed us somewhere other than the default hosted
+// endpoint, or supplied a key. Used to warn (not block) before spending a call.
+function isLlmConfigured() {
+  return Boolean(process.env.LLM_API_URL || process.env.LLM_KEY)
+}
+
+async function chatJson(userText, opts = {}) {
+  const { apiUrl, apiKey, model } = llmConfig()
+  const system = opts.system || 'You are a creative AI assistant. Output JSON.'
+  const temperature = opts.temperature ?? 0.3
+
+  try {
+    const resp = await fetch(apiUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
+      },
+      body: JSON.stringify({
+        model,
+        temperature,
+        messages: [
+          { role: 'system', content: system },
+          { role: 'user', content: userText },
+        ],
+        response_format: { type: 'json_object' },
+      }),
+      signal: AbortSignal.timeout(opts.timeoutMs || 30000),
+    })
+
+    if (!resp.ok) return { error: `HTTP ${resp.status}` }
+    const json = await resp.json()
+    const text = json.choices?.[0]?.message?.content || ''
+    try {
+      return { data: JSON.parse(text) }
+    } catch {
+      return { data: text }
+    }
+  } catch (err) {
+    return { error: String(err) }
+  }
+}
+
+module.exports = { chatJson, llmConfig, isLlmConfigured }

package/server/memory-store.js

@@ -34,6 +34,84 @@ function appendJsonl(file, value) {
   fs.appendFileSync(file, `${JSON.stringify(value)}\n`, 'utf8')
 }
 
+function writeJsonl(file, values) {
+  fs.mkdirSync(path.dirname(file), { recursive: true })
+  fs.writeFileSync(
+    file,
+    values.length > 0 ? values.map((value) => JSON.stringify(value)).join('\n') + '\n' : '',
+    'utf8'
+  )
+}
+
+// --- Duplicate / quality analysis (mio.memory.analyze) ---------------------
+// Threshold is intentionally conservative: two records are "duplicates" only
+// when their token sets are nearly identical, because a false positive here
+// suggests archiving a record the user still needs.
+const MEMORY_DUPLICATE_SIMILARITY = 0.75
+const MEMORY_SHORT_CONTENT_MIN = 20
+// Pairwise Jaccard is O(n^2); beyond this many active records we skip the
+// duplicate scan and report `duplicatesSkipped` instead of hanging.
+const MAX_PAIRWISE_MEMORY = 1500
+
+function contentTokens(value) {
+  const text = String(value || '').toLowerCase()
+  const latin = text.match(/[a-z0-9]+/g) || []
+  const chars = text.match(/[\u4e00-\u9fff]/g) || []
+  const bigrams = []
+  for (let i = 0; i < chars.length - 1; i++) {
+    bigrams.push(`${chars[i]}${chars[i + 1]}`)
+  }
+  return [...latin, ...bigrams]
+}
+
+function jaccardSimilarity(a, b) {
+  const setA = new Set(contentTokens(a))
+  const setB = new Set(contentTokens(b))
+  if (setA.size === 0 || setB.size === 0) return 0
+  let intersection = 0
+  for (const token of setA) {
+    if (setB.has(token)) intersection += 1
+  }
+  const union = setA.size + setB.size - intersection
+  return union === 0 ? 0 : intersection / union
+}
+
+// Union-find over pairs above the similarity threshold, so "A~B, B~C" collapses
+// into one group of three rather than three overlapping pairs.
+function findDuplicateGroups(records, limit) {
+  const parent = new Map()
+  for (const record of records) parent.set(record.id, record.id)
+  const find = (id) => {
+    while (parent.get(id) !== id) {
+      parent.set(id, parent.get(parent.get(id)))
+      id = parent.get(id)
+    }
+    return id
+  }
+  const union = (a, b) => {
+    const ra = find(a)
+    const rb = find(b)
+    if (ra !== rb) parent.set(ra, rb)
+  }
+  for (let i = 0; i < records.length; i++) {
+    for (let j = i + 1; j < records.length; j++) {
+      if (jaccardSimilarity(records[i].content, records[j].content) >= MEMORY_DUPLICATE_SIMILARITY) {
+        union(records[i].id, records[j].id)
+      }
+    }
+  }
+  const byRoot = new Map()
+  for (const record of records) {
+    const root = find(record.id)
+    if (!byRoot.has(root)) byRoot.set(root, [])
+    byRoot.get(root).push(record)
+  }
+  return Array.from(byRoot.values())
+    .filter((group) => group.length >= 2)
+    .sort((a, b) => b.length - a.length)
+    .slice(0, limit)
+}
+
 function normalizeTags(tags) {
   if (Array.isArray(tags)) return tags.map((tag) => String(tag).trim()).filter(Boolean)
   if (typeof tags === 'string') {
@@ -159,12 +237,21 @@ function createMemoryStore(options = {}) {
   // Hook that persists a recent-query entry (phase 0 auto-claim input).
   // Optional: the CLI passes null because no task_outcome can follow a
   // terminal command, so recording a query there would only add noise.
-  const recordQuery = options.recordQuery || null
+  //
+  // Prefer passing `queryLog` (a createQueryLog instance) so memory and task
+  // stores share one implementation of queries.jsonl; `recordQuery` remains
+  // supported as a plain callback for callers that do not have a queryLog.
+  const queryLog = options.queryLog || null
+  const recordQuery =
+    options.recordQuery || (queryLog ? (entry) => queryLog.record(entry) : null)
+  const reuseMatchWindowMs =
+    options.reuseMatchWindowMs || (queryLog ? queryLog.windowMs : 60 * 60 * 1000)
   const verifiedFilter = options.verifiedFilter || defaultVerifiedFilter
 
   const memoryPath = path.join(dataDir, 'memory.jsonl')
   const experienceReusePath = path.join(dataDir, 'experience_reuse.jsonl')
   const tracePath = path.join(dataDir, 'traces.jsonl')
+  const forgetAuditPath = path.join(dataDir, 'memory-forget-audit.jsonl')
 
   function loadEvidenceWeights() {
     const weights = new Map()
@@ -234,7 +321,7 @@ function createMemoryStore(options = {}) {
         resultIds: results.map((record) => record.id).filter(Boolean),
         resultSources: results.map((record) => record.source || null),
         timestamp,
-        expiresAt: timestamp + (options.reuseMatchWindowMs || 60 * 60 * 1000),
+        expiresAt: timestamp + reuseMatchWindowMs,
       })
     }
     return {
@@ -349,18 +436,431 @@ function createMemoryStore(options = {}) {
     return record
   }
 
+  // Quality report over the active (non-archived) records for a project:
+  // kind histogram, low-quality records, and near-duplicate groups. This is the
+  // diagnostic half of memory hygiene — `archiveMemory` is the fix.
+  function analyzeMemory(args = {}) {
+    const project = args.project || projectName()
+    const limit = Math.min(Math.max(Number(args.limit) || 5, 1), 20)
+    const records = readJsonl(memoryPath).filter(
+      (record) => !project || record.project === project || !record.project
+    )
+    const archived = records.filter((record) => record.archived === true).length
+    const activeRecords = records.filter((record) => record.archived !== true)
+    const total = activeRecords.length
+    const layerCounts = { project: 0, global: 0 }
+    for (const record of activeRecords) {
+      if (record.scope === 'global' || !record.project) layerCounts.global += 1
+      else layerCounts.project += 1
+    }
+
+    const byKind = {}
+    const lowQuality = []
+    for (const record of activeRecords) {
+      const kind = String(record.kind || '').trim() || 'unknown'
+      byKind[kind] = (byKind[kind] || 0) + 1
+      const content = String(record.content || '').trim()
+      const issues = []
+      if (!content) issues.push('missing content')
+      else if (content.length < MEMORY_SHORT_CONTENT_MIN) issues.push('content too short')
+      if (!record.kind || !String(record.kind).trim()) issues.push('missing kind')
+      if (issues.length > 0) {
+        lowQuality.push({
+          id: record.id,
+          timestamp: record.timestamp,
+          kind: record.kind || null,
+          content: content.slice(0, 200) || null,
+          issues,
+        })
+      }
+    }
+
+    let duplicateGroups = []
+    let duplicatesSkipped = false
+    if (activeRecords.length > MAX_PAIRWISE_MEMORY) {
+      duplicatesSkipped = true
+    } else {
+      duplicateGroups = findDuplicateGroups(activeRecords, limit).map((group) => ({
+        size: group.length,
+        records: group.map((record) => ({
+          id: record.id,
+          timestamp: record.timestamp,
+          kind: record.kind || null,
+          content: String(record.content || '').slice(0, 200),
+        })),
+      }))
+    }
+
+    const issues = {
+      lowQuality: lowQuality.length,
+      duplicateGroups: duplicateGroups.length,
+      duplicatesSkipped,
+    }
+
+    const suggestions = []
+    if (duplicateGroups.length > 0) {
+      suggestions.push(
+        `Found ${duplicateGroups.length} duplicate group(s); consider archiving or merging duplicates to keep recall precise.`
+      )
+    }
+    if (lowQuality.length > 0) {
+      suggestions.push(
+        `${lowQuality.length} low-quality record(s) (missing kind/content or too short); consider fixing or removing them.`
+      )
+    }
+    if (total === 0) {
+      suggestions.push('No memory records for this project yet.')
+    } else if (duplicateGroups.length === 0 && lowQuality.length === 0) {
+      suggestions.push('Memory looks healthy; no dedup or quality fixes needed.')
+    }
+
+    return {
+      project: project || null,
+      total,
+      archived,
+      layers: layerCounts,
+      byKind,
+      issues,
+      duplicates: duplicateGroups,
+      lowQuality: lowQuality.slice(0, limit),
+      suggestions,
+    }
+  }
+
+  function migrateMemory(args = {}) {
+    const idsInput = Array.isArray(args.ids) ? args.ids : []
+    const ids = idsInput.map((value) => String(value).trim()).filter(Boolean)
+    if (ids.length === 0) throw new Error('memory.migrate requires ids (array of memory record ids)')
+    const targetScope = normalizeScope(args.scope)
+    if (targetScope === 'all') throw new Error('memory.migrate scope must be project or global')
+    const project = args.project || projectName()
+    const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
+
+    const records = readJsonl(memoryPath)
+    const idSet = new Set(ids)
+    const updated = []
+    const unchanged = []
+    const notFound = [...ids]
+    for (const record of records) {
+      if (!idSet.has(record.id)) continue
+      const fromScope = record.scope || 'project'
+      const fromProject = record.project || null
+      if (fromScope === targetScope) {
+        unchanged.push(record.id)
+      } else {
+        record.scope = targetScope
+        record.project = targetScope === 'global' ? null : project
+        record.migratedAt = new Date().toISOString()
+        record.migratedFrom = fromScope === 'global' ? 'global' : fromProject || 'project'
+        if (reason) record.migrateReason = reason
+        updated.push(record.id)
+      }
+      const index = notFound.indexOf(record.id)
+      if (index >= 0) notFound.splice(index, 1)
+    }
+    writeJsonl(memoryPath, records)
+
+    return {
+      project: project || null,
+      scope: targetScope,
+      updated,
+      updatedCount: updated.length,
+      unchanged,
+      notFound,
+    }
+  }
+
+  // Merge near-duplicate records into one survivor.
+  //
+  // Safety stance (measured, not assumed): of 22 duplicate groups in a real
+  // 860-record store, 19 held byte-identical content and 3 held *divergent*
+  // content. Concatenating divergent bodies is actively harmful -- a real
+  // example produced a record carrying both `Token 来源` and `凭证来源` for the
+  // same field plus a duplicated header. So by default this only merges groups
+  // whose members are byte-identical, and reports the rest as `divergent`
+  // for the caller to inspect. Nothing is ever silently concatenated.
+  //
+  // Merging is archive-based, so it is reversible with archiveMemory({restore}).
+  function mergeMemory(args = {}) {
+    const idsInput = Array.isArray(args.ids) ? args.ids : []
+    const ids = idsInput.map((value) => String(value).trim()).filter(Boolean)
+    if (ids.length < 2) throw new Error('memory.merge requires at least 2 ids')
+    const project = args.project || projectName()
+    const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
+    // keep: explicit survivor id. When absent, the newest record wins.
+    const keep = args.keep ? String(args.keep).trim() : null
+    // allowDivergent: opt-in escape hatch. Still refuses if no keep is named,
+    // because "which divergence wins" is then undefined.
+    const allowDivergent = args.allowDivergent === true || args.allowDivergent === 'true'
+
+    const records = readJsonl(memoryPath)
+    const idSet = new Set(ids)
+    // The project filter must be applied once, up front. Applying it only in the
+    // archive loop let a mismatched project still rewrite the survivor's
+    // `supersedes` list, recording an absorption of records that were never
+    // archived -- audit metadata pointing at nothing.
+    const allMembers = records.filter((record) => idSet.has(record.id))
+    const members = project
+      ? allMembers.filter((record) => !record.project || record.project === project)
+      : allMembers
+    const notFound = ids.filter((id) => !allMembers.some((record) => record.id === id))
+    const outOfScope = allMembers
+      .filter((record) => !members.some((member) => member.id === record.id))
+      .map((record) => record.id)
+
+    if (members.length < 2) {
+      return {
+        project: project || null,
+        merged: false,
+        reason:
+          outOfScope.length > 0
+            ? 'fewer than 2 ids are in this project; records belong to another project'
+            : 'fewer than 2 ids resolved to records',
+        survivor: null,
+        archived: [],
+        archivedCount: 0,
+        skipped: [],
+        divergent: [],
+        outOfScope,
+        notFound,
+      }
+    }
+
+    const contents = new Set(members.map((record) => String(record.content || '')))
+    const divergentFrom = contents.size > 1
+
+    if (divergentFrom && !allowDivergent) {
+      return {
+        project: project || null,
+        merged: false,
+        reason: 'content differs across records; refusing to concatenate',
+        survivor: null,
+        archived: [],
+        archivedCount: 0,
+        skipped: [],
+        divergent: members.map((record) => ({
+          id: record.id,
+          timestamp: record.timestamp || null,
+          contentLength: String(record.content || '').length,
+          content: String(record.content || '').slice(0, 200),
+        })),
+        outOfScope,
+        notFound,
+        hint: 'Re-run with --keep <id> --allow-divergent to choose the surviving body explicitly.',
+      }
+    }
+
+    if (divergentFrom && allowDivergent && !keep) {
+      throw new Error('memory.merge with divergent content requires keep (the id whose body survives)')
+    }
+
+    // Survivor selection: explicit keep, else newest by timestamp.
+    let survivor = null
+    if (keep) {
+      survivor = members.find((record) => record.id === keep)
+      if (!survivor) throw new Error(`memory.merge keep id not in the merge set: ${keep}`)
+    } else {
+      survivor = members.reduce((best, record) => {
+        const a = Date.parse(record.timestamp || 0) || 0
+        const b = Date.parse(best.timestamp || 0) || 0
+        return a > b ? record : best
+      })
+    }
+
+    const superseded = members
+      .filter((record) => record.id !== survivor.id)
+      .map((record) => record.id)
+    const mergedAt = new Date().toISOString()
+
+    // Union tags so a merge never loses retrieval keywords.
+    const tagSet = new Set(normalizeTags(survivor.tags))
+    for (const record of members) {
+      for (const tag of normalizeTags(record.tags)) tagSet.add(tag)
+    }
+    survivor.tags = Array.from(tagSet)
+
+    // Audit trail: which records this one absorbed, and when.
+    const priorSuperseded = Array.isArray(survivor.supersedes) ? survivor.supersedes : []
+    survivor.supersedes = Array.from(new Set([...priorSuperseded, ...superseded]))
+    survivor.mergedAt = mergedAt
+    survivor.mergedCount = survivor.supersedes.length
+    if (divergentFrom) survivor.mergeDivergent = true
+    if (reason) survivor.mergeReason = reason
+
+    const archived = []
+    const skipped = []
+    const memberIds = new Set(members.map((member) => member.id))
+    for (const record of records) {
+      if (record.id === survivor.id) continue
+      // members already passed the project filter, so no re-check is needed here.
+      if (!memberIds.has(record.id)) continue
+      if (record.archived === true) {
+        skipped.push(record.id)
+        continue
+      }
+      record.archived = true
+      record.archivedAt = mergedAt
+      record.archiveReason = `merged-into:${survivor.id}`
+      record.mergedInto = survivor.id
+      archived.push(record.id)
+    }
+    writeJsonl(memoryPath, records)
+
+    return {
+      project: project || null,
+      merged: true,
+      divergentContent: divergentFrom,
+      survivor: {
+        id: survivor.id,
+        timestamp: survivor.timestamp || null,
+        contentLength: String(survivor.content || '').length,
+        supersedes: survivor.supersedes,
+        tags: survivor.tags,
+      },
+      archived,
+      archivedCount: archived.length,
+      skipped,
+      divergent: [],
+      outOfScope,
+      notFound,
+    }
+  }
+
+  // Soft delete: archived records stay on disk but drop out of memory.query and
+  // memory.analyze. Reversible via `restore: true`.
+  function archiveMemory(args = {}) {
+    const idsInput = Array.isArray(args.ids) ? args.ids : []
+    const ids = idsInput.map((value) => String(value).trim()).filter(Boolean)
+    if (ids.length === 0) throw new Error('memory.archive requires ids (array of memory record ids)')
+    const project = args.project || projectName()
+    const restore = args.restore === true || args.restore === 'true'
+    const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
+
+    const records = readJsonl(memoryPath)
+    const idSet = new Set(ids)
+    const archived = []
+    // Ids that exist but are already in the requested state. Reported separately
+    // from `notFound` so callers can tell "already done" from "bad id".
+    const skipped = []
+    const notFound = [...ids]
+    for (const record of records) {
+      if (!idSet.has(record.id)) continue
+      if (project && record.project && record.project !== project) continue
+      if (restore) {
+        if (record.archived === true) {
+          delete record.archived
+          delete record.archivedAt
+          delete record.archiveReason
+          archived.push(record.id)
+        } else {
+          skipped.push(record.id)
+        }
+      } else if (record.archived !== true) {
+        record.archived = true
+        record.archivedAt = new Date().toISOString()
+        if (reason) record.archiveReason = reason
+        archived.push(record.id)
+      } else {
+        skipped.push(record.id)
+      }
+      const index = notFound.indexOf(record.id)
+      if (index >= 0) notFound.splice(index, 1)
+    }
+    writeJsonl(memoryPath, records)
+
+    return {
+      project: project || null,
+      restored: restore,
+      archived,
+      archivedCount: archived.length,
+      skipped,
+      notFound,
+    }
+  }
+
+  // forget is the only hard delete in the memory surface: archive is reversible
+  // (archived: true), merge is archive-based, and prune --memory is age-based.
+  // Because the record is gone afterwards, an audit entry is written FIRST and
+  // the delete only proceeds once that succeeded -- otherwise a "forgotten"
+  // record would leave no trace of what it said.
+  //
+  // The audit keeps an excerpt rather than the full body: enough to answer
+  // "what was removed" later, without quietly retaining everything the user
+  // asked to delete.
+  function forgetMemory(args = {}) {
+    const idsInput = Array.isArray(args.ids) ? args.ids : []
+    const ids = idsInput.map((value) => String(value).trim()).filter(Boolean)
+    if (ids.length === 0) throw new Error('memory.forget requires ids (array of memory record ids)')
+    const project = args.project || projectName()
+    const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
+    const by = args.by ? String(args.by).trim().slice(0, 80) : null
+
+    const records = readJsonl(memoryPath)
+    const idSet = new Set(ids)
+    const forgotten = []
+    const notFound = [...ids]
+    const kept = []
+    for (const record of records) {
+      if (!idSet.has(record.id)) {
+        kept.push(record)
+        continue
+      }
+      // Out-of-project ids stay on file and are reported as notFound, matching
+      // how archiveMemory scopes by project.
+      if (project && record.project && record.project !== project) {
+        kept.push(record)
+        continue
+      }
+      const index = notFound.indexOf(record.id)
+      if (index >= 0) notFound.splice(index, 1)
+      const content = String(record.content || '')
+      appendJsonl(forgetAuditPath, {
+        id: createId('forget'),
+        forgottenId: record.id,
+        project: record.project || null,
+        timestamp: new Date().toISOString(),
+        reason,
+        by,
+        kind: record.kind || null,
+        contentLength: content.length,
+        excerpt: content.replace(/\s+/g, ' ').slice(0, 120),
+      })
+      forgotten.push(record.id)
+    }
+
+    if (forgotten.length > 0) writeJsonl(memoryPath, kept)
+
+    return {
+      project: project || null,
+      forgotten,
+      forgottenCount: forgotten.length,
+      notFound,
+      auditPath: forgetAuditPath,
+    }
+  }
+
   return {
     memoryPath,
     experienceReusePath,
     tracePath,
+    forgetAuditPath,
     queryMemory,
     recordMemory,
     queryTraces,
+    analyzeMemory,
+    archiveMemory,
+    forgetMemory,
+    mergeMemory,
+    migrateMemory,
     loadEvidenceWeights,
     // pure helpers, exported for tests and other consumers
     createId,
     readJsonl,
     appendJsonl,
+    writeJsonl,
+    contentTokens,
+    jaccardSimilarity,
+    findDuplicateGroups,
     normalizeTags,
     tokenize,
     normalizeScope,
@@ -373,4 +873,6 @@ function createMemoryStore(options = {}) {
   }
 }
 
-module.exports = { createMemoryStore }
+// Module-level exports let sibling stores (e.g. experience-store.js) reuse the
+// same JSONL primitives instead of duplicating them and drifting apart.
+module.exports = { createMemoryStore, createId, readJsonl, appendJsonl, writeJsonl }