mio-agent-runtime 0.5.23 → 0.7.0

package/server/subscription-store.js

@@ -0,0 +1,213 @@
+'use strict'
+
+// Shared subscription + digest store: the single implementation behind
+// mio.observer.subscribe and mio.observer.digest.
+//
+// Kept separate from observer-store.js on purpose: that one is the research
+// pipeline and is project-local (<cwd>/.local/observer), while subscriptions,
+// traces and the digest cursor all live under the MIO_HOME data dir. Mixing them
+// would mean one store with two different notions of "where the data lives".
+//
+// The digest is cursor-based: digest_state.json records, per subscription, the
+// timestamp of the newest event already delivered. Running it again therefore
+// returns only what is new -- which is the point, but also why a digest is not
+// idempotent and the cursor must be written even when nothing matched (otherwise
+// a subscription created just now would replay the whole trace history).
+
+const fs = require('fs')
+const path = require('path')
+const { createId, readJsonl, writeJsonl } = require('./memory-store.js')
+
+const SUBSCRIPTION_DEFAULT_TTL_DAYS = 30
+const MAX_SUBSCRIPTIONS_PER_AGENT = 20
+const MAX_DIGEST_EVENTS = 100
+
+function subscribeKey(agent, project, eventTypes, topic) {
+  return `${agent}|${project}|${eventTypes.join(',')}|${topic || ''}`
+}
+
+function createSubscriptionStore(options = {}) {
+  const dataDir = options.dataDir
+  const projectName = options.projectName || (() => null)
+  // Named resolveAgentId so subscribe() can still use `agent` as a local.
+  const resolveAgentId = options.agentId || (() => 'mcp')
+
+  const subscriptionPath = path.join(dataDir, 'subscriptions.jsonl')
+  const tracePath = path.join(dataDir, 'traces.jsonl')
+  const digestStatePath = path.join(dataDir, 'digest_state.json')
+
+  function readDigestState() {
+    if (!fs.existsSync(digestStatePath)) return {}
+    try {
+      const parsed = JSON.parse(fs.readFileSync(digestStatePath, 'utf8'))
+      return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {}
+    } catch (_) {
+      return {}
+    }
+  }
+
+  function writeDigestState(state) {
+    fs.mkdirSync(dataDir, { recursive: true })
+    fs.writeFileSync(digestStatePath, JSON.stringify(state), 'utf8')
+  }
+
+  function loadActiveSubscriptions(agent) {
+    const now = Date.now()
+    return readJsonl(subscriptionPath)
+      .filter((subscription) => subscription && subscription.agent === agent)
+      .filter((subscription) => {
+        if (subscription.expiresAt) {
+          const expires = new Date(subscription.expiresAt).getTime()
+          if (Number.isFinite(expires) && expires <= now) return false
+        }
+        return true
+      })
+  }
+
+  function subscribe(args = {}) {
+    const agent = resolveAgentId() || 'mcp'
+    const project = args.project || projectName()
+    const eventTypes = Array.isArray(args.eventTypes)
+      ? args.eventTypes.map((value) => String(value).trim().toLowerCase()).filter(Boolean)
+      : []
+    const topic = args.topic ? String(args.topic).trim().slice(0, 120) : ''
+    const ttlDays = Number(args.ttlDays) || SUBSCRIPTION_DEFAULT_TTL_DAYS
+    const ttlMs = Math.max(1, ttlDays) * 24 * 60 * 60 * 1000
+
+    const subscriptions = readJsonl(subscriptionPath)
+    const key = subscribeKey(agent, project, eventTypes, topic)
+    let existing = subscriptions.find((subscription) => {
+      return (
+        subscription &&
+        subscribeKey(
+          subscription.agent,
+          subscription.project,
+          subscription.eventTypes || [],
+          subscription.topic || '',
+        ) === key
+      )
+    })
+    const now = new Date()
+    const expiresAt = new Date(now.getTime() + ttlMs).toISOString()
+    if (existing) {
+      existing.updatedAt = now.toISOString()
+      existing.expiresAt = expiresAt
+    } else {
+      existing = {
+        id: createId('sub'),
+        agent,
+        project,
+        eventTypes,
+        topic: topic || null,
+        createdAt: now.toISOString(),
+        updatedAt: now.toISOString(),
+        expiresAt,
+      }
+      subscriptions.push(existing)
+    }
+    const agentSubscriptions = subscriptions.filter(
+      (subscription) => subscription.agent === agent,
+    )
+    if (agentSubscriptions.length > MAX_SUBSCRIPTIONS_PER_AGENT) {
+      throw new Error(`Too many subscriptions for agent ${agent} (max ${MAX_SUBSCRIPTIONS_PER_AGENT})`)
+    }
+    writeJsonl(subscriptionPath, subscriptions)
+    return { subscribed: true, subscription: existing, project }
+  }
+
+  function digest(args = {}) {
+    const agent = resolveAgentId() || 'mcp'
+    const project = args.project || projectName()
+    const limit = Math.min(Math.max(Number(args.limit) || 20, 1), MAX_DIGEST_EVENTS)
+    // An empty array means "no filter", not "match nothing" -- otherwise a
+    // caller that omits event types (or passes []) would silently exclude every
+    // subscription that declares any types at all.
+    const filterEventTypes =
+      Array.isArray(args.eventTypes) && args.eventTypes.length > 0
+        ? args.eventTypes.map((value) => String(value).trim().toLowerCase()).filter(Boolean)
+        : null
+
+    let subscriptions = loadActiveSubscriptions(agent)
+    if (args.project) {
+      subscriptions = subscriptions.filter((subscription) => subscription.project === project)
+    }
+    if (filterEventTypes) {
+      subscriptions = subscriptions.filter((subscription) => {
+        if ((subscription.eventTypes || []).length === 0) return true
+        return filterEventTypes.some((eventType) =>
+          (subscription.eventTypes || []).includes(eventType),
+        )
+      })
+    }
+
+    const state = readDigestState()
+    const traces = readJsonl(tracePath)
+    const digestTime = new Date().toISOString()
+    const events = []
+    const seenIds = new Set()
+    const matchedSubscriptionIds = new Set()
+
+    for (const subscription of subscriptions) {
+      const cursor = state[subscription.id] || null
+      const cursorTime = cursor ? Date.parse(cursor) : null
+      const subscriptionEventTypes = subscription.eventTypes || []
+      let lastEventTime = null
+      for (const event of traces) {
+        if (subscription.project && event.project && event.project !== subscription.project) {
+          continue
+        }
+        if (
+          subscriptionEventTypes.length > 0 &&
+          !subscriptionEventTypes.includes(String(event.event_type || '').toLowerCase())
+        ) {
+          continue
+        }
+        if (subscription.topic) {
+          const haystack = `${event.event_type || ''} ${JSON.stringify(event.payload || {})}`
+            .toLowerCase()
+          if (!haystack.includes(subscription.topic.toLowerCase())) continue
+        }
+        const eventTime = event.timestamp ? Date.parse(event.timestamp) : null
+        if (cursorTime !== null && eventTime !== null && eventTime <= cursorTime) continue
+        if (eventTime !== null && (lastEventTime === null || eventTime > lastEventTime)) {
+          lastEventTime = eventTime
+        }
+        if (seenIds.has(event.id)) continue
+        seenIds.add(event.id)
+        events.push({
+          id: event.id,
+          trace_id: event.trace_id || null,
+          event_type: event.event_type,
+          outcome: event.outcome || null,
+          timestamp: event.timestamp,
+          agent: event.agent || null,
+          payload: event.payload || null,
+        })
+        matchedSubscriptionIds.add(subscription.id)
+        if (events.length >= limit) break
+      }
+      // Advance the cursor even when nothing matched: a brand-new subscription
+      // has no cursor, and without this it would replay the entire history.
+      if (lastEventTime !== null) {
+        state[subscription.id] = new Date(lastEventTime).toISOString()
+      } else if (cursor === null) {
+        state[subscription.id] = digestTime
+      }
+      if (events.length >= limit) break
+    }
+
+    writeDigestState(state)
+    return {
+      project,
+      agent,
+      subscriptionCount: subscriptions.length,
+      matchedSubscriptions: Array.from(matchedSubscriptionIds),
+      count: events.length,
+      events,
+    }
+  }
+
+  return { subscribe, digest, subscriptionPath, digestStatePath }
+}
+
+module.exports = { createSubscriptionStore, subscribeKey }

package/server/task-store.js

@@ -0,0 +1,385 @@
+'use strict'
+
+// Shared task store: the single implementation behind mio.task.route, consumed
+// by both the MCP server and the CLI (`mio task route`). It answers "which
+// verified experiences apply to this kind of task, and which agents have
+// actually succeeded at it" by cross-referencing memory + experience reuse,
+// with an agent-health signal from the latest digest snapshot.
+//
+// Extracted from mio-intelligence-mcp/index.js so the terminal and the MCP tool
+// can never route differently.
+//
+// One behavioural knob is deliberate: `persistQuery`. The MCP call feeds the
+// query log so a later task_outcome can auto-claim route-driven reuse, which is
+// load-bearing for the experience loop. A terminal `mio task route` is an
+// inspection command and must NOT write to queries.jsonl on every invocation,
+// so the CLI passes persistQuery: false. Default stays true so MCP behaviour is
+// unchanged.
+
+const fs = require('fs')
+const path = require('path')
+const { createId, readJsonl, appendJsonl, writeJsonl } = require('./memory-store.js')
+const { REUSE_STATUS_FILTERS } = require('./experience-store.js')
+const { createQueryLog } = require('./query-log.js')
+
+const AGENT_HEALTH_FRESH_MS = 48 * 3600000
+
+function createTaskStore(options) {
+  const dataDir = options.dataDir
+  const projectName = options.projectName || (() => null)
+  const memoryStore = options.memoryStore
+  // Who is asking. The MCP passes the runtime agent id; the CLI passes 'cli'.
+  // Named resolveAgentId so recordTaskOutcome can still use `agentId` as a local.
+  const resolveAgentId = options.agentId || (() => 'mcp')
+
+  const {
+    normalizeScope,
+    isGlobalRecord,
+    matchesProjectScope,
+    scoreRecord,
+    loadEvidenceWeights,
+  } = memoryStore
+
+  const memoryPath = path.join(dataDir, 'memory.jsonl')
+  const experienceReusePath = path.join(dataDir, 'experience_reuse.jsonl')
+  const tracePath = path.join(dataDir, 'traces.jsonl')
+  const agentsPath = path.join(dataDir, 'agents.jsonl')
+
+  function readAgentHealth() {
+    const file = path.join(dataDir, 'digest', 'latest.json')
+    if (!fs.existsSync(file)) return null
+    try {
+      const snapshot = JSON.parse(fs.readFileSync(file, 'utf8'))
+      const ts = Date.parse(snapshot.generatedAt || '')
+      const generatedAt = Number.isFinite(ts) ? snapshot.generatedAt : null
+      if (!Number.isFinite(ts) || Date.now() - ts > AGENT_HEALTH_FRESH_MS) {
+        return { stale: true, generatedAt, best: null, degraded: [], agents: [] }
+      }
+      const agents = Array.isArray(snapshot.agents) ? snapshot.agents : []
+      const reliable = agents.filter((a) => Number(a.total) >= 5)
+      const best =
+        reliable.length > 0
+          ? reliable.reduce((a, b) => (b.successRate > a.successRate ? b : a))
+          : null
+      const degraded = reliable
+        .filter((a) => a.successRate < 50)
+        .map((a) => ({ agent: a.agent, successRate: a.successRate, total: a.total }))
+      return {
+        stale: false,
+        generatedAt,
+        periodDays: snapshot.periodDays || null,
+        best: best ? { agent: best.agent, successRate: best.successRate, total: best.total } : null,
+        degraded,
+        agents: agents.map((a) => ({ agent: a.agent, total: a.total, successRate: a.successRate })),
+      }
+    } catch (_) {
+      return null
+    }
+  }
+
+  // queries.jsonl is owned by the shared query log (see ../query-log.js). The
+  // MCP server and the CLI hand in the same instance the memory store uses, so
+  // there is exactly one reader/writer for this file.
+  const queryLog = options.queryLog || createQueryLog({ dataDir })
+  const loadRecentQueries = () => queryLog.load()
+  const persistRecentQueries = (entries) => queryLog.persist(entries)
+  const REUSE_MATCH_WINDOW_MS = queryLog.windowMs
+
+  function routeTask(args = {}) {
+    const task = String(args.task || '').trim()
+    if (!task) throw new Error('task.route requires a non-empty task')
+    const project = args.project || projectName()
+    const scope = normalizeScope(args.scope)
+    const limit = Math.min(Math.max(Number(args.limit) || 5, 1), 10)
+
+    const memories = readJsonl(memoryPath).filter((record) => record.archived !== true)
+    const memoryById = new Map(memories.map((record) => [record.id, record]))
+    const evidence = loadEvidenceWeights()
+
+    // Match verified reuse experiences whose source memory is relevant to the task.
+    const routeByMemory = new Map()
+    for (const record of readJsonl(experienceReusePath)) {
+      if (!REUSE_STATUS_FILTERS.verified(record)) continue
+      const memory = memoryById.get(record.experienceId)
+      if (!memory || !matchesProjectScope(memory, project, scope)) continue
+      const score = scoreRecord(memory, task, project)
+      if (score <= 0) continue
+      let entry = routeByMemory.get(record.experienceId)
+      if (!entry) {
+        entry = {
+          experienceId: record.experienceId,
+          memory,
+          weight: 0,
+          confirmed: false,
+          sourceAgents: new Set(),
+          targetAgents: new Set(),
+          reuseCount: 0,
+        }
+        routeByMemory.set(record.experienceId, entry)
+      }
+      entry.reuseCount += 1
+      if (record.confirmed === true) entry.confirmed = true
+      if (record.sourceAgent) entry.sourceAgents.add(record.sourceAgent)
+      if (record.targetAgent) entry.targetAgents.add(record.targetAgent)
+      entry.weight = Math.max(entry.weight, score + (entry.confirmed ? 2 : 1.5))
+    }
+
+    const routes = [...routeByMemory.values()]
+      .sort((a, b) => b.weight - a.weight)
+      .slice(0, limit)
+      .map((entry) => ({
+        experienceId: entry.experienceId,
+        reuseCount: entry.reuseCount,
+        confirmed: entry.confirmed,
+        sourceAgents: [...entry.sourceAgents].filter(Boolean),
+        targetAgents: [...entry.targetAgents].filter(Boolean),
+        score: Number(entry.weight.toFixed(2)),
+        memory: {
+          id: entry.memory.id,
+          timestamp: entry.memory.timestamp,
+          kind: entry.memory.kind,
+          content: entry.memory.content,
+          tags: entry.memory.tags || [],
+          project: entry.memory.project || null,
+          scope: entry.memory.scope || 'project',
+        },
+      }))
+
+    // Broader context: top related memories not already routed.
+    const routedIds = new Set(routes.map((route) => route.experienceId))
+    const scoreRelated = (record) =>
+      scoreRecord(record, task, project, evidence) +
+      (scope === 'all' && isGlobalRecord(record) ? 0.5 : 0)
+    const relatedMemories = memories
+      .filter((record) => !routedIds.has(record.id))
+      .filter((record) => matchesProjectScope(record, project, scope))
+      .filter((record) => scoreRelated(record) > 0)
+      .sort((a, b) => scoreRelated(b) - scoreRelated(a))
+      .slice(0, Math.max(limit, 5))
+      .map((record) => {
+        const ev = evidence.get(record.id)
+        return {
+          id: record.id,
+          timestamp: record.timestamp,
+          kind: record.kind,
+          content: record.content,
+          tags: record.tags || [],
+          project: record.project || null,
+          scope: record.scope || 'project',
+          evidence: ev
+            ? { reuseCount: ev.reuseCount, confirmedCount: ev.confirmedCount }
+            : null,
+        }
+      })
+
+    // Feed the query log so a later task_outcome can auto-claim route-driven
+    // reuse. Skipped when persistQuery is false (the CLI, which is read-only).
+    if (args.persistQuery !== false && (routes.length > 0 || relatedMemories.length > 0)) {
+      const now = Date.now()
+      const entries = loadRecentQueries()
+      entries.push({
+        agent: resolveAgentId() || 'mcp',
+        project,
+        query: task,
+        resultIds: [...routedIds, ...relatedMemories.map((record) => record.id)].filter(Boolean),
+        resultSources: [
+          ...routes.map(() => 'experience'),
+          ...relatedMemories.map((record) => record.source || null),
+        ],
+        timestamp: now,
+        expiresAt: now + REUSE_MATCH_WINDOW_MS,
+      })
+      persistRecentQueries(entries)
+    }
+
+    let suggestion
+    if (routes.length > 0) {
+      suggestion =
+        `Route ${routes.length} verified experience(s) to this task; apply the top match first and record the outcome with mio.observer.ingest / mio.experience.reuse.`
+    } else if (relatedMemories.length > 0) {
+      suggestion =
+        'No verified experience matches yet. Consult the related memories, then record the outcome so this task can seed future routes.'
+    } else {
+      suggestion =
+        'No matching experience or memory found. Proceed with care and record the outcome to grow the experience base.'
+    }
+
+    // Agent-health routing signal from the latest digest snapshot (value loop:
+    // collected task outcomes influence who should execute the next task).
+    const agentHealth = readAgentHealth()
+    let routingSignal = null
+    if (agentHealth && !agentHealth.stale && agentHealth.best) {
+      const parts = []
+      if (agentHealth.best.successRate >= 80) {
+        parts.push(
+          `prefer ${agentHealth.best.agent} (${agentHealth.best.successRate}% success over ${agentHealth.best.total} tasks)`,
+        )
+      }
+      for (const bad of agentHealth.degraded) {
+        parts.push(`avoid ${bad.agent} (${bad.successRate}% success over ${bad.total} tasks)`)
+      }
+      if (parts.length > 0) {
+        routingSignal = `Agent health (last ${agentHealth.periodDays || '?'}d): ${parts.join('; ')}.`
+        suggestion = `${suggestion} ${routingSignal}`
+      }
+    }
+
+    return {
+      task,
+      project,
+      scope,
+      count: routes.length,
+      routes,
+      relatedMemories,
+      agentHealth,
+      summary: {
+        verifiedRoutes: routes.length,
+        relatedMemories: relatedMemories.length,
+        routingSignal,
+        suggestion,
+      },
+    }
+  }
+
+  function buildAutoClaim(entry, event, targetAgent) {
+    const crossSourceIndex = entry.resultSources.findIndex(
+      (source) => source && String(source).toLowerCase() !== targetAgent.toLowerCase()
+    )
+    const sourceAgent =
+      crossSourceIndex >= 0 ? String(entry.resultSources[crossSourceIndex]) : targetAgent
+    const experienceId =
+      entry.resultIds[crossSourceIndex >= 0 ? crossSourceIndex : 0] ||
+      entry.resultIds[0] ||
+      'unknown'
+    const outcomeImproved = String(event.outcome || '').toLowerCase() === 'success'
+    return {
+      id: createId('xfer'),
+      timestamp: new Date().toISOString(),
+      sourceAgent,
+      targetAgent,
+      experienceId,
+      reuse: true,
+      behaviorChanged: false,
+      outcomeImproved,
+      project: entry.project,
+      source: 'auto_claim',
+      traceId: event.trace_id || null,
+      notes: `Auto-claimed: memory.query "${entry.query}" matched task_outcome ${event.outcome || 'unknown'} in trace ${event.trace_id || 'unknown'}.`,
+    }
+  }
+
+  // Phase 0 auto-claim: a task_outcome arriving within the match window of a
+  // prior query is attributed to that query's results, so reuse evidence does
+  // not depend on the agent self-reporting mio.experience.reuse. Shared with
+  // observer.ingest, which is why it lives in the store rather than the MCP
+  // handler.
+  function autoClaimExperienceReuse(event) {
+    if (String(event.event_type || '').toLowerCase() !== 'task_outcome') return []
+    const targetAgent = String(event.agent || '').trim()
+    if (!targetAgent) return []
+    const now = Date.now()
+    const claims = []
+    const remaining = []
+    const entries = loadRecentQueries()
+    for (const entry of entries) {
+      const expired = now - entry.timestamp > REUSE_MATCH_WINDOW_MS
+      const matches =
+        entry.agent === targetAgent && (!event.project || entry.project === event.project)
+      if (expired || matches) {
+        if (matches && !expired) {
+          claims.push(buildAutoClaim(entry, event, targetAgent))
+        }
+        continue
+      }
+      remaining.push(entry)
+    }
+    persistRecentQueries(remaining)
+    for (const evidence of claims) {
+      appendJsonl(experienceReusePath, evidence)
+    }
+    return claims
+  }
+
+  function recordTaskOutcome(args = {}) {
+    const outcome = String(args.outcome || '').trim().toLowerCase()
+    if (!['success', 'failure', 'aborted'].includes(outcome)) {
+      throw new Error('task.record_outcome requires outcome to be success, failure, or aborted')
+    }
+    const agentId = String(args.agentId || resolveAgentId() || 'unknown').trim()
+    const project = args.project || projectName()
+    const task = String(args.task || '').trim()
+    const summary = String(args.summary || '').trim()
+    const verification = String(args.verification || '').trim()
+    const traceId = String(args.traceId || '').trim() || createId('trace:' + agentId + ':' + project)
+
+    // 1. Record trace event
+    const event = {
+      id: createId('trace'),
+      timestamp: new Date().toISOString(),
+      trace_id: traceId,
+      event_type: 'task_outcome',
+      outcome,
+      payload: { task, summary, verification },
+      agent: agentId,
+      host: 'mcp',
+      project,
+    }
+    appendJsonl(tracePath, event)
+
+    // 2. Auto-claim experience reuse
+    const autoClaims = autoClaimExperienceReuse(event)
+
+    // 3. Update agent registry
+    const now = new Date().toISOString()
+    const agents = readJsonl(agentsPath)
+    const agent = agents.find((a) => a.agentId === agentId && a.project === project)
+    if (agent) {
+      agent.lastSeenAt = now
+      agent.taskCount = (agent.taskCount || 0) + 1
+      agent.successCount = (agent.successCount || 0) + (outcome === 'success' ? 1 : 0)
+      agent.failureCount = (agent.failureCount || 0) + (outcome === 'failure' ? 1 : 0)
+      writeJsonl(agentsPath, agents)
+    }
+
+    return {
+      recorded: true,
+      event: { id: event.id, trace_id: traceId, outcome, agent: agentId, project },
+      agentUpdated: !!agent,
+      autoClaims: autoClaims.length > 0 ? autoClaims : undefined,
+    }
+  }
+
+  function ingestObservation(args = {}) {
+    const traceId = String(args.trace_id || '').trim()
+    const eventType = String(args.event_type || '').trim()
+    if (!traceId) throw new Error('observer.ingest requires trace_id')
+    if (!eventType) throw new Error('observer.ingest requires event_type')
+    const event = {
+      id: createId('trace'),
+      timestamp: new Date().toISOString(),
+      trace_id: traceId,
+      event_type: eventType,
+      outcome: args.outcome || null,
+      payload: args.payload || {},
+      agent: args.agent || resolveAgentId(),
+      host: args.host || 'mcp',
+      project: args.project || projectName(),
+    }
+    appendJsonl(tracePath, event)
+    const autoClaims = autoClaimExperienceReuse(event)
+    if (autoClaims.length > 0) {
+      return { recorded: true, event, autoClaims }
+    }
+    return { recorded: true, event }
+  }
+
+  return {
+    routeTask,
+    readAgentHealth,
+    autoClaimExperienceReuse,
+    recordTaskOutcome,
+    ingestObservation,
+  }
+}
+
+module.exports = { createTaskStore }