mio-agent-runtime 0.5.23 → 0.7.0

package/server/observer-store.js

@@ -0,0 +1,228 @@
+'use strict'
+
+// Shared observer research-pipeline store: the single implementation behind the
+// mio.observer.* MCP tools and the `mio observer` CLI command. Keeping it here
+// rather than inlined in the MCP server means both entry points read the same
+// files the same way and cannot drift -- same rationale as memory-store.js,
+// experience-store.js and policy-store.js.
+//
+// Default base directory matches the MCP server: <cwd>/.local/observer. This is
+// deliberately project-local (unlike the MIO_HOME-backed stores) because the
+// research pipeline writes into the project it observes; both the CLI flag
+// --base-dir and the MCP argument baseDir override it.
+
+const fs = require('fs')
+const path = require('path')
+
+// Observer research pipeline (optional -- installed via @akemi-mio/observer).
+// Loaded lazily at module level so the CLI can report availability without the
+// MCP server having to pass its own handles down.
+let ObserverStore = null
+let ObserverService = null
+try {
+  const obs = require('@akemi-mio/observer')
+  ObserverStore = obs.ObserverStore
+  ObserverService = obs.ObserverService
+} catch {}
+
+// Subdirectories that make up the pipeline. `observerStatus` counts files in
+// each, so adding a stage here also makes it show up in the status output.
+const OBSERVER_SUBDIRS = [
+  'observations',
+  'trends',
+  'topics',
+  'research',
+  'insights',
+  'world_model',
+  'essays',
+]
+
+function isObserverAvailable() {
+  return Boolean(ObserverStore)
+}
+
+function readJsonSafe(filePath) {
+  if (!fs.existsSync(filePath)) return null
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'))
+  } catch {
+    return null
+  }
+}
+
+function readJsonlSafe(filePath) {
+  if (!fs.existsSync(filePath)) return []
+  try {
+    return fs
+      .readFileSync(filePath, 'utf8')
+      .trim()
+      .split('\n')
+      .filter(Boolean)
+      .map((l) => {
+        try {
+          return JSON.parse(l)
+        } catch {
+          return null
+        }
+      })
+      .filter(Boolean)
+  } catch {
+    return []
+  }
+}
+
+// Reads a directory of per-item JSON files, newest file name first.
+function readJsonDir(dirPath, limit) {
+  if (!fs.existsSync(dirPath)) return []
+  const files = fs.readdirSync(dirPath).filter((f) => f.endsWith('.json')).sort().reverse()
+  return files.slice(0, limit).map((f) => readJsonSafe(path.join(dirPath, f))).filter(Boolean)
+}
+
+function createObserverStore(options = {}) {
+  const defaultBaseDir = options.baseDir || path.join(process.cwd(), '.local', 'observer')
+
+  // Per-call args win over the store default, matching the MCP handler contract
+  // where the caller may pass baseDir on any tool call.
+  function baseDirOf(args = {}) {
+    return args.baseDir || defaultBaseDir
+  }
+
+  function status(args = {}) {
+    const baseDir = baseDirOf(args)
+    const result = {}
+    for (const d of OBSERVER_SUBDIRS) {
+      const dir = path.join(baseDir, d)
+      result[d] = fs.existsSync(dir)
+        ? fs.readdirSync(dir).filter((f) => !f.startsWith('.')).length
+        : 0
+    }
+    return result
+  }
+
+  function worldModel(args = {}) {
+    const baseDir = baseDirOf(args)
+    const wmDir = path.join(baseDir, 'world_model')
+    return {
+      entities: readJsonSafe(path.join(wmDir, 'entities.json')) || [],
+      events: readJsonSafe(path.join(wmDir, 'events.json')) || [],
+      trends: readJsonSafe(path.join(wmDir, 'trends.json')) || [],
+      narratives: readJsonSafe(path.join(wmDir, 'narratives.json')) || [],
+    }
+  }
+
+  function trends(args = {}) {
+    const baseDir = baseDirOf(args)
+    const trendsDir = path.join(baseDir, 'trends')
+    if (args.date) {
+      const report = readJsonSafe(path.join(trendsDir, `${args.date}.json`))
+      return report ? [report] : []
+    }
+    return readJsonDir(trendsDir, args.limit || 7)
+  }
+
+  function research(args = {}) {
+    return readJsonDir(path.join(baseDirOf(args), 'research'), args.limit || 10)
+  }
+
+  function insights(args = {}) {
+    return readJsonDir(path.join(baseDirOf(args), 'insights'), args.limit || 20)
+  }
+
+  function essays(args = {}) {
+    const baseDir = baseDirOf(args)
+    const type = args.type || 'published'
+    const essaysDir = path.join(baseDir, 'essays', type)
+    if (!fs.existsSync(essaysDir)) return []
+    const files = fs.readdirSync(essaysDir).filter((f) => f.endsWith('.md')).sort().reverse()
+    return files.slice(0, args.limit || 10).map((f) => {
+      const full = path.join(essaysDir, f)
+      return {
+        file: f,
+        type,
+        content: fs.readFileSync(full, 'utf8'),
+        created: fs.statSync(full).birthtime.toISOString(),
+      }
+    })
+  }
+
+  // Daily summaries for the last N days, used by the DAG view.
+  function dag(args = {}) {
+    const baseDir = baseDirOf(args)
+    const days = args.days || 7
+    const summaries = []
+    const todayDate = new Date()
+    for (let i = 0; i < days; i++) {
+      const d = new Date(todayDate)
+      d.setDate(d.getDate() - i)
+      const dateStr = d.toISOString().slice(0, 10)
+      const items = readJsonlSafe(path.join(baseDir, 'summaries', `${dateStr}.jsonl`))
+      if (items.length) summaries.push({ date: dateStr, summaries: items })
+    }
+    return { summaries, summaryCount: summaries.reduce((a, s) => a + s.summaries.length, 0) }
+  }
+
+  // ObserverService.collectBySource and FermentationEngine.ferment are both
+  // async. They must be awaited here: a Promise is `typeof 'object'` with no
+  // own enumerable keys, so the previous synchronous version of collect()
+  // happily "iterated" one and pushed nothing -- every call reported
+  // `collected: 0` no matter how much the collectors actually returned.
+  async function collect(args = {}) {
+    if (!ObserverService) throw new Error('@akemi-mio/observer not installed')
+    const service = new ObserverService({ baseDir: baseDirOf(args), collectorConfig: {} })
+    const sources = args.sources || ['bilibili', 'hackernews', 'github', 'douyin', 'rss']
+    const allObs = []
+    for (const src of sources) {
+      try {
+        const collectFn = service.collectBySource || service.collect
+        if (!collectFn) {
+          throw new Error('ObserverService exposes neither collectBySource nor collect')
+        }
+        const result = await collectFn.call(service, [src], args.keywords || [], args.limit || 20)
+        if (Array.isArray(result)) allObs.push(...result)
+        else if (result && typeof result === 'object') {
+          // Record<sourceName, Observation[]>: keep every list, in key order.
+          for (const items of Object.values(result)) {
+            if (Array.isArray(items)) allObs.push(...items)
+          }
+        }
+      } catch (e) {
+        allObs.push({ source: src, error: e.message })
+      }
+    }
+    return { collected: allObs.length, observations: allObs }
+  }
+
+  async function ferment(args = {}) {
+    if (!ObserverService) throw new Error('@akemi-mio/observer not installed')
+    const service = new ObserverService({ baseDir: baseDirOf(args), collectorConfig: {} })
+    const session = args.session || 'afternoon'
+    try {
+      const fermentation = service.getFermentation()
+      if (fermentation && fermentation.ferment) return await fermentation.ferment(session)
+      return { status: 'fermentation engine available but no ferment method' }
+    } catch (e) {
+      return { error: e.message }
+    }
+  }
+
+  return {
+    baseDir: defaultBaseDir,
+    status,
+    worldModel,
+    trends,
+    research,
+    insights,
+    essays,
+    dag,
+    collect,
+    ferment,
+  }
+}
+
+module.exports = {
+  createObserverStore,
+  isObserverAvailable,
+  readJsonSafe,
+  readJsonlSafe,
+  OBSERVER_SUBDIRS,
+}

package/server/policy-store.js

@@ -0,0 +1,263 @@
+'use strict'
+
+// Shared policy store: the single implementation of mio.policy.check, consumed
+// by both the MCP server (index.js) and the CLI (`mio policy check`). Keeping it
+// here rather than inlined in the MCP server means both entry points score
+// risk identically and cannot drift -- same rationale as memory-store.js and
+// experience-store.js.
+//
+// Unlike those stores, the CLI deliberately points this one at the *global*
+// MIO_HOME rather than the per-project directory the MCP server uses, so that
+// `mio policy check` sees the same history as `mio recall` / `mio traces`.
+
+const path = require('path')
+
+const { readJsonl } = require('./memory-store.js')
+
+// Outcomes that count against an action. `retry` is included because a retried
+// action is evidence the first attempt did not simply work.
+const FAILURE_OUTCOMES = new Set(['failure', 'error', 'aborted', 'retry'])
+
+function isFailureOutcome(value) {
+  return FAILURE_OUTCOMES.has(String(value || '').toLowerCase())
+}
+
+function summarizeTrace(event) {
+  const payload = event.payload
+  if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
+    if (typeof payload.summary === 'string' && payload.summary.trim()) {
+      return payload.summary.trim().slice(0, 160)
+    }
+    if (typeof payload.tool === 'string' && payload.tool.trim()) {
+      return `tool: ${payload.tool.trim().slice(0, 120)}`
+    }
+    if (typeof payload.error === 'string' && payload.error.trim()) {
+      return payload.error.trim().slice(0, 160)
+    }
+  }
+  return String(event.event_type || 'trace').slice(0, 120)
+}
+
+function buildPolicyGuidance({ total, failures, risk, failureExamples, relatedMemories }) {
+  const guidance = {
+    level: 'none',
+    rationale: null,
+    saferAlternatives: [],
+    verificationSteps: [],
+    avoid: [],
+    hardGate: false,
+  }
+  if (total === 0) {
+    guidance.rationale =
+      'No historical evidence for this action yet. Treat as normal risk and record the outcome.'
+    return guidance
+  }
+  const verifiedMemories = relatedMemories.filter((record) => record.evidence)
+  const hasVerified = verifiedMemories.length > 0
+  if (risk >= 0.4 && failures >= 2) {
+    guidance.level = hasVerified ? 'actionable' : 'advisory'
+    guidance.rationale =
+      'Repeated historical failures. Prefer an alternative approach and add rollback/verification.'
+    guidance.avoid = failureExamples
+      .map((example) => example.summary)
+      .filter(Boolean)
+      .slice(0, 3)
+  } else if (risk >= 0.2 || hasVerified) {
+    guidance.level = hasVerified ? 'actionable' : 'advisory'
+    guidance.rationale = hasVerified
+      ? 'Verified experience exists for this action; apply it before proceeding.'
+      : 'Moderate historical risk. Add verification before proceeding.'
+  } else {
+    guidance.level = 'advisory'
+    guidance.rationale = 'Low historical risk. Proceed with normal checks and record the outcome.'
+  }
+  if (hasVerified) {
+    guidance.saferAlternatives = verifiedMemories.slice(0, 3).map((record) => ({
+      memoryId: record.id,
+      reuseCount: record.evidence.reuseCount,
+      content: String(record.content || '').slice(0, 300),
+    }))
+  }
+  guidance.verificationSteps = [
+    'Add an explicit verification step after execution (dry-run, rollback plan, or assertion).',
+    'Record the outcome with mio.observer.ingest so the evidence base improves.',
+  ]
+  return guidance
+}
+
+// Risk score thresholds. `unknown` (no history) is deliberately distinct from
+// `low` (history exists and is clean) -- they call for different behavior.
+const POLICY_RISK_HIGH = 0.4
+const POLICY_RISK_MODERATE = 0.2
+
+// A token appearing in more than this share of the trace corpus carries no
+// discriminative power -- it is a structural JSON key (`task`, `outcome`,
+// `summary`, `project`, `host`, `sessionid`...) rather than a description of
+// what the action does. Measured on a real 920-trace corpus, those keys sit
+// above 85%, while meaningful verbs sit far lower.
+const POLICY_GENERIC_TOKEN_SHARE = 0.5
+
+// Below this many matched traces the failure ratio is too noisy to call a
+// level, so it is reported but flagged.
+const POLICY_MIN_SAMPLE = 5
+
+// Tokens that appear in most traces regardless of subject. Matching on these is
+// what makes `mio policy check task` return the entire corpus, so the caller is
+// warned rather than shown a confident-looking but meaningless risk level.
+function collectGenericTokens(traces, tokenize) {
+  const total = traces.length
+  const generic = new Set()
+  if (total === 0) return generic
+  const counts = new Map()
+  for (const event of traces) {
+    const seen = new Set(
+      tokenize(`${event.event_type || ''} ${JSON.stringify(event.payload || {})}`)
+    )
+    for (const token of seen) counts.set(token, (counts.get(token) || 0) + 1)
+  }
+  for (const [token, count] of counts) {
+    if (count / total >= POLICY_GENERIC_TOKEN_SHARE) generic.add(token)
+  }
+  return generic
+}
+
+function createPolicyStore(options = {}) {
+  const dataDir = options.dataDir
+  const projectName = options.projectName || (() => null)
+  const memoryStore = options.memoryStore
+  // scoreRecord / tokenize / loadEvidenceWeights all live in the memory store
+  // and must match the ranking used by mio.memory.query.
+  const tokenize = options.tokenize || memoryStore.tokenize
+  const scoreRecord = options.scoreRecord || memoryStore.scoreRecord
+  const loadEvidenceWeights = options.loadEvidenceWeights || memoryStore.loadEvidenceWeights
+
+  const tracePath = options.tracePath || path.join(dataDir, 'traces.jsonl')
+  const memoryPath = options.memoryPath || path.join(dataDir, 'memory.jsonl')
+
+  function policyCheck(args = {}) {
+    const action = String(args.action || '').trim()
+    if (!action) throw new Error('policy.check requires action')
+    const project = args.project || projectName()
+    const actionTokens = tokenize(action)
+    const allTraces = readJsonl(tracePath)
+    const scopedTraces = allTraces.filter(
+      (event) => !(project && event.project && event.project !== project)
+    )
+    const traces = scopedTraces.filter((event) => {
+      const haystack = tokenize(`${event.event_type || ''} ${JSON.stringify(event.payload || {})}`)
+      return actionTokens.some((token) => haystack.includes(token))
+    })
+    const total = traces.length
+    const failures = traces.filter((event) => isFailureOutcome(event.outcome)).length
+    const risk = total > 0 ? failures / total : null
+    const riskLevel =
+      total === 0
+        ? 'unknown'
+        : risk >= POLICY_RISK_HIGH
+          ? 'high'
+          : risk >= POLICY_RISK_MODERATE
+            ? 'moderate'
+            : 'low'
+    const outcomeCounts = traces.reduce((counts, event) => {
+      const outcome = String(event.outcome || 'unknown').toLowerCase() || 'unknown'
+      counts[outcome] = (counts[outcome] || 0) + 1
+      return counts
+    }, {})
+    const failureExamples = traces
+      .filter((event) => isFailureOutcome(event.outcome))
+      .sort((a, b) => new Date(b.timestamp || 0).getTime() - new Date(a.timestamp || 0).getTime())
+      .slice(0, 3)
+      .map((event) => ({
+        trace_id: event.trace_id || event.id || null,
+        event_type: event.event_type || null,
+        outcome: event.outcome || null,
+        timestamp: event.timestamp || null,
+        agent: event.agent || null,
+        summary: summarizeTrace(event),
+      }))
+    let suggestion
+    if (total === 0) {
+      suggestion = 'No history for this action. Treat as normal risk and record the outcome.'
+    } else if (risk >= POLICY_RISK_HIGH) {
+      suggestion = 'Historically risky. Prefer an alternative approach or add rollback/verification.'
+    } else if (risk >= POLICY_RISK_MODERATE) {
+      suggestion = 'Moderate risk. Add verification before proceeding.'
+    } else {
+      suggestion = 'Low historical risk. Proceed with normal checks.'
+    }
+    const evidence = loadEvidenceWeights()
+    const relatedMemories = readJsonl(memoryPath)
+      .filter((record) => scoreRecord(record, action, project, evidence) > 0)
+      .sort(
+        (a, b) => scoreRecord(b, action, project, evidence) - scoreRecord(a, action, project, evidence)
+      )
+      .slice(0, 3)
+      .map((record) => {
+        const ev = evidence.get(record.id)
+        return {
+          id: record.id,
+          timestamp: record.timestamp,
+          kind: record.kind,
+          content: String(record.content || '').slice(0, 300),
+          evidence: ev ? { reuseCount: ev.reuseCount, confirmedCount: ev.confirmedCount } : null,
+        }
+      })
+    const genericTokens = collectGenericTokens(scopedTraces, tokenize)
+    // Only flag when the match is *carried* by generic tokens: an action like
+    // "npm publish" also contains "npm", which is common but still meaningful.
+    const genericMatched = actionTokens.filter((token) => genericTokens.has(token))
+    const meaningfulTokens = actionTokens.filter((token) => !genericTokens.has(token))
+    const lowSignal =
+      actionTokens.length > 0 && meaningfulTokens.length === 0 && genericMatched.length > 0
+
+    return {
+      action,
+      project,
+      total,
+      failures,
+      risk: risk === null ? null : Number(risk.toFixed(3)),
+      riskLevel,
+      outcomeCounts,
+      failureExamples,
+      suggestion,
+      guidance: buildPolicyGuidance({
+        total,
+        failures,
+        risk,
+        failureExamples,
+        relatedMemories,
+      }),
+      related_memories: relatedMemories,
+      // Additive diagnostics. Existing consumers (mio.policy.check) ignore
+      // these; the CLI uses them to avoid presenting a confident risk level
+      // derived from tokens that match everything.
+      diagnostics: {
+        lowSignal,
+        genericTokens: genericMatched,
+        meaningfulTokens,
+        sampleSize: total,
+        lowSample: total > 0 && total < POLICY_MIN_SAMPLE,
+      },
+    }
+  }
+
+  return {
+    tracePath,
+    memoryPath,
+    policyCheck,
+    // pure helpers, exported for tests and other consumers
+    isFailureOutcome,
+    summarizeTrace,
+    buildPolicyGuidance,
+  }
+}
+
+module.exports = {
+  createPolicyStore,
+  FAILURE_OUTCOMES,
+  isFailureOutcome,
+  summarizeTrace,
+  buildPolicyGuidance,
+  POLICY_RISK_HIGH,
+  POLICY_RISK_MODERATE,
+}

package/server/query-log.js

@@ -0,0 +1,66 @@
+'use strict'
+
+// Shared query log: the single implementation behind queries.jsonl.
+//
+// Two consumers need it and cannot reach each other: memoryStore records a
+// query during queryMemory (phase 0 auto-claim input), and taskStore reads the
+// log to attribute a later task_outcome to that query. Passing one store into
+// the other would be a cycle, so the log is its own module and both get an
+// instance handed in.
+//
+// This replaces what used to be two copies of load/persist -- one inlined in
+// mio-intelligence-mcp/index.js, one in task-store.js -- which had to be kept
+// equivalent by hand.
+
+const fs = require('fs')
+const path = require('path')
+const { readJsonl } = require('./memory-store.js')
+
+const MAX_RECENT_QUERIES = 200
+
+// Evaluated once at load, matching the previous inlined behaviour. Tests that
+// need a different window should set MIO_REUSE_MATCH_WINDOW_MIN before require.
+const REUSE_MATCH_WINDOW_MS = (() => {
+  const minutes = Number(process.env.MIO_REUSE_MATCH_WINDOW_MIN)
+  return Number.isFinite(minutes) && minutes > 0 ? minutes * 60 * 1000 : 60 * 60 * 1000
+})()
+
+function createQueryLog(options = {}) {
+  const dataDir = options.dataDir
+  const queryPath = path.join(dataDir, 'queries.jsonl')
+
+  function load() {
+    const now = Date.now()
+    return readJsonl(queryPath)
+      .filter(
+        (entry) =>
+          entry &&
+          typeof entry.agent === 'string' &&
+          entry.agent &&
+          typeof entry.expiresAt === 'number' &&
+          entry.expiresAt > now
+      )
+      .slice(-MAX_RECENT_QUERIES)
+  }
+
+  function persist(entries) {
+    const limited = entries.slice(-MAX_RECENT_QUERIES)
+    fs.mkdirSync(dataDir, { recursive: true })
+    fs.writeFileSync(
+      queryPath,
+      limited.length > 0 ? limited.map((entry) => JSON.stringify(entry)).join('\n') + '\n' : '',
+      'utf8'
+    )
+    return limited
+  }
+
+  function record(entry) {
+    const entries = load()
+    entries.push(entry)
+    return persist(entries)
+  }
+
+  return { queryPath, windowMs: REUSE_MATCH_WINDOW_MS, load, persist, record }
+}
+
+module.exports = { createQueryLog, REUSE_MATCH_WINDOW_MS, MAX_RECENT_QUERIES }