mio-agent-runtime 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,9 +29,63 @@ function readJsonl(file) {
29
29
  .filter(Boolean)
30
30
  }
31
31
 
32
+ // --- cross-call read cache ------------------------------------------------
33
+ // The MCP server is long-lived and re-reads the same append-only logs on every
34
+ // tool call. On a 1094-record store that is ~7 ms to read+parse memory.jsonl
35
+ // and ~13 ms more to rebuild every record's tokenized haystack -- about 20 ms
36
+ // per query, against ~0.4 ms when both are warm.
37
+ //
38
+ // Validity: (size, mtimeMs) is checked on every call, and every write that goes
39
+ // through appendJsonl/writeJsonl drops the entry explicitly. The one window
40
+ // left is an outside process rewriting a file in the same millisecond to the
41
+ // exact same byte count; the consequence is a single stale query, not corrupt
42
+ // data. `mio prune` (retention.js) now writes through this module's writeJsonl
43
+ // (or calls dropCache directly), so its prunes invalidate the cache explicitly
44
+ // instead of relying on the size check.
45
+ const readCache = new Map()
46
+
47
+ // Haystacks are memoized against the record objects the cache hands out, so a
48
+ // fresh parse produces fresh objects and rebuilds them. Reassigned (not just
49
+ // cleared) on every write because mergeMemory() mutates `survivor.tags` in
50
+ // place, and tags are part of the haystack -- a WeakMap cannot be cleared, so
51
+ // it is replaced.
52
+ let haystackCache = new WeakMap()
53
+
54
+ function dropCache(file) {
55
+ readCache.delete(file)
56
+ haystackCache = new WeakMap()
57
+ }
58
+
59
+ function readJsonlCached(file) {
60
+ let st
61
+ try {
62
+ st = fs.statSync(file)
63
+ } catch (_) {
64
+ dropCache(file)
65
+ return []
66
+ }
67
+ const hit = readCache.get(file)
68
+ if (hit !== undefined && hit.size === st.size && hit.mtimeMs === st.mtimeMs) {
69
+ return hit.records
70
+ }
71
+ const records = readJsonl(file)
72
+ readCache.set(file, { size: st.size, mtimeMs: st.mtimeMs, records })
73
+ return records
74
+ }
75
+
76
+ function haystackFor(record) {
77
+ let hay = haystackCache.get(record)
78
+ if (hay === undefined) {
79
+ hay = splitHaystack(`${record.content || ''} ${(record.tags || []).join(' ')} ${record.kind || ''}`)
80
+ haystackCache.set(record, hay)
81
+ }
82
+ return hay
83
+ }
84
+
32
85
  function appendJsonl(file, value) {
33
86
  fs.mkdirSync(path.dirname(file), { recursive: true })
34
87
  fs.appendFileSync(file, `${JSON.stringify(value)}\n`, 'utf8')
88
+ dropCache(file)
35
89
  }
36
90
 
37
91
  function writeJsonl(file, values) {
@@ -41,6 +95,7 @@ function writeJsonl(file, values) {
41
95
  values.length > 0 ? values.map((value) => JSON.stringify(value)).join('\n') + '\n' : '',
42
96
  'utf8'
43
97
  )
98
+ dropCache(file)
44
99
  }
45
100
 
46
101
  // --- Duplicate / quality analysis (mio.memory.analyze) ---------------------
@@ -165,27 +220,60 @@ function cjkBigrams(text) {
165
220
  return bigrams
166
221
  }
167
222
 
223
+ // Split a record's searchable text once. The scorer below used to run four
224
+ // regex passes per record -- [a-z0-9]+ twice (tokenize + latinTokens) and
225
+ // [\u4e00-\u9fff] twice (tokenize + cjkBigrams) -- and materialised a `cjk:`
226
+ // string for every single CJK character, ~1200 allocations for a 600-character
227
+ // record, on every query. One pass that keeps latin words and CJK characters
228
+ // apart is exact: the two token kinds come from disjoint character classes, so
229
+ // looking each up in its own collection scores identically.
230
+ //
231
+ // Measured on the real 1094-record store: 22.05 ms -> 18.16 ms for a full
232
+ // scoring pass (1.21x), with 5470 (record, query) pairs scoring bit-identical.
233
+ const HAYSTACK_PART_RE = /[a-z0-9]+|[\u4e00-\u9fff]/g
234
+
235
+ function splitHaystack(content) {
236
+ const parts = String(content || '').toLowerCase().match(HAYSTACK_PART_RE) || []
237
+ const latin = []
238
+ const cjkChars = []
239
+ for (let i = 0; i < parts.length; i += 1) {
240
+ const part = parts[i]
241
+ // CJK ideographs are all above U+2000; latin words and digits are not.
242
+ if (part.charCodeAt(0) > 0x2000) cjkChars.push(part)
243
+ else latin.push(part)
244
+ }
245
+ const cjkSet = new Set(cjkChars)
246
+ const bigrams = new Set()
247
+ for (let i = 0; i + 1 < cjkChars.length; i += 1) {
248
+ bigrams.add(cjkChars[i] + cjkChars[i + 1])
249
+ }
250
+ return { latin, cjkSet, bigrams }
251
+ }
252
+
168
253
  function scoreRecord(record, queryText, project, evidence) {
169
254
  if (project && record.project && record.project !== project) return 0
170
- const content = `${record.content || ''} ${(record.tags || []).join(' ')} ${record.kind || ''}`
171
- const haystackTokens = tokenize(content)
172
- const haystackLatin = latinTokens(content)
255
+ // Memoized against the record object itself: see haystackFor above.
256
+ const hay = haystackFor(record)
173
257
  const queryTokens = tokenize(queryText)
174
258
  let score = 0
175
259
  for (const token of queryTokens) {
176
- if (haystackTokens.includes(token)) score += 2
260
+ if (token.startsWith('cjk:')) {
261
+ if (hay.cjkSet.has(token.slice(4))) score += 2
262
+ } else if (hay.latin.includes(token)) {
263
+ score += 2
264
+ }
177
265
  }
178
266
  const queryLatin = queryTokens.filter((token) => !token.startsWith('cjk:'))
179
267
  for (const token of queryLatin) {
180
268
  if (
181
269
  token.length >= 3 &&
182
- haystackLatin.some((item) => item.startsWith(token) || token.startsWith(item))
270
+ hay.latin.some((item) => item.startsWith(token) || token.startsWith(item))
183
271
  ) {
184
272
  score += 1
185
273
  }
186
274
  }
187
275
  const queryBigrams = cjkBigrams(queryText)
188
- const haystackBigrams = cjkBigrams(content)
276
+ const haystackBigrams = hay.bigrams
189
277
  if (queryBigrams.size > 0 && haystackBigrams.size > 0) {
190
278
  let matched = 0
191
279
  for (const bigram of queryBigrams) {
@@ -255,7 +343,7 @@ function createMemoryStore(options = {}) {
255
343
 
256
344
  function loadEvidenceWeights() {
257
345
  const weights = new Map()
258
- for (const record of readJsonl(experienceReusePath)) {
346
+ for (const record of readJsonlCached(experienceReusePath)) {
259
347
  if (!verifiedFilter(record)) continue
260
348
  const id = record.experienceId
261
349
  if (!id) continue
@@ -286,16 +374,24 @@ function createMemoryStore(options = {}) {
286
374
  const scoreFor = (record) =>
287
375
  scoreRecord(record, query, project, evidence) +
288
376
  (scope === 'all' && isGlobalRecord(record) ? 0.5 : 0)
289
- const results = readJsonl(memoryPath)
290
- .filter((record) => record.archived !== true)
291
- .filter((record) => matchesMemoryFilters(record, kind, tags))
292
- .filter((record) => matchesProjectScope(record, project, scope))
293
- .filter((record) => scoreFor(record) > 0)
294
- .sort((a, b) => {
295
- return scoreFor(b) - scoreFor(a)
296
- })
377
+ // Score every candidate exactly once, then sort the scored pairs. Calling
378
+ // scoreFor() inside the comparator re-ran the whole tokenizer + CJK bigram
379
+ // scan O(n log n) times instead of O(n): measured on the real 1094-record
380
+ // store, one query invoked the scorer 6214 times (5.7x) and spent 84 ms of
381
+ // its 111 ms there. Sorting precomputed pairs is behaviour-identical --
382
+ // Array#sort is stable, so ties keep their original file order either way.
383
+ const scored = []
384
+ for (const record of readJsonlCached(memoryPath)) {
385
+ if (record.archived === true) continue
386
+ if (!matchesMemoryFilters(record, kind, tags)) continue
387
+ if (!matchesProjectScope(record, project, scope)) continue
388
+ const score = scoreFor(record)
389
+ if (score > 0) scored.push({ record, score })
390
+ }
391
+ const results = scored
392
+ .sort((a, b) => b.score - a.score)
297
393
  .slice(0, limit)
298
- .map((record) => {
394
+ .map(({ record }) => {
299
395
  const ev = evidence.get(record.id)
300
396
  if (!ev) return record
301
397
  return {
@@ -361,7 +457,7 @@ function createMemoryStore(options = {}) {
361
457
  const limit = Math.min(Math.max(Number(args.limit) || 20, 1), 200)
362
458
  const includePayload = args.include_payload !== false
363
459
 
364
- const all = readJsonl(tracePath)
460
+ const all = readJsonlCached(tracePath)
365
461
  const matched = all.filter((trace) => {
366
462
  if (!trace || typeof trace !== 'object') return false
367
463
  if (project && trace.project !== project) return false
@@ -442,7 +538,7 @@ function createMemoryStore(options = {}) {
442
538
  function analyzeMemory(args = {}) {
443
539
  const project = args.project || projectName()
444
540
  const limit = Math.min(Math.max(Number(args.limit) || 5, 1), 20)
445
- const records = readJsonl(memoryPath).filter(
541
+ const records = readJsonlCached(memoryPath).filter(
446
542
  (record) => !project || record.project === project || !record.project
447
543
  )
448
544
  const archived = records.filter((record) => record.archived === true).length
@@ -536,7 +632,7 @@ function createMemoryStore(options = {}) {
536
632
  const project = args.project || projectName()
537
633
  const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
538
634
 
539
- const records = readJsonl(memoryPath)
635
+ const records = readJsonlCached(memoryPath)
540
636
  const idSet = new Set(ids)
541
637
  const updated = []
542
638
  const unchanged = []
@@ -593,7 +689,7 @@ function createMemoryStore(options = {}) {
593
689
  // because "which divergence wins" is then undefined.
594
690
  const allowDivergent = args.allowDivergent === true || args.allowDivergent === 'true'
595
691
 
596
- const records = readJsonl(memoryPath)
692
+ const records = readJsonlCached(memoryPath)
597
693
  const idSet = new Set(ids)
598
694
  // The project filter must be applied once, up front. Applying it only in the
599
695
  // archive loop let a mismatched project still rewrite the survivor's
@@ -736,7 +832,7 @@ function createMemoryStore(options = {}) {
736
832
  const restore = args.restore === true || args.restore === 'true'
737
833
  const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
738
834
 
739
- const records = readJsonl(memoryPath)
835
+ const records = readJsonlCached(memoryPath)
740
836
  const idSet = new Set(ids)
741
837
  const archived = []
742
838
  // Ids that exist but are already in the requested state. Reported separately
@@ -795,7 +891,7 @@ function createMemoryStore(options = {}) {
795
891
  const reason = args.reason ? String(args.reason).trim().slice(0, 200) : null
796
892
  const by = args.by ? String(args.by).trim().slice(0, 80) : null
797
893
 
798
- const records = readJsonl(memoryPath)
894
+ const records = readJsonlCached(memoryPath)
799
895
  const idSet = new Set(ids)
800
896
  const forgotten = []
801
897
  const notFound = [...ids]
@@ -875,4 +971,4 @@ function createMemoryStore(options = {}) {
875
971
 
876
972
  // Module-level exports let sibling stores (e.g. experience-store.js) reuse the
877
973
  // same JSONL primitives instead of duplicating them and drifting apart.
878
- module.exports = { createMemoryStore, createId, readJsonl, appendJsonl, writeJsonl }
974
+ module.exports = { createMemoryStore, createId, readJsonl, readJsonlCached, appendJsonl, writeJsonl }
@@ -13,6 +13,7 @@ const { createMemoryStore } = require('../memory-store.js')
13
13
  const { createExperienceStore, REUSE_STATUS_FILTERS } = require('../experience-store.js')
14
14
  const { createPolicyStore } = require('../policy-store.js')
15
15
  const { createAgentStore } = require('../agent-store.js')
16
+ const { createEvaluationStore } = require('../evaluation-store.js')
16
17
  const { createTaskStore } = require('../task-store.js')
17
18
  const { createQueryLog } = require('../query-log.js')
18
19
  const { createSubscriptionStore } = require('../subscription-store.js')
@@ -216,6 +217,10 @@ const { policyCheck } = policyStore
216
217
  const agentStore = createAgentStore({ dataDir, projectName })
217
218
  const { listAgents, reportAgent, registerAgent } = agentStore
218
219
 
220
+ // ADR-017 evaluation metrics, same store the CLI uses (`mio agents evaluation`).
221
+ const evaluationStore = createEvaluationStore({ dataDir, projectName })
222
+ const evaluateAgents = (args) => evaluationStore.evaluate(args)
223
+
219
224
  // task.route delegates to the shared store so the CLI (`mio task route`) and
220
225
  // this MCP tool route identically. The MCP keeps feeding the query log
221
226
  // (auto-claim depends on it); the CLI passes persistQuery: false.
@@ -541,6 +546,18 @@ const TOOLS = [
541
546
  },
542
547
  },
543
548
  },
549
+ {
550
+ name: 'mio.agent.evaluation',
551
+ description:
552
+ 'ADR-017 evaluation-period metrics: route adoption rate, behaviour change rate, recall quality, and data hygiene (pending auto-claim ratio). Each metric reports its own denominator, so a null rate means "no samples" rather than "zero".',
553
+ inputSchema: {
554
+ type: 'object',
555
+ properties: {
556
+ project: { type: 'string', description: 'Project name filter. Defaults to current repository.' },
557
+ since: { type: 'string', description: 'ISO timestamp; only count records at or after it.' },
558
+ },
559
+ },
560
+ },
544
561
  {
545
562
  name: 'mio.evolution.report',
546
563
  description: 'Generate cross-agent evolution report: ecosystem summary, agent performance comparison, cross-agent patterns, memory health, and optimization suggestions (ADR-017 Evolution plane).',
@@ -843,6 +860,18 @@ const TOOLS = [
843
860
  },
844
861
  },
845
862
  },
863
+ {
864
+ name: 'mio.observer.pipeline',
865
+ description: 'Run the Observer research DAG (collect -> trend -> tension -> research -> multi-brain -> compose -> world model -> publish). Heavy: network plus several LLM calls. Previews by default; pass run:true to execute.',
866
+ inputSchema: {
867
+ type: 'object',
868
+ properties: {
869
+ mode: { type: 'string', description: 'Writing mode: neutral | analytical | creative (default: analytical)' },
870
+ run: { type: 'boolean', description: 'Execute the pipeline. Default false returns a dry-run plan without network or LLM calls.' },
871
+ baseDir: { type: 'string', description: 'Observer data directory' },
872
+ },
873
+ },
874
+ },
846
875
  {
847
876
  name: 'mio.observer.essays',
848
877
  description: 'List published or draft essays from the Observer.',
@@ -924,6 +953,10 @@ function observerFerment(args = {}) {
924
953
  return observerStore.ferment(args)
925
954
  }
926
955
 
956
+ function observerPipeline(args = {}) {
957
+ return observerStore.pipeline(args)
958
+ }
959
+
927
960
  function observerEssays(args = {}) {
928
961
  return observerStore.essays(args)
929
962
  }
@@ -976,6 +1009,8 @@ async function callTool(name, args = {}) {
976
1009
  return listAgents(args)
977
1010
  case 'mio.agent.report':
978
1011
  return reportAgent(args)
1012
+ case 'mio.agent.evaluation':
1013
+ return evaluateAgents(args)
979
1014
  case 'mio.evolution.report':
980
1015
  return evolutionReport(args)
981
1016
  case 'mio.evolution.status':
@@ -1026,6 +1061,8 @@ async function callTool(name, args = {}) {
1026
1061
  return observerCollect(args)
1027
1062
  case 'mio.observer.ferment':
1028
1063
  return observerFerment(args)
1064
+ case 'mio.observer.pipeline':
1065
+ return observerPipeline(args)
1029
1066
  case 'mio.observer.essays':
1030
1067
  return observerEssays(args)
1031
1068
  case 'mio.observer.dag':
@@ -1,7 +1,7 @@
1
1
  'use strict'
2
2
 
3
- const fs = require('fs')
4
3
  const path = require('path')
4
+ const { readJsonlCached } = require('../memory-store.js')
5
5
 
6
6
  const PHASE0_THRESHOLDS = Object.freeze({
7
7
  minHosts: 2,
@@ -55,22 +55,6 @@ function normalizeEventType(value) {
55
55
  return eventType || null
56
56
  }
57
57
 
58
- function readJsonl(filePath) {
59
- if (!fs.existsSync(filePath)) return []
60
- return fs
61
- .readFileSync(filePath, 'utf8')
62
- .split(/\r?\n/)
63
- .filter(Boolean)
64
- .map((line) => {
65
- try {
66
- return JSON.parse(line)
67
- } catch (_) {
68
- return null
69
- }
70
- })
71
- .filter(Boolean)
72
- }
73
-
74
58
  function filterByProject(records, project) {
75
59
  if (!project) return records
76
60
  return records.filter((record) => readString(record, 'project') === project)
@@ -279,9 +263,9 @@ function analyzePhase0(dataset) {
279
263
 
280
264
  function loadPhase0(dataDir, project) {
281
265
  const normalizedProject = project && project.trim() ? project.trim() : undefined
282
- const memories = readJsonl(path.join(dataDir, 'memory.jsonl')).map(asRecord).filter(Boolean)
283
- const traces = readJsonl(path.join(dataDir, 'traces.jsonl')).map(asRecord).filter(Boolean)
284
- const reuseRecords = readJsonl(path.join(dataDir, 'experience_reuse.jsonl'))
266
+ const memories = readJsonlCached(path.join(dataDir, 'memory.jsonl')).map(asRecord).filter(Boolean)
267
+ const traces = readJsonlCached(path.join(dataDir, 'traces.jsonl')).map(asRecord).filter(Boolean)
268
+ const reuseRecords = readJsonlCached(path.join(dataDir, 'experience_reuse.jsonl'))
285
269
  .map(asRecord)
286
270
  .filter(Boolean)
287
271
 
@@ -14,6 +14,19 @@
14
14
  const fs = require('fs')
15
15
  const path = require('path')
16
16
 
17
+ // The observer pipeline is one of the LLM users, so it must read the same
18
+ // resolved model configuration as creativity / insight. `isLlmConfigured()`
19
+ // gates the override: when the user never configured an LLM we pass nothing and
20
+ // let @akemi-mio/observer keep its own LLM_* / OBSERVER_* / Ollama fallback, so
21
+ // an existing local-Ollama setup is not silently redirected to the hosted
22
+ // default. This is the D2 fix -- the observer used to hardcode Ollama and
23
+ // ignore `mio config llm` entirely.
24
+ const { llmConfig, isLlmConfigured } = require('./llm-client.js')
25
+
26
+ function observerLlmConfig() {
27
+ return isLlmConfigured() ? llmConfig() : undefined
28
+ }
29
+
17
30
  // Observer research pipeline (optional -- installed via @akemi-mio/observer).
18
31
  // Loaded lazily at module level so the CLI can report availability without the
19
32
  // MCP server having to pass its own handles down.
@@ -31,10 +44,12 @@ function loadObserver() {
31
44
 
32
45
  let ObserverStore = null
33
46
  let ObserverService = null
47
+ let ObserverLlmService = null
34
48
  try {
35
49
  const obs = loadObserver()
36
50
  ObserverStore = obs.ObserverStore
37
51
  ObserverService = obs.ObserverService
52
+ ObserverLlmService = obs.ObserverLlmService
38
53
  } catch {}
39
54
 
40
55
  // Subdirectories that make up the pipeline. `observerStatus` counts files in
@@ -62,6 +77,17 @@ function readJsonSafe(filePath) {
62
77
  }
63
78
  }
64
79
 
80
+ // Today's DAG task id, matching DagStateMachine's `dag_YYYYMMDD` (LOCAL date).
81
+ // Duplicated here rather than importing DagStateMachine because its constructor
82
+ // mkdir's the dag directory, and the dry run below must not touch disk.
83
+ function todayTaskId() {
84
+ const d = new Date()
85
+ const y = d.getFullYear()
86
+ const m = String(d.getMonth() + 1).padStart(2, '0')
87
+ const day = String(d.getDate()).padStart(2, '0')
88
+ return `dag_${y}${m}${day}`
89
+ }
90
+
65
91
  function readJsonlSafe(filePath) {
66
92
  if (!fs.existsSync(filePath)) return []
67
93
  try {
@@ -184,7 +210,7 @@ function createObserverStore(options = {}) {
184
210
  // object made path.resolve() throw before any collector ran, so collect()
185
211
  // was dead on arrival for every caller. (collectorConfig was never read by
186
212
  // the package; it was invented here.)
187
- const service = new ObserverService(baseDirOf(args))
213
+ const service = new ObserverService(baseDirOf(args), observerLlmConfig())
188
214
  const sources = args.sources || ['bilibili', 'hackernews', 'github', 'douyin', 'rss']
189
215
  const allObs = []
190
216
  for (const src of sources) {
@@ -210,7 +236,7 @@ function createObserverStore(options = {}) {
210
236
 
211
237
  async function ferment(args = {}) {
212
238
  if (!ObserverService) throw new Error('@akemi-mio/observer not installed')
213
- const service = new ObserverService(baseDirOf(args))
239
+ const service = new ObserverService(baseDirOf(args), observerLlmConfig())
214
240
  const session = args.session || 'afternoon'
215
241
  try {
216
242
  const fermentation = service.getFermentation()
@@ -221,6 +247,81 @@ function createObserverStore(options = {}) {
221
247
  }
222
248
  }
223
249
 
250
+ // Drives the full research DAG (collect → trend → tension → research →
251
+ // multi-brain → compose → world model → publish → self-evolve). This is the
252
+ // entry point D1 found missing: runPipeline/tickPipeline/forcePipeline had no
253
+ // caller anywhere in mio-agent-runtime, so trends/research/insights stayed
254
+ // empty forever unless someone hand-instantiated the service.
255
+ //
256
+ // It is heavy (network + several LLM calls) and writes files, so like every
257
+ // other side-effecting command here it previews by default: without
258
+ // `run: true` it reads today's DAG state and the resolved LLM endpoint and
259
+ // returns a plan without constructing the service, which would mkdir the dag
260
+ // directory. That keeps the README doc gate and the MCP live gate (both call
261
+ // with empty arguments) offline and instant.
262
+ async function pipeline(args = {}) {
263
+ if (!ObserverService) throw new Error('@akemi-mio/observer not installed')
264
+ const baseDir = baseDirOf(args)
265
+ const mode = args.mode || 'analytical'
266
+ const llmConfig = observerLlmConfig()
267
+
268
+ if (!args.run) {
269
+ const taskId = todayTaskId()
270
+ const dag = readJsonSafe(path.join(baseDir, 'dag', `${taskId}.json`))
271
+ // Guard the diagnostic: an older published @akemi-mio/observer lacks
272
+ // ObserverLlmService.getInfo(), and a preview must never throw because of
273
+ // a package-version skew (the mio-cli side ships ahead of the package).
274
+ let llm = null
275
+ if (ObserverLlmService) {
276
+ try {
277
+ const svc = new ObserverLlmService(llmConfig)
278
+ if (svc && typeof svc.getInfo === 'function') llm = svc.getInfo()
279
+ } catch (_) {
280
+ llm = null
281
+ }
282
+ }
283
+ return {
284
+ dryRun: true,
285
+ baseDir,
286
+ mode,
287
+ observerAvailable: true,
288
+ taskId,
289
+ todayState: dag ? dag.state : null,
290
+ todayCompleted: Boolean(dag && dag.state === 'COMPLETED'),
291
+ llm,
292
+ hint: 'Re-run with --run (CLI) or {"run": true} (MCP) to execute the full DAG.',
293
+ }
294
+ }
295
+
296
+ const service = new ObserverService(baseDir, llmConfig)
297
+ const envelope = await service.forcePipeline(mode)
298
+ if (!envelope) {
299
+ return {
300
+ dryRun: false,
301
+ baseDir,
302
+ mode,
303
+ ran: true,
304
+ completed: false,
305
+ reason: 'forcePipeline returned null (today already completed, or a run is in progress)',
306
+ }
307
+ }
308
+ const payload = envelope.payload || {}
309
+ return {
310
+ dryRun: false,
311
+ baseDir,
312
+ mode,
313
+ ran: true,
314
+ completed: envelope.dagState && envelope.dagState.state === 'COMPLETED',
315
+ type: envelope.type,
316
+ taskId: envelope.dagState ? envelope.dagState.taskId : null,
317
+ topic: payload.topic || null,
318
+ sections: Array.isArray(payload.sections) ? payload.sections.length : 0,
319
+ insightId: payload.id || null,
320
+ durationMs: envelope.metadata ? envelope.metadata.taskDurationMs : null,
321
+ llmCalls: envelope.metadata ? envelope.metadata.llmCalls : null,
322
+ }
323
+ }
324
+
224
325
  return {
225
326
  baseDir: defaultBaseDir,
226
327
  status,
@@ -232,6 +333,7 @@ function createObserverStore(options = {}) {
232
333
  dag,
233
334
  collect,
234
335
  ferment,
336
+ pipeline,
235
337
  }
236
338
  }
237
339
 
@@ -12,7 +12,7 @@
12
12
 
13
13
  const path = require('path')
14
14
 
15
- const { readJsonl } = require('./memory-store.js')
15
+ const { readJsonlCached } = require('./memory-store.js')
16
16
 
17
17
  // Outcomes that count against an action. `retry` is included because a retried
18
18
  // action is evidence the first attempt did not simply work.
@@ -139,7 +139,7 @@ function createPolicyStore(options = {}) {
139
139
  if (!action) throw new Error('policy.check requires action')
140
140
  const project = args.project || projectName()
141
141
  const actionTokens = tokenize(action)
142
- const allTraces = readJsonl(tracePath)
142
+ const allTraces = readJsonlCached(tracePath)
143
143
  const scopedTraces = allTraces.filter(
144
144
  (event) => !(project && event.project && event.project !== project)
145
145
  )
@@ -186,7 +186,7 @@ function createPolicyStore(options = {}) {
186
186
  suggestion = 'Low historical risk. Proceed with normal checks.'
187
187
  }
188
188
  const evidence = loadEvidenceWeights()
189
- const relatedMemories = readJsonl(memoryPath)
189
+ const relatedMemories = readJsonlCached(memoryPath)
190
190
  .filter((record) => scoreRecord(record, action, project, evidence) > 0)
191
191
  .sort(
192
192
  (a, b) => scoreRecord(b, action, project, evidence) - scoreRecord(a, action, project, evidence)
@@ -12,9 +12,8 @@
12
12
  // mio-intelligence-mcp/index.js, one in task-store.js -- which had to be kept
13
13
  // equivalent by hand.
14
14
 
15
- const fs = require('fs')
16
15
  const path = require('path')
17
- const { readJsonl } = require('./memory-store.js')
16
+ const { readJsonlCached, writeJsonl } = require('./memory-store.js')
18
17
 
19
18
  const MAX_RECENT_QUERIES = 200
20
19
 
@@ -31,7 +30,7 @@ function createQueryLog(options = {}) {
31
30
 
32
31
  function load() {
33
32
  const now = Date.now()
34
- return readJsonl(queryPath)
33
+ return readJsonlCached(queryPath)
35
34
  .filter(
36
35
  (entry) =>
37
36
  entry &&
@@ -45,12 +44,7 @@ function createQueryLog(options = {}) {
45
44
 
46
45
  function persist(entries) {
47
46
  const limited = entries.slice(-MAX_RECENT_QUERIES)
48
- fs.mkdirSync(dataDir, { recursive: true })
49
- fs.writeFileSync(
50
- queryPath,
51
- limited.length > 0 ? limited.map((entry) => JSON.stringify(entry)).join('\n') + '\n' : '',
52
- 'utf8'
53
- )
47
+ writeJsonl(queryPath, limited)
54
48
  return limited
55
49
  }
56
50
 
@@ -8,6 +8,7 @@
8
8
 
9
9
  const fs = require('fs')
10
10
  const path = require('path')
11
+ const { writeJsonl } = require('./memory-store.js')
11
12
 
12
13
  // Age-based JSONL stores (records carry an ISO `timestamp` field).
13
14
  const AGE_BASED_FILES = ['traces.jsonl', 'experience_reuse.jsonl']
@@ -41,15 +42,6 @@ function createRetention(options = {}) {
41
42
  return backup
42
43
  }
43
44
 
44
- function writeJsonl(file, values) {
45
- fs.mkdirSync(path.dirname(file), { recursive: true })
46
- fs.writeFileSync(
47
- file,
48
- values.length > 0 ? values.map((value) => JSON.stringify(value)).join('\n') + '\n' : '',
49
- 'utf8'
50
- )
51
- }
52
-
53
45
  function planAgeBased(name, cutoff) {
54
46
  const file = path.join(home, name)
55
47
  const records = readJsonl(file)