nlm-memory 0.4.2 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (285) hide show
  1. package/README.md +72 -34
  2. package/dist/cli/nlm.js +223 -33
  3. package/dist/cli/nlm.js.map +1 -1
  4. package/dist/core/adapters/cursor.d.ts +45 -0
  5. package/dist/core/adapters/cursor.js +397 -0
  6. package/dist/core/adapters/cursor.js.map +1 -0
  7. package/dist/core/adapters/from-source.js +10 -0
  8. package/dist/core/adapters/from-source.js.map +1 -1
  9. package/dist/core/adapters/windsurf.d.ts +44 -0
  10. package/dist/core/adapters/windsurf.js +299 -0
  11. package/dist/core/adapters/windsurf.js.map +1 -0
  12. package/dist/core/hook/claude-settings.d.ts +12 -5
  13. package/dist/core/hook/claude-settings.js +21 -6
  14. package/dist/core/hook/claude-settings.js.map +1 -1
  15. package/dist/core/sources/source-registry.d.ts +1 -1
  16. package/dist/core/sources/source-registry.js +18 -0
  17. package/dist/core/sources/source-registry.js.map +1 -1
  18. package/dist/core/storage/sqlite-session-store.d.ts +2 -0
  19. package/dist/core/storage/sqlite-session-store.js +38 -2
  20. package/dist/core/storage/sqlite-session-store.js.map +1 -1
  21. package/dist/hook/hook-auth.d.ts +13 -0
  22. package/dist/hook/hook-auth.js +19 -0
  23. package/dist/hook/hook-auth.js.map +1 -0
  24. package/dist/hook/prompt-recall-hook.js +7 -1
  25. package/dist/hook/prompt-recall-hook.js.map +1 -1
  26. package/dist/hook/session-start-hook.js +4 -1
  27. package/dist/hook/session-start-hook.js.map +1 -1
  28. package/dist/hook/stop-hook.js +4 -1
  29. package/dist/hook/stop-hook.js.map +1 -1
  30. package/dist/http/app.d.ts +2 -0
  31. package/dist/http/app.js +76 -1
  32. package/dist/http/app.js.map +1 -1
  33. package/dist/install/claude-code.js +1 -1
  34. package/dist/install/claude-code.js.map +1 -1
  35. package/dist/install/cursor.d.ts +25 -0
  36. package/dist/install/cursor.js +43 -0
  37. package/dist/install/cursor.js.map +1 -0
  38. package/dist/install/nlm-dir-perms.d.ts +19 -0
  39. package/dist/install/nlm-dir-perms.js +43 -0
  40. package/dist/install/nlm-dir-perms.js.map +1 -0
  41. package/dist/install/ollama.d.ts +18 -1
  42. package/dist/install/ollama.js +62 -7
  43. package/dist/install/ollama.js.map +1 -1
  44. package/dist/install/setup.d.ts +4 -0
  45. package/dist/install/setup.js +141 -18
  46. package/dist/install/setup.js.map +1 -1
  47. package/dist/install/windsurf.d.ts +25 -0
  48. package/dist/install/windsurf.js +43 -0
  49. package/dist/install/windsurf.js.map +1 -0
  50. package/dist/mcp/server.js +20 -1
  51. package/dist/mcp/server.js.map +1 -1
  52. package/dist/shared/types.d.ts +4 -0
  53. package/dist/ui/assets/{index-BA6IpU8g.css → index-Beo8psd-.css} +1 -1
  54. package/dist/ui/assets/index-CSPTTeeM.js +69 -0
  55. package/dist/ui/index.html +2 -2
  56. package/package.json +26 -1
  57. package/plugin/scripts/prompt-recall-hook.mjs +55 -4
  58. package/plugin/scripts/stop-hook.mjs +57 -6
  59. package/.agents/plugins/marketplace.json +0 -20
  60. package/.github/workflows/ci.yml +0 -30
  61. package/dist/ui/assets/index-B_qIVV0k.js +0 -69
  62. package/docs/methodology/re-derivation-rate.md +0 -112
  63. package/docs/methodology/useful-hit-rate.md +0 -79
  64. package/docs/plans/2026-05-20-fts5-lexical-recall.md +0 -1088
  65. package/docs/plans/2026-05-20-recall-daemon-wedge-fix.md +0 -662
  66. package/docs/plans/2026-05-20-recall-hook-design.md +0 -131
  67. package/docs/plans/2026-05-20-recall-hook-implementation.md +0 -1222
  68. package/docs/plans/desktop-product.md +0 -69
  69. package/docs/plans/factstore-design.md +0 -236
  70. package/logs/CHANGELOG/CHANGELOG-2026.md +0 -1389
  71. package/logs/CHANGELOG/CHANGELOG.md +0 -337
  72. package/migrations/000_initial_schema.sql +0 -174
  73. package/migrations/001_entity_type_rename.sql +0 -17
  74. package/migrations/002_adapter_state_extend.sql +0 -12
  75. package/migrations/003_session_embeddings.sql +0 -11
  76. package/migrations/004_facts.sql +0 -46
  77. package/migrations/005_sources.sql +0 -31
  78. package/migrations/006_providers.sql +0 -33
  79. package/migrations/007_source_tokens.sql +0 -17
  80. package/migrations/008_fts_rebuild.sql +0 -9
  81. package/migrations/009_session_embedding_chunks.sql +0 -46
  82. package/migrations/010_sources_opencode.sql +0 -30
  83. package/migrations/011_sources_hermes_agent.sql +0 -30
  84. package/migrations/012_sources_aider.sql +0 -30
  85. package/migrations/013_adapter_state_failure_count.sql +0 -12
  86. package/plugin-hermes-agent/README.md +0 -49
  87. package/plugin-hermes-agent/__init__.py +0 -75
  88. package/plugin-hermes-agent/plugin.yaml +0 -15
  89. package/scripts/backfill-citations.mjs +0 -0
  90. package/scripts/build-codex-plugin.mjs +0 -61
  91. package/scripts/deepseek-probe.mjs +0 -67
  92. package/scripts/extract-triples.mjs +0 -207
  93. package/scripts/longmemeval/embedding-cache.ts +0 -77
  94. package/scripts/longmemeval/fetch-dataset.sh +0 -25
  95. package/scripts/longmemeval/run-harness.ts +0 -315
  96. package/scripts/longmemeval/scorer.ts +0 -99
  97. package/scripts/longmemeval/tsconfig.json +0 -9
  98. package/scripts/longmemeval/types.ts +0 -35
  99. package/scripts/nlm-daily-digest.py +0 -239
  100. package/scripts/nlm-daily-digest.sh +0 -28
  101. package/src/cli/classify-parity.ts +0 -257
  102. package/src/cli/launchctl-helpers.ts +0 -49
  103. package/src/cli/nlm.ts +0 -885
  104. package/src/core/actions/actions-log.ts +0 -118
  105. package/src/core/actions/overlay.ts +0 -117
  106. package/src/core/adapters/aider.ts +0 -205
  107. package/src/core/adapters/claude-code.ts +0 -293
  108. package/src/core/adapters/common.ts +0 -54
  109. package/src/core/adapters/from-source.ts +0 -57
  110. package/src/core/adapters/hermes-agent.ts +0 -240
  111. package/src/core/adapters/hermes.ts +0 -277
  112. package/src/core/adapters/jsonl-generic.ts +0 -208
  113. package/src/core/adapters/opencode.ts +0 -281
  114. package/src/core/adapters/pi.ts +0 -264
  115. package/src/core/classifier/prompt.ts +0 -200
  116. package/src/core/dataset/build-dataset.ts +0 -463
  117. package/src/core/embedding/chunk-body.ts +0 -76
  118. package/src/core/embedding/embed-backfill.ts +0 -210
  119. package/src/core/embedding/embed-normalize.ts +0 -135
  120. package/src/core/facts/backfill-facts.ts +0 -254
  121. package/src/core/facts/extract-facts.ts +0 -50
  122. package/src/core/hook/citation-detect.ts +0 -124
  123. package/src/core/hook/cite-memo.ts +0 -68
  124. package/src/core/hook/claude-settings.ts +0 -166
  125. package/src/core/hook/gate.ts +0 -25
  126. package/src/core/hook/hook-log.ts +0 -41
  127. package/src/core/hook/memo-sweep.ts +0 -164
  128. package/src/core/hook/memo.ts +0 -67
  129. package/src/core/hook/pointer-block.ts +0 -26
  130. package/src/core/hook/select.ts +0 -32
  131. package/src/core/hook/transcript.ts +0 -121
  132. package/src/core/ingest/ingest-session.ts +0 -111
  133. package/src/core/providers/provider-models.ts +0 -100
  134. package/src/core/providers/provider-registry.ts +0 -196
  135. package/src/core/recall/citation-log.ts +0 -108
  136. package/src/core/recall/filter.ts +0 -27
  137. package/src/core/recall/index.ts +0 -6
  138. package/src/core/recall/match-fields.ts +0 -40
  139. package/src/core/recall/query-log.ts +0 -149
  140. package/src/core/recall/query-shape.ts +0 -66
  141. package/src/core/recall/recall-service.ts +0 -320
  142. package/src/core/recall/recent-log.ts +0 -59
  143. package/src/core/recall/tokenize.ts +0 -18
  144. package/src/core/recall/useful-scan.ts +0 -336
  145. package/src/core/recall-facts/fact-query-log.ts +0 -150
  146. package/src/core/recall-facts/fact-recall-service.ts +0 -327
  147. package/src/core/scheduler/scan-once.ts +0 -142
  148. package/src/core/scheduler/scheduler.ts +0 -225
  149. package/src/core/sources/source-registry.ts +0 -260
  150. package/src/core/storage/db-restore.ts +0 -133
  151. package/src/core/storage/live-status.ts +0 -45
  152. package/src/core/storage/migrate.ts +0 -72
  153. package/src/core/storage/sqlite-fact-store.ts +0 -304
  154. package/src/core/storage/sqlite-session-store.ts +0 -765
  155. package/src/hook/prompt-recall-hook.ts +0 -174
  156. package/src/hook/session-end-hook.ts +0 -81
  157. package/src/hook/session-start-hook.ts +0 -165
  158. package/src/hook/stop-hook.ts +0 -236
  159. package/src/http/app.ts +0 -1137
  160. package/src/install/claude-code.ts +0 -128
  161. package/src/install/codex.ts +0 -367
  162. package/src/install/hermes-agent.ts +0 -76
  163. package/src/install/hermes.ts +0 -78
  164. package/src/install/ollama.ts +0 -211
  165. package/src/install/setup.ts +0 -368
  166. package/src/llm/classifier-box.ts +0 -64
  167. package/src/llm/deepseek-client.ts +0 -150
  168. package/src/llm/env-autoload.ts +0 -55
  169. package/src/llm/ollama-client.ts +0 -189
  170. package/src/mcp/server.ts +0 -534
  171. package/src/ports/fact-store.ts +0 -102
  172. package/src/ports/llm-client.ts +0 -52
  173. package/src/ports/logger.ts +0 -16
  174. package/src/ports/session-store.ts +0 -45
  175. package/src/ports/transcript-adapter.ts +0 -55
  176. package/src/shared/types.ts +0 -145
  177. package/src/ui/App.tsx +0 -58
  178. package/src/ui/components/PromoteOpenButton.tsx +0 -65
  179. package/src/ui/components/SessionDrawer.tsx +0 -136
  180. package/src/ui/components/SideNav.tsx +0 -162
  181. package/src/ui/components/Skeleton.tsx +0 -107
  182. package/src/ui/index.html +0 -13
  183. package/src/ui/lib/actions.ts +0 -30
  184. package/src/ui/lib/api.ts +0 -92
  185. package/src/ui/lib/dataset.ts +0 -141
  186. package/src/ui/lib/registries.ts +0 -155
  187. package/src/ui/lib/view-settings.ts +0 -41
  188. package/src/ui/main.tsx +0 -15
  189. package/src/ui/pages/Live.tsx +0 -229
  190. package/src/ui/pages/Pulse.tsx +0 -415
  191. package/src/ui/pages/Recall.tsx +0 -190
  192. package/src/ui/pages/River.tsx +0 -308
  193. package/src/ui/pages/Search.tsx +0 -93
  194. package/src/ui/pages/Stub.tsx +0 -9
  195. package/src/ui/pages/Thread.tsx +0 -262
  196. package/src/ui/pages/settings/Classifier.tsx +0 -227
  197. package/src/ui/pages/settings/Data.tsx +0 -190
  198. package/src/ui/pages/settings/Index.tsx +0 -65
  199. package/src/ui/pages/settings/Labels.tsx +0 -224
  200. package/src/ui/pages/settings/Providers.tsx +0 -305
  201. package/src/ui/pages/settings/SettingsSubnav.tsx +0 -28
  202. package/src/ui/pages/settings/Sources.tsx +0 -326
  203. package/src/ui/pages/settings/Views.tsx +0 -96
  204. package/src/ui/styles.css +0 -1766
  205. package/src/ui/tsconfig.json +0 -21
  206. package/src/ui/vite.config.ts +0 -19
  207. package/tests/fixtures/claude_code/short_session.jsonl +0 -2
  208. package/tests/fixtures/claude_code/standard_iso.jsonl +0 -4
  209. package/tests/fixtures/claude_code/tool_heavy.jsonl +0 -8
  210. package/tests/fixtures/claude_code/with_subagent.jsonl +0 -7
  211. package/tests/fixtures/facts.ts +0 -17
  212. package/tests/fixtures/golden-corpus.ts +0 -85
  213. package/tests/fixtures/hermes/paired_request_dump.json +0 -24
  214. package/tests/fixtures/hermes/paired_session.json +0 -23
  215. package/tests/fixtures/hermes/request_dump.json +0 -28
  216. package/tests/fixtures/hermes/session_iso.json +0 -38
  217. package/tests/fixtures/hermes/session_unix.json +0 -38
  218. package/tests/fixtures/hermes/system_only.json +0 -18
  219. package/tests/fixtures/pi/error-connection-abort.jsonl +0 -8
  220. package/tests/fixtures/pi/short-successful.jsonl +0 -5
  221. package/tests/fixtures/pi/with-custom-message.jsonl +0 -6
  222. package/tests/fixtures/sessions.ts +0 -22
  223. package/tests/integration/backfill-facts.test.ts +0 -362
  224. package/tests/integration/citation-explicit.test.ts +0 -111
  225. package/tests/integration/cite-event.test.ts +0 -169
  226. package/tests/integration/cite-memo.test.ts +0 -87
  227. package/tests/integration/db-restore.test.ts +0 -153
  228. package/tests/integration/embed-backfill.test.ts +0 -176
  229. package/tests/integration/fact-supersedence.test.ts +0 -313
  230. package/tests/integration/fts-index.test.ts +0 -60
  231. package/tests/integration/getbyids-sqlite.test.ts +0 -60
  232. package/tests/integration/hermes-agent-hooks.test.ts +0 -248
  233. package/tests/integration/hook-claude-settings.test.ts +0 -205
  234. package/tests/integration/hook-log.test.ts +0 -54
  235. package/tests/integration/hook-memo.test.ts +0 -68
  236. package/tests/integration/hook-pre-compact.test.ts +0 -105
  237. package/tests/integration/hook-subagent-start.test.ts +0 -102
  238. package/tests/integration/http.test.ts +0 -401
  239. package/tests/integration/keyword-search-fts.test.ts +0 -66
  240. package/tests/integration/mcp-recall-logging.test.ts +0 -88
  241. package/tests/integration/mcp.test.ts +0 -248
  242. package/tests/integration/memo-sweep.test.ts +0 -91
  243. package/tests/integration/prompt-recall-hook.test.ts +0 -88
  244. package/tests/integration/provider-registry.test.ts +0 -107
  245. package/tests/integration/recall-golden.test.ts +0 -59
  246. package/tests/integration/recall-sqlite.test.ts +0 -169
  247. package/tests/integration/scheduler.test.ts +0 -391
  248. package/tests/integration/session-end-hook.test.ts +0 -48
  249. package/tests/integration/session-start-hook.test.ts +0 -126
  250. package/tests/integration/source-registry.test.ts +0 -120
  251. package/tests/integration/sqlite-fact-store.test.ts +0 -346
  252. package/tests/integration/stop-hook.test.ts +0 -560
  253. package/tests/integration/wal-checkpoint.test.ts +0 -49
  254. package/tests/unit/cli/launchctl-helpers.test.ts +0 -60
  255. package/tests/unit/core/adapters/aider.test.ts +0 -230
  256. package/tests/unit/core/adapters/claude-code.test.ts +0 -118
  257. package/tests/unit/core/adapters/hermes-agent.test.ts +0 -329
  258. package/tests/unit/core/adapters/hermes.test.ts +0 -81
  259. package/tests/unit/core/adapters/jsonl-generic.test.ts +0 -142
  260. package/tests/unit/core/adapters/opencode.test.ts +0 -354
  261. package/tests/unit/core/adapters/pi.test.ts +0 -110
  262. package/tests/unit/core/classifier/prompt.test.ts +0 -126
  263. package/tests/unit/core/embedding/chunk-body.test.ts +0 -100
  264. package/tests/unit/core/facts/extract-facts.test.ts +0 -117
  265. package/tests/unit/core/filter.test.ts +0 -40
  266. package/tests/unit/core/hook/citation-detect-cite-session.test.ts +0 -96
  267. package/tests/unit/core/hook/citation-detect.test.ts +0 -124
  268. package/tests/unit/core/hook/gate.test.ts +0 -29
  269. package/tests/unit/core/hook/pointer-block.test.ts +0 -22
  270. package/tests/unit/core/hook/select.test.ts +0 -66
  271. package/tests/unit/core/match-fields.test.ts +0 -39
  272. package/tests/unit/core/mcp-cite-session.test.ts +0 -51
  273. package/tests/unit/core/providers/provider-models.test.ts +0 -101
  274. package/tests/unit/core/query-shape.test.ts +0 -92
  275. package/tests/unit/core/recall-facts/fact-recall-service.test.ts +0 -258
  276. package/tests/unit/core/recall-service.test.ts +0 -200
  277. package/tests/unit/core/storage/live-status.test.ts +0 -54
  278. package/tests/unit/core/tokenize.test.ts +0 -32
  279. package/tests/unit/core/useful-scan.test.ts +0 -537
  280. package/tests/unit/llm/embed.test.ts +0 -93
  281. package/tests/unit/llm/ollama-client.test.ts +0 -124
  282. package/tests/unit/scripts/longmemeval-scorer.test.ts +0 -114
  283. package/tsconfig.json +0 -31
  284. package/tsconfig.test.json +0 -11
  285. package/vitest.config.ts +0 -22
@@ -1,362 +0,0 @@
1
- /**
2
- * Phase B.5 — backfillFacts integration. Seeds a real SQLite store with
3
- * sessions that have no facts, runs the backfill module against a stub
4
- * classifier, and verifies facts land + supersedence fires + state file
5
- * resumes correctly.
6
- */
7
-
8
- import { mkdtempSync, rmSync, existsSync, readFileSync } from "node:fs";
9
- import { tmpdir } from "node:os";
10
- import { join, resolve } from "node:path";
11
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
12
- import { backfillFacts } from "../../src/core/facts/backfill-facts.js";
13
- import { SqliteFactStore } from "../../src/core/storage/sqlite-fact-store.js";
14
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
15
- import type {
16
- ClassifyResult,
17
- EmbedResult,
18
- ExtractedFact,
19
- LLMClient,
20
- } from "../../src/ports/llm-client.js";
21
- import { LLMUnreachableError } from "../../src/ports/llm-client.js";
22
- import { makeSession } from "../fixtures/sessions.js";
23
-
24
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
25
-
26
- class ScriptedClassifier implements LLMClient {
27
- calls: string[] = [];
28
- constructor(
29
- private readonly results: Map<string, ClassifyResult>,
30
- private readonly errorIds: Set<string> = new Set(),
31
- ) {}
32
- async embed(): Promise<EmbedResult> {
33
- throw new Error("not used");
34
- }
35
- async classify(transcript: string): Promise<ClassifyResult> {
36
- this.calls.push(transcript);
37
- if (this.errorIds.has(transcript)) {
38
- throw new LLMUnreachableError("test-stub");
39
- }
40
- const result = this.results.get(transcript);
41
- if (!result) throw new Error(`no scripted result for transcript: ${transcript.slice(0, 60)}`);
42
- return result;
43
- }
44
- }
45
-
46
- function classifyResult(
47
- facts: ExtractedFact[],
48
- confidence = 0.9,
49
- ): ClassifyResult {
50
- return {
51
- label: "L",
52
- summary: "S",
53
- entities: [],
54
- decisions: [],
55
- open: [],
56
- confidence,
57
- facts,
58
- };
59
- }
60
-
61
- describe("backfillFacts", () => {
62
- let tmp: string;
63
- let store: SqliteSessionStore;
64
- let factStore: SqliteFactStore;
65
- let statePath: string;
66
-
67
- beforeEach(() => {
68
- tmp = mkdtempSync(join(tmpdir(), "nlm-b5-"));
69
- store = new SqliteSessionStore({
70
- dbPath: join(tmp, "canonical.sqlite"),
71
- migrationsDir: MIGRATIONS_DIR,
72
- });
73
- factStore = new SqliteFactStore(store.rawDb());
74
- statePath = join(tmp, "backfill_facts.state");
75
- });
76
-
77
- afterEach(() => {
78
- store.close();
79
- rmSync(tmp, { recursive: true, force: true });
80
- });
81
-
82
- it("writes facts for sessions that have none, skips sessions with existing facts", async () => {
83
- // Two sessions need backfill; one already has a fact.
84
- store.insertSessionForTest(makeSession({
85
- id: "sess_old1", body: "BODY-OLD-1", startedAt: "2026-05-17T10:00:00Z",
86
- }));
87
- store.insertSessionForTest(makeSession({
88
- id: "sess_old2", body: "BODY-OLD-2", startedAt: "2026-05-17T11:00:00Z",
89
- }));
90
- store.insertSessionForTest(makeSession({
91
- id: "sess_done", body: "BODY-DONE", startedAt: "2026-05-17T12:00:00Z",
92
- }));
93
- // Pre-existing fact on sess_done — backfill should skip it.
94
- await factStore.insert({
95
- id: "f_pre", kind: "decision", subject: "x", predicate: "framework",
96
- value: "v", sourceSessionId: "sess_done", sourceQuote: null,
97
- createdAt: "2026-05-17T12:00:00Z", supersededBy: null, confidence: 0.9,
98
- });
99
-
100
- const classifier = new ScriptedClassifier(new Map([
101
- ["BODY-OLD-1", classifyResult([
102
- { kind: "decision", subject: "nlm-memory-ts", predicate: "framework", value: "Hono" },
103
- ])],
104
- ["BODY-OLD-2", classifyResult([
105
- { kind: "attribute", subject: "mac-pro", predicate: "endpoint", value: ":8080" },
106
- { kind: "attribute", subject: "mac-pro", predicate: "model", value: "qwen2.5-3b" },
107
- ])],
108
- ]));
109
-
110
- const report = await backfillFacts({
111
- store, factStore, classifier, statePath,
112
- });
113
-
114
- expect(report.total).toBe(2); // sess_done excluded by the NOT EXISTS clause
115
- expect(report.processed).toBe(2);
116
- expect(report.factsWritten).toBe(3);
117
- expect(classifier.calls).toHaveLength(2);
118
-
119
- expect(await factStore.listBySession("sess_old1")).toHaveLength(1);
120
- expect(await factStore.listBySession("sess_old2")).toHaveLength(2);
121
- expect(await factStore.listBySession("sess_done")).toHaveLength(1); // untouched
122
- });
123
-
124
- it("supersedence fires across backfill iterations (B.4 in the backfill path)", async () => {
125
- // Two sessions, both assert framework= for the same subject — newer wins.
126
- store.insertSessionForTest(makeSession({
127
- id: "sess_early", body: "BODY-EARLY", startedAt: "2026-05-17T10:00:00Z",
128
- }));
129
- store.insertSessionForTest(makeSession({
130
- id: "sess_late", body: "BODY-LATE", startedAt: "2026-05-18T10:00:00Z",
131
- }));
132
- const classifier = new ScriptedClassifier(new Map([
133
- ["BODY-EARLY", classifyResult([
134
- { kind: "decision", subject: "x", predicate: "framework", value: "Fastify" },
135
- ])],
136
- ["BODY-LATE", classifyResult([
137
- { kind: "decision", subject: "x", predicate: "framework", value: "Hono" },
138
- ])],
139
- ]));
140
-
141
- await backfillFacts({ store, factStore, classifier, statePath });
142
-
143
- const current = await factStore.findCurrent("x", "framework");
144
- expect(current?.value).toBe("Hono");
145
- expect(current?.sourceSessionId).toBe("sess_late");
146
-
147
- const chains = await factStore.getHistory("x", "framework");
148
- expect(chains[0]?.history.map((f) => f.value)).toEqual(["Hono", "Fastify"]);
149
- });
150
-
151
- it("dry-run reports counts without writing", async () => {
152
- store.insertSessionForTest(makeSession({
153
- id: "sess_dry", body: "BODY-DRY", startedAt: "2026-05-17T10:00:00Z",
154
- }));
155
- const classifier = new ScriptedClassifier(new Map([
156
- ["BODY-DRY", classifyResult([
157
- { kind: "decision", subject: "x", predicate: "framework", value: "v" },
158
- ])],
159
- ]));
160
-
161
- const report = await backfillFacts({
162
- store, factStore, classifier, statePath, dryRun: true,
163
- });
164
-
165
- expect(report.processed).toBe(1);
166
- expect(report.factsWritten).toBe(1);
167
- expect(await factStore.listBySession("sess_dry")).toHaveLength(0); // not written
168
- expect(existsSync(statePath)).toBe(false); // dry-run never touches state
169
- });
170
-
171
- it("state file is written after the run and used to skip done ids on reprocess re-runs", async () => {
172
- store.insertSessionForTest(makeSession({
173
- id: "sess_a", body: "BODY-A", startedAt: "2026-05-17T10:00:00Z",
174
- }));
175
- store.insertSessionForTest(makeSession({
176
- id: "sess_b", body: "BODY-B", startedAt: "2026-05-17T11:00:00Z",
177
- }));
178
- const classifier = new ScriptedClassifier(new Map([
179
- ["BODY-A", classifyResult([
180
- { kind: "decision", subject: "x", predicate: "framework", value: "v" },
181
- ])],
182
- ["BODY-B", classifyResult([
183
- { kind: "decision", subject: "y", predicate: "framework", value: "v" },
184
- ])],
185
- ]));
186
-
187
- const r1 = await backfillFacts({ store, factStore, classifier, statePath });
188
- expect(r1.processed).toBe(2);
189
- expect(JSON.parse(readFileSync(statePath, "utf8")).done.sort()).toEqual([
190
- "sess_a", "sess_b",
191
- ]);
192
-
193
- // Without reprocess, the SQL eligibility filter excludes both — the
194
- // happy-path "resume" is implicit (rows already have facts).
195
- const r2 = await backfillFacts({
196
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
197
- });
198
- expect(r2.total).toBe(0);
199
-
200
- // With reprocess, eligibility drops the NOT-EXISTS clause; the state
201
- // file is what keeps a resumed run from re-classifying done ids. Under
202
- // the post-fix semantics, state-file ids are filtered out BEFORE the
203
- // work queue is built, so `total` is 0 (empty work queue) and
204
- // `skippedAlreadyDone` reports the pre-filter count.
205
- const r3 = await backfillFacts({
206
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
207
- reprocess: true,
208
- });
209
- expect(r3.total).toBe(0);
210
- expect(r3.skippedAlreadyDone).toBe(2);
211
- expect(r3.processed).toBe(0);
212
- });
213
-
214
- it("--from skips sessions with id <= cutoff", async () => {
215
- store.insertSessionForTest(makeSession({
216
- id: "sess_aaa", body: "BODY-A", startedAt: "2026-05-17T10:00:00Z",
217
- }));
218
- store.insertSessionForTest(makeSession({
219
- id: "sess_zzz", body: "BODY-Z", startedAt: "2026-05-17T11:00:00Z",
220
- }));
221
- const classifier = new ScriptedClassifier(new Map([
222
- ["BODY-Z", classifyResult([
223
- { kind: "decision", subject: "z", predicate: "framework", value: "v" },
224
- ])],
225
- ]));
226
- const report = await backfillFacts({
227
- store, factStore, classifier, statePath, from: "sess_aaa",
228
- });
229
- expect(report.total).toBe(1);
230
- expect(classifier.calls).toEqual(["BODY-Z"]);
231
- });
232
-
233
- it("limit caps the batch size", async () => {
234
- for (let i = 0; i < 5; i++) {
235
- store.insertSessionForTest(makeSession({
236
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
237
- }));
238
- }
239
- const map = new Map<string, ClassifyResult>();
240
- for (let i = 0; i < 5; i++) {
241
- map.set(`BODY-${i}`, classifyResult([
242
- { kind: "decision", subject: `s${i}`, predicate: "framework", value: "v" },
243
- ]));
244
- }
245
- const classifier = new ScriptedClassifier(map);
246
- const report = await backfillFacts({
247
- store, factStore, classifier, statePath, limit: 2,
248
- });
249
- expect(report.total).toBe(2);
250
- expect(report.processed).toBe(2);
251
- expect(classifier.calls).toHaveLength(2);
252
- });
253
-
254
- it("limit counts processable sessions, not raw SQL rows (filters state-file done BEFORE limit)", async () => {
255
- // 5 sessions in the corpus. 3 are already done in the state file (e.g.
256
- // previously hit low-confidence). With --limit 2, the OLD behavior would
257
- // slice the first 2 SQL rows (both done) and process 0; the NEW behavior
258
- // filters out the 3 done ids and then processes the next 2 untouched
259
- // sessions — actually doing 2 sessions worth of work as the operator
260
- // expects.
261
- for (let i = 0; i < 5; i++) {
262
- store.insertSessionForTest(makeSession({
263
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
264
- }));
265
- }
266
- // Pre-populate state file as if sess_0, sess_1, sess_2 already done
267
- // (e.g. via prior low-confidence runs).
268
- const fs = await import("node:fs");
269
- fs.writeFileSync(statePath, JSON.stringify({ done: ["sess_0", "sess_1", "sess_2"] }));
270
-
271
- const classifier = new ScriptedClassifier(new Map([
272
- ["BODY-3", classifyResult([
273
- { kind: "decision", subject: "s3", predicate: "framework", value: "v" },
274
- ])],
275
- ["BODY-4", classifyResult([
276
- { kind: "decision", subject: "s4", predicate: "framework", value: "v" },
277
- ])],
278
- ]));
279
- const report = await backfillFacts({
280
- store, factStore, classifier, statePath, limit: 2,
281
- });
282
- expect(report.skippedAlreadyDone).toBe(3); // pre-filter count
283
- expect(report.total).toBe(2); // work queue after pre-filter
284
- expect(report.processed).toBe(2); // both processed
285
- expect(classifier.calls.sort()).toEqual(["BODY-3", "BODY-4"]);
286
- });
287
-
288
- it("low-confidence sessions get marked done so a re-run doesn't re-classify them", async () => {
289
- store.insertSessionForTest(makeSession({
290
- id: "sess_low", body: "BODY-LOW", startedAt: "2026-05-17T10:00:00Z",
291
- }));
292
- const classifier = new ScriptedClassifier(new Map([
293
- ["BODY-LOW", classifyResult(
294
- [{ kind: "decision", subject: "x", predicate: "framework", value: "v" }],
295
- 0.2,
296
- )],
297
- ]));
298
- const r1 = await backfillFacts({ store, factStore, classifier, statePath });
299
- expect(r1.skippedLowConfidence).toBe(1);
300
- expect(r1.factsWritten).toBe(0);
301
-
302
- // Re-run uses the state file to skip rather than retrying.
303
- const r2 = await backfillFacts({
304
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
305
- });
306
- expect(r2.skippedAlreadyDone).toBe(1);
307
- });
308
-
309
- it("stops the whole run when classifier reports LLMUnreachable (don't burn API)", async () => {
310
- for (let i = 0; i < 3; i++) {
311
- store.insertSessionForTest(makeSession({
312
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
313
- }));
314
- }
315
- const classifier = new ScriptedClassifier(
316
- new Map(),
317
- new Set(["BODY-0"]), // first session immediately errors
318
- );
319
- const report = await backfillFacts({ store, factStore, classifier, statePath });
320
- expect(report.classifyFailures).toBe(1);
321
- expect(report.processed).toBe(0);
322
- // Should NOT call the classifier on the remaining 2 sessions.
323
- expect(classifier.calls).toEqual(["BODY-0"]);
324
- });
325
-
326
- it("excludes sessions started at or after the script's cutoff (race with live ingest)", async () => {
327
- // Insert a session with a startedAt in the future relative to NOW.
328
- const future = new Date(Date.now() + 60 * 60 * 1000).toISOString();
329
- store.insertSessionForTest(makeSession({
330
- id: "sess_future", body: "BODY-FUTURE", startedAt: future,
331
- }));
332
- const classifier = new ScriptedClassifier(new Map());
333
- const report = await backfillFacts({ store, factStore, classifier, statePath });
334
- expect(report.total).toBe(0);
335
- });
336
-
337
- it("reprocess=true re-classifies sessions that already have facts", async () => {
338
- store.insertSessionForTest(makeSession({
339
- id: "sess_repro", body: "BODY-REPRO", startedAt: "2026-05-17T10:00:00Z",
340
- }));
341
- await factStore.insert({
342
- id: "f_existing", kind: "decision", subject: "x", predicate: "framework",
343
- value: "old", sourceSessionId: "sess_repro", sourceQuote: null,
344
- createdAt: "2026-05-17T10:00:00Z", supersededBy: null, confidence: 0.9,
345
- });
346
-
347
- const classifier = new ScriptedClassifier(new Map([
348
- ["BODY-REPRO", classifyResult([
349
- { kind: "decision", subject: "x", predicate: "framework", value: "new" },
350
- ])],
351
- ]));
352
- const report = await backfillFacts({
353
- store, factStore, classifier, statePath, reprocess: true,
354
- });
355
- expect(report.processed).toBe(1);
356
-
357
- // The DELETE+insertManyInTxn pattern in applyFactsInTxn wipes the old
358
- // fact (same source_session_id) and writes the new one.
359
- const all = await factStore.listBySession("sess_repro");
360
- expect(all.map((f) => f.value)).toEqual(["new"]);
361
- });
362
- });
@@ -1,111 +0,0 @@
1
- /**
2
- * POST /api/citation/explicit endpoint integration. Verifies that the
3
- * cite_session MCP tool's daemon POST path writes to the citation log
4
- * with kind "tool_use" and source "mcp_tool".
5
- */
6
-
7
- import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
8
- import { tmpdir } from "node:os";
9
- import { join, resolve } from "node:path";
10
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
11
- import type { Hono } from "hono";
12
- import { RecallService } from "../../src/core/recall/recall-service.js";
13
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
14
- import { createApp } from "../../src/http/app.js";
15
- import type { EmbedResult, LLMClient } from "../../src/ports/llm-client.js";
16
-
17
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
18
-
19
- class FixedEmbedder implements LLMClient {
20
- async embed(): Promise<EmbedResult> {
21
- return { vector: new Float32Array(768), model: "fixed-test" };
22
- }
23
- async classify(): Promise<never> {
24
- throw new Error("not used");
25
- }
26
- }
27
-
28
- describe("POST /api/citation/explicit", () => {
29
- let tmp: string;
30
- let store: SqliteSessionStore;
31
- let app: Hono;
32
- let citationLogPath: string;
33
-
34
- beforeEach(() => {
35
- tmp = mkdtempSync(join(tmpdir(), "nlm-citation-explicit-"));
36
- store = new SqliteSessionStore({
37
- dbPath: join(tmp, "canonical.sqlite"),
38
- migrationsDir: MIGRATIONS_DIR,
39
- });
40
- const recall = new RecallService({ store, llm: new FixedEmbedder() });
41
- citationLogPath = join(tmp, "citation-log.jsonl");
42
- app = createApp({ recall, store, liveStore: store, citationLogPath });
43
- });
44
-
45
- afterEach(() => {
46
- store.close();
47
- rmSync(tmp, { recursive: true, force: true });
48
- });
49
-
50
- it("logs a citation entry and returns logged:true", async () => {
51
- const res = await app.request("/api/citation/explicit", {
52
- method: "POST",
53
- headers: { "Content-Type": "application/json" },
54
- body: JSON.stringify({
55
- id: "cc_sub_a139f4ab7ca5aa909",
56
- conversation_id: "conv_test_001",
57
- }),
58
- });
59
- expect(res.status).toBe(200);
60
- const json = (await res.json()) as Record<string, unknown>;
61
- expect(json["logged"]).toBe(true);
62
- expect(json["id"]).toBe("cc_sub_a139f4ab7ca5aa909");
63
- expect(json["source"]).toBe("mcp_tool");
64
- });
65
-
66
- it("writes to the citation log with kind tool_use", async () => {
67
- await app.request("/api/citation/explicit", {
68
- method: "POST",
69
- headers: { "Content-Type": "application/json" },
70
- body: JSON.stringify({
71
- id: "cc_sub_a139f4ab7ca5aa909",
72
- reason: "Confirmed FTS5 decision.",
73
- }),
74
- });
75
- expect(existsSync(citationLogPath)).toBe(true);
76
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
77
- const entry = JSON.parse(lines[0]!) as Record<string, unknown>;
78
- expect(entry["cited_id"]).toBe("cc_sub_a139f4ab7ca5aa909");
79
- expect(entry["kind"]).toBe("tool_use");
80
- expect(entry["response_preview"]).toBe("Confirmed FTS5 decision.");
81
- });
82
-
83
- it("returns 400 when id is missing", async () => {
84
- const res = await app.request("/api/citation/explicit", {
85
- method: "POST",
86
- headers: { "Content-Type": "application/json" },
87
- body: JSON.stringify({ conversation_id: "conv_001" }),
88
- });
89
- expect(res.status).toBe(400);
90
- });
91
-
92
- it("returns 400 when body is not JSON", async () => {
93
- const res = await app.request("/api/citation/explicit", {
94
- method: "POST",
95
- headers: { "Content-Type": "text/plain" },
96
- body: "not json",
97
- });
98
- expect(res.status).toBe(400);
99
- });
100
-
101
- it("defaults conversation_id to mcp_tool when absent", async () => {
102
- await app.request("/api/citation/explicit", {
103
- method: "POST",
104
- headers: { "Content-Type": "application/json" },
105
- body: JSON.stringify({ id: "cc_sub_a139f4ab7ca5aa909" }),
106
- });
107
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
108
- const entry = JSON.parse(lines[0]!) as Record<string, unknown>;
109
- expect(entry["conversation_id"]).toBe("mcp_tool");
110
- });
111
- });
@@ -1,169 +0,0 @@
1
- /**
2
- * /api/recall/cite-event endpoint integration. Exercises the citation log
3
- * append + readback via Hono app.request() against a real RecallService.
4
- */
5
-
6
- import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
7
- import { tmpdir } from "node:os";
8
- import { join, resolve } from "node:path";
9
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
10
- import type { Hono } from "hono";
11
- import { RecallService } from "../../src/core/recall/recall-service.js";
12
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
13
- import { createApp } from "../../src/http/app.js";
14
- import type { EmbedResult, LLMClient } from "../../src/ports/llm-client.js";
15
-
16
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
17
-
18
- class FixedEmbedder implements LLMClient {
19
- async embed(): Promise<EmbedResult> {
20
- return { vector: new Float32Array(768), model: "fixed-test" };
21
- }
22
- async classify(): Promise<never> {
23
- throw new Error("not used");
24
- }
25
- }
26
-
27
- describe("POST /api/recall/cite-event", () => {
28
- let tmp: string;
29
- let store: SqliteSessionStore;
30
- let app: Hono;
31
- let citationLogPath: string;
32
-
33
- beforeEach(() => {
34
- tmp = mkdtempSync(join(tmpdir(), "nlm-cite-"));
35
- store = new SqliteSessionStore({
36
- dbPath: join(tmp, "canonical.sqlite"),
37
- migrationsDir: MIGRATIONS_DIR,
38
- });
39
- const recall = new RecallService({ store, llm: new FixedEmbedder() });
40
- citationLogPath = join(tmp, "citation-log.jsonl");
41
- app = createApp({ recall, store, liveStore: store, citationLogPath });
42
- });
43
-
44
- afterEach(() => {
45
- store.close();
46
- rmSync(tmp, { recursive: true, force: true });
47
- });
48
-
49
- it("appends a citation entry and returns ok", async () => {
50
- const res = await app.request("/api/recall/cite-event", {
51
- method: "POST",
52
- headers: { "content-type": "application/json" },
53
- body: JSON.stringify({
54
- conversation_id: "conv-x",
55
- cited_id: "cc_sub_a139f4ab7ca5aa909",
56
- response_preview: "Per cc_sub_a139f4ab7ca5aa909 we chose FTS5.",
57
- }),
58
- });
59
- expect(res.status).toBe(200);
60
- const body = (await res.json()) as { ok: boolean };
61
- expect(body.ok).toBe(true);
62
-
63
- expect(existsSync(citationLogPath)).toBe(true);
64
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
65
- expect(lines).toHaveLength(1);
66
- const entry = JSON.parse(lines[0] ?? "{}") as Record<string, unknown>;
67
- expect(entry["conversation_id"]).toBe("conv-x");
68
- expect(entry["cited_id"]).toBe("cc_sub_a139f4ab7ca5aa909");
69
- expect(entry["response_preview"]).toBe(
70
- "Per cc_sub_a139f4ab7ca5aa909 we chose FTS5.",
71
- );
72
- expect(typeof entry["ts"]).toBe("string");
73
- });
74
-
75
- it("persists the kind field when provided (tool_use)", async () => {
76
- const res = await app.request("/api/recall/cite-event", {
77
- method: "POST",
78
- headers: { "content-type": "application/json" },
79
- body: JSON.stringify({
80
- conversation_id: "conv-mcp",
81
- cited_id: "cc_sub_a139f4ab7ca5aa909",
82
- kind: "tool_use",
83
- }),
84
- });
85
- expect(res.status).toBe(200);
86
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
87
- const entry = JSON.parse(lines[lines.length - 1] ?? "{}") as Record<string, unknown>;
88
- expect(entry["kind"]).toBe("tool_use");
89
- });
90
-
91
- it("rejects missing conversation_id", async () => {
92
- const res = await app.request("/api/recall/cite-event", {
93
- method: "POST",
94
- headers: { "content-type": "application/json" },
95
- body: JSON.stringify({ cited_id: "cc_sub_x" }),
96
- });
97
- expect(res.status).toBe(400);
98
- });
99
-
100
- it("rejects missing cited_id", async () => {
101
- const res = await app.request("/api/recall/cite-event", {
102
- method: "POST",
103
- headers: { "content-type": "application/json" },
104
- body: JSON.stringify({ conversation_id: "conv-x" }),
105
- });
106
- expect(res.status).toBe(400);
107
- });
108
-
109
- it("rejects non-JSON body", async () => {
110
- const res = await app.request("/api/recall/cite-event", {
111
- method: "POST",
112
- headers: { "content-type": "application/json" },
113
- body: "not json",
114
- });
115
- expect(res.status).toBe(400);
116
- });
117
-
118
- it("GET /api/recall/cite-stats aggregates appended citations", async () => {
119
- await app.request("/api/recall/cite-event", {
120
- method: "POST",
121
- headers: { "content-type": "application/json" },
122
- body: JSON.stringify({
123
- conversation_id: "c1",
124
- cited_id: "cc_sub_aaa111",
125
- }),
126
- });
127
- await app.request("/api/recall/cite-event", {
128
- method: "POST",
129
- headers: { "content-type": "application/json" },
130
- body: JSON.stringify({
131
- conversation_id: "c2",
132
- cited_id: "cc_sub_aaa111",
133
- }),
134
- });
135
- await app.request("/api/recall/cite-event", {
136
- method: "POST",
137
- headers: { "content-type": "application/json" },
138
- body: JSON.stringify({
139
- conversation_id: "c3",
140
- cited_id: "cc_sub_bbb222",
141
- }),
142
- });
143
-
144
- const res = await app.request("/api/recall/cite-stats?days=7");
145
- expect(res.status).toBe(200);
146
- const stats = (await res.json()) as {
147
- total: number;
148
- distinct_ids: number;
149
- top_ids: { id: string; count: number }[];
150
- log_present: boolean;
151
- };
152
- expect(stats.total).toBe(3);
153
- expect(stats.distinct_ids).toBe(2);
154
- expect(stats.log_present).toBe(true);
155
- expect(stats.top_ids[0]?.id).toBe("cc_sub_aaa111");
156
- expect(stats.top_ids[0]?.count).toBe(2);
157
- });
158
-
159
- it("GET /api/recall/cite-stats returns zero-totals when log is absent", async () => {
160
- const res = await app.request("/api/recall/cite-stats?days=14");
161
- expect(res.status).toBe(200);
162
- const stats = (await res.json()) as {
163
- total: number;
164
- log_present: boolean;
165
- };
166
- expect(stats.total).toBe(0);
167
- expect(stats.log_present).toBe(false);
168
- });
169
- });