nlm-memory 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/README.md +72 -34
  2. package/dist/cli/nlm.js +2 -1
  3. package/dist/cli/nlm.js.map +1 -1
  4. package/dist/http/app.js +2 -1
  5. package/dist/http/app.js.map +1 -1
  6. package/dist/mcp/server.js +20 -1
  7. package/dist/mcp/server.js.map +1 -1
  8. package/dist/ui/assets/{index-C8cpwbYJ.css → index-Beo8psd-.css} +1 -1
  9. package/dist/ui/assets/{index-CB50QnL-.js → index-CSPTTeeM.js} +8 -8
  10. package/dist/ui/index.html +2 -2
  11. package/package.json +26 -1
  12. package/.agents/plugins/marketplace.json +0 -20
  13. package/.github/workflows/ci.yml +0 -30
  14. package/docs/methodology/re-derivation-rate.md +0 -112
  15. package/docs/methodology/useful-hit-rate.md +0 -79
  16. package/docs/plans/2026-05-20-fts5-lexical-recall.md +0 -1088
  17. package/docs/plans/2026-05-20-recall-daemon-wedge-fix.md +0 -662
  18. package/docs/plans/2026-05-20-recall-hook-design.md +0 -131
  19. package/docs/plans/2026-05-20-recall-hook-implementation.md +0 -1222
  20. package/docs/plans/desktop-product.md +0 -69
  21. package/docs/plans/factstore-design.md +0 -236
  22. package/logs/CHANGELOG/CHANGELOG-2026.md +0 -1575
  23. package/logs/CHANGELOG/CHANGELOG.md +0 -209
  24. package/migrations/000_initial_schema.sql +0 -174
  25. package/migrations/001_entity_type_rename.sql +0 -17
  26. package/migrations/002_adapter_state_extend.sql +0 -12
  27. package/migrations/003_session_embeddings.sql +0 -11
  28. package/migrations/004_facts.sql +0 -46
  29. package/migrations/005_sources.sql +0 -31
  30. package/migrations/006_providers.sql +0 -33
  31. package/migrations/007_source_tokens.sql +0 -17
  32. package/migrations/008_fts_rebuild.sql +0 -9
  33. package/migrations/009_session_embedding_chunks.sql +0 -46
  34. package/migrations/010_sources_opencode.sql +0 -30
  35. package/migrations/011_sources_hermes_agent.sql +0 -30
  36. package/migrations/012_sources_aider.sql +0 -30
  37. package/migrations/013_adapter_state_failure_count.sql +0 -12
  38. package/migrations/014_sources_cursor.sql +0 -30
  39. package/migrations/015_sources_windsurf.sql +0 -30
  40. package/plugin-hermes-agent/README.md +0 -49
  41. package/plugin-hermes-agent/__init__.py +0 -75
  42. package/plugin-hermes-agent/plugin.yaml +0 -15
  43. package/scripts/backfill-citations.mjs +0 -0
  44. package/scripts/build-codex-plugin.mjs +0 -61
  45. package/scripts/deepseek-probe.mjs +0 -67
  46. package/scripts/extract-triples.mjs +0 -207
  47. package/scripts/longmemeval/embedding-cache.ts +0 -77
  48. package/scripts/longmemeval/fetch-dataset.sh +0 -25
  49. package/scripts/longmemeval/run-harness.ts +0 -315
  50. package/scripts/longmemeval/scorer.ts +0 -99
  51. package/scripts/longmemeval/tsconfig.json +0 -9
  52. package/scripts/longmemeval/types.ts +0 -35
  53. package/scripts/nlm-daily-digest.py +0 -239
  54. package/scripts/nlm-daily-digest.sh +0 -28
  55. package/src/cli/classify-parity.ts +0 -257
  56. package/src/cli/launchctl-helpers.ts +0 -49
  57. package/src/cli/nlm.ts +0 -1078
  58. package/src/core/actions/actions-log.ts +0 -118
  59. package/src/core/actions/overlay.ts +0 -117
  60. package/src/core/adapters/aider.ts +0 -205
  61. package/src/core/adapters/claude-code.ts +0 -293
  62. package/src/core/adapters/common.ts +0 -54
  63. package/src/core/adapters/cursor.ts +0 -486
  64. package/src/core/adapters/from-source.ts +0 -67
  65. package/src/core/adapters/hermes-agent.ts +0 -240
  66. package/src/core/adapters/hermes.ts +0 -277
  67. package/src/core/adapters/jsonl-generic.ts +0 -208
  68. package/src/core/adapters/opencode.ts +0 -281
  69. package/src/core/adapters/pi.ts +0 -264
  70. package/src/core/adapters/windsurf.ts +0 -386
  71. package/src/core/classifier/prompt.ts +0 -200
  72. package/src/core/dataset/build-dataset.ts +0 -463
  73. package/src/core/embedding/chunk-body.ts +0 -76
  74. package/src/core/embedding/embed-backfill.ts +0 -210
  75. package/src/core/embedding/embed-normalize.ts +0 -135
  76. package/src/core/facts/backfill-facts.ts +0 -254
  77. package/src/core/facts/extract-facts.ts +0 -50
  78. package/src/core/hook/citation-detect.ts +0 -124
  79. package/src/core/hook/cite-memo.ts +0 -68
  80. package/src/core/hook/claude-settings.ts +0 -187
  81. package/src/core/hook/gate.ts +0 -25
  82. package/src/core/hook/hook-log.ts +0 -41
  83. package/src/core/hook/memo-sweep.ts +0 -164
  84. package/src/core/hook/memo.ts +0 -67
  85. package/src/core/hook/pointer-block.ts +0 -26
  86. package/src/core/hook/select.ts +0 -32
  87. package/src/core/hook/transcript.ts +0 -121
  88. package/src/core/ingest/ingest-session.ts +0 -111
  89. package/src/core/providers/provider-models.ts +0 -100
  90. package/src/core/providers/provider-registry.ts +0 -196
  91. package/src/core/recall/citation-log.ts +0 -108
  92. package/src/core/recall/filter.ts +0 -27
  93. package/src/core/recall/index.ts +0 -6
  94. package/src/core/recall/match-fields.ts +0 -40
  95. package/src/core/recall/query-log.ts +0 -149
  96. package/src/core/recall/query-shape.ts +0 -66
  97. package/src/core/recall/recall-service.ts +0 -320
  98. package/src/core/recall/recent-log.ts +0 -59
  99. package/src/core/recall/tokenize.ts +0 -18
  100. package/src/core/recall/useful-scan.ts +0 -336
  101. package/src/core/recall-facts/fact-query-log.ts +0 -150
  102. package/src/core/recall-facts/fact-recall-service.ts +0 -327
  103. package/src/core/scheduler/scan-once.ts +0 -142
  104. package/src/core/scheduler/scheduler.ts +0 -225
  105. package/src/core/sources/source-registry.ts +0 -278
  106. package/src/core/storage/db-restore.ts +0 -133
  107. package/src/core/storage/live-status.ts +0 -45
  108. package/src/core/storage/migrate.ts +0 -72
  109. package/src/core/storage/sqlite-fact-store.ts +0 -304
  110. package/src/core/storage/sqlite-session-store.ts +0 -810
  111. package/src/hook/hook-auth.ts +0 -18
  112. package/src/hook/prompt-recall-hook.ts +0 -180
  113. package/src/hook/session-end-hook.ts +0 -81
  114. package/src/hook/session-start-hook.ts +0 -168
  115. package/src/hook/stop-hook.ts +0 -239
  116. package/src/http/app.ts +0 -1215
  117. package/src/install/claude-code.ts +0 -128
  118. package/src/install/codex.ts +0 -367
  119. package/src/install/cursor.ts +0 -68
  120. package/src/install/hermes-agent.ts +0 -76
  121. package/src/install/hermes.ts +0 -78
  122. package/src/install/nlm-dir-perms.ts +0 -55
  123. package/src/install/ollama.ts +0 -284
  124. package/src/install/setup.ts +0 -489
  125. package/src/install/windsurf.ts +0 -68
  126. package/src/llm/classifier-box.ts +0 -64
  127. package/src/llm/deepseek-client.ts +0 -150
  128. package/src/llm/env-autoload.ts +0 -55
  129. package/src/llm/ollama-client.ts +0 -189
  130. package/src/mcp/server.ts +0 -534
  131. package/src/ports/fact-store.ts +0 -102
  132. package/src/ports/llm-client.ts +0 -52
  133. package/src/ports/logger.ts +0 -16
  134. package/src/ports/session-store.ts +0 -45
  135. package/src/ports/transcript-adapter.ts +0 -55
  136. package/src/shared/types.ts +0 -149
  137. package/src/ui/App.tsx +0 -58
  138. package/src/ui/components/PromoteOpenButton.tsx +0 -65
  139. package/src/ui/components/SessionDrawer.tsx +0 -199
  140. package/src/ui/components/SideNav.tsx +0 -162
  141. package/src/ui/components/Skeleton.tsx +0 -107
  142. package/src/ui/index.html +0 -13
  143. package/src/ui/lib/actions.ts +0 -30
  144. package/src/ui/lib/api.ts +0 -92
  145. package/src/ui/lib/dataset.ts +0 -141
  146. package/src/ui/lib/registries.ts +0 -155
  147. package/src/ui/lib/view-settings.ts +0 -41
  148. package/src/ui/main.tsx +0 -15
  149. package/src/ui/pages/Live.tsx +0 -229
  150. package/src/ui/pages/Pulse.tsx +0 -415
  151. package/src/ui/pages/Recall.tsx +0 -190
  152. package/src/ui/pages/River.tsx +0 -354
  153. package/src/ui/pages/Search.tsx +0 -386
  154. package/src/ui/pages/Stub.tsx +0 -9
  155. package/src/ui/pages/Thread.tsx +0 -473
  156. package/src/ui/pages/settings/Classifier.tsx +0 -227
  157. package/src/ui/pages/settings/Data.tsx +0 -190
  158. package/src/ui/pages/settings/Index.tsx +0 -65
  159. package/src/ui/pages/settings/Labels.tsx +0 -224
  160. package/src/ui/pages/settings/Providers.tsx +0 -305
  161. package/src/ui/pages/settings/SettingsSubnav.tsx +0 -28
  162. package/src/ui/pages/settings/Sources.tsx +0 -326
  163. package/src/ui/pages/settings/Views.tsx +0 -96
  164. package/src/ui/styles.css +0 -1890
  165. package/src/ui/tsconfig.json +0 -21
  166. package/src/ui/vite.config.ts +0 -19
  167. package/tests/fixtures/claude_code/short_session.jsonl +0 -2
  168. package/tests/fixtures/claude_code/standard_iso.jsonl +0 -4
  169. package/tests/fixtures/claude_code/tool_heavy.jsonl +0 -8
  170. package/tests/fixtures/claude_code/with_subagent.jsonl +0 -7
  171. package/tests/fixtures/facts.ts +0 -17
  172. package/tests/fixtures/golden-corpus.ts +0 -85
  173. package/tests/fixtures/hermes/paired_request_dump.json +0 -24
  174. package/tests/fixtures/hermes/paired_session.json +0 -23
  175. package/tests/fixtures/hermes/request_dump.json +0 -28
  176. package/tests/fixtures/hermes/session_iso.json +0 -38
  177. package/tests/fixtures/hermes/session_unix.json +0 -38
  178. package/tests/fixtures/hermes/system_only.json +0 -18
  179. package/tests/fixtures/pi/error-connection-abort.jsonl +0 -8
  180. package/tests/fixtures/pi/short-successful.jsonl +0 -5
  181. package/tests/fixtures/pi/with-custom-message.jsonl +0 -6
  182. package/tests/fixtures/sessions.ts +0 -22
  183. package/tests/integration/backfill-facts.test.ts +0 -362
  184. package/tests/integration/citation-explicit.test.ts +0 -111
  185. package/tests/integration/cite-event.test.ts +0 -169
  186. package/tests/integration/cite-memo.test.ts +0 -87
  187. package/tests/integration/db-restore.test.ts +0 -153
  188. package/tests/integration/embed-backfill.test.ts +0 -176
  189. package/tests/integration/fact-supersedence.test.ts +0 -313
  190. package/tests/integration/fts-index.test.ts +0 -60
  191. package/tests/integration/getbyids-sqlite.test.ts +0 -100
  192. package/tests/integration/hermes-agent-hooks.test.ts +0 -248
  193. package/tests/integration/hook-claude-settings.test.ts +0 -218
  194. package/tests/integration/hook-log.test.ts +0 -54
  195. package/tests/integration/hook-memo.test.ts +0 -68
  196. package/tests/integration/hook-pre-compact.test.ts +0 -105
  197. package/tests/integration/hook-subagent-start.test.ts +0 -102
  198. package/tests/integration/http.test.ts +0 -401
  199. package/tests/integration/keyword-search-fts.test.ts +0 -66
  200. package/tests/integration/mcp-recall-logging.test.ts +0 -88
  201. package/tests/integration/mcp.test.ts +0 -260
  202. package/tests/integration/memo-sweep.test.ts +0 -91
  203. package/tests/integration/prompt-recall-hook.test.ts +0 -88
  204. package/tests/integration/provider-registry.test.ts +0 -107
  205. package/tests/integration/recall-golden.test.ts +0 -59
  206. package/tests/integration/recall-sqlite.test.ts +0 -169
  207. package/tests/integration/scheduler.test.ts +0 -391
  208. package/tests/integration/session-end-hook.test.ts +0 -48
  209. package/tests/integration/session-start-hook.test.ts +0 -126
  210. package/tests/integration/source-registry.test.ts +0 -122
  211. package/tests/integration/sqlite-fact-store.test.ts +0 -346
  212. package/tests/integration/stop-hook.test.ts +0 -560
  213. package/tests/integration/wal-checkpoint.test.ts +0 -49
  214. package/tests/unit/cli/launchctl-helpers.test.ts +0 -60
  215. package/tests/unit/core/adapters/aider.test.ts +0 -230
  216. package/tests/unit/core/adapters/claude-code.test.ts +0 -118
  217. package/tests/unit/core/adapters/cursor.test.ts +0 -485
  218. package/tests/unit/core/adapters/hermes-agent.test.ts +0 -329
  219. package/tests/unit/core/adapters/hermes.test.ts +0 -81
  220. package/tests/unit/core/adapters/jsonl-generic.test.ts +0 -142
  221. package/tests/unit/core/adapters/opencode.test.ts +0 -354
  222. package/tests/unit/core/adapters/pi.test.ts +0 -110
  223. package/tests/unit/core/adapters/windsurf.test.ts +0 -416
  224. package/tests/unit/core/classifier/prompt.test.ts +0 -126
  225. package/tests/unit/core/embedding/chunk-body.test.ts +0 -100
  226. package/tests/unit/core/facts/extract-facts.test.ts +0 -117
  227. package/tests/unit/core/filter.test.ts +0 -40
  228. package/tests/unit/core/hook/citation-detect-cite-session.test.ts +0 -96
  229. package/tests/unit/core/hook/citation-detect.test.ts +0 -124
  230. package/tests/unit/core/hook/gate.test.ts +0 -29
  231. package/tests/unit/core/hook/pointer-block.test.ts +0 -22
  232. package/tests/unit/core/hook/select.test.ts +0 -66
  233. package/tests/unit/core/match-fields.test.ts +0 -39
  234. package/tests/unit/core/mcp-cite-session.test.ts +0 -51
  235. package/tests/unit/core/providers/provider-models.test.ts +0 -101
  236. package/tests/unit/core/query-shape.test.ts +0 -92
  237. package/tests/unit/core/recall-facts/fact-recall-service.test.ts +0 -258
  238. package/tests/unit/core/recall-service.test.ts +0 -200
  239. package/tests/unit/core/storage/live-status.test.ts +0 -54
  240. package/tests/unit/core/tokenize.test.ts +0 -32
  241. package/tests/unit/core/useful-scan.test.ts +0 -537
  242. package/tests/unit/llm/embed.test.ts +0 -93
  243. package/tests/unit/llm/ollama-client.test.ts +0 -124
  244. package/tests/unit/scripts/longmemeval-scorer.test.ts +0 -114
  245. package/tsconfig.json +0 -31
  246. package/tsconfig.test.json +0 -11
  247. package/vitest.config.ts +0 -22
@@ -1,362 +0,0 @@
1
- /**
2
- * Phase B.5 — backfillFacts integration. Seeds a real SQLite store with
3
- * sessions that have no facts, runs the backfill module against a stub
4
- * classifier, and verifies facts land + supersedence fires + state file
5
- * resumes correctly.
6
- */
7
-
8
- import { mkdtempSync, rmSync, existsSync, readFileSync } from "node:fs";
9
- import { tmpdir } from "node:os";
10
- import { join, resolve } from "node:path";
11
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
12
- import { backfillFacts } from "../../src/core/facts/backfill-facts.js";
13
- import { SqliteFactStore } from "../../src/core/storage/sqlite-fact-store.js";
14
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
15
- import type {
16
- ClassifyResult,
17
- EmbedResult,
18
- ExtractedFact,
19
- LLMClient,
20
- } from "../../src/ports/llm-client.js";
21
- import { LLMUnreachableError } from "../../src/ports/llm-client.js";
22
- import { makeSession } from "../fixtures/sessions.js";
23
-
24
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
25
-
26
- class ScriptedClassifier implements LLMClient {
27
- calls: string[] = [];
28
- constructor(
29
- private readonly results: Map<string, ClassifyResult>,
30
- private readonly errorIds: Set<string> = new Set(),
31
- ) {}
32
- async embed(): Promise<EmbedResult> {
33
- throw new Error("not used");
34
- }
35
- async classify(transcript: string): Promise<ClassifyResult> {
36
- this.calls.push(transcript);
37
- if (this.errorIds.has(transcript)) {
38
- throw new LLMUnreachableError("test-stub");
39
- }
40
- const result = this.results.get(transcript);
41
- if (!result) throw new Error(`no scripted result for transcript: ${transcript.slice(0, 60)}`);
42
- return result;
43
- }
44
- }
45
-
46
- function classifyResult(
47
- facts: ExtractedFact[],
48
- confidence = 0.9,
49
- ): ClassifyResult {
50
- return {
51
- label: "L",
52
- summary: "S",
53
- entities: [],
54
- decisions: [],
55
- open: [],
56
- confidence,
57
- facts,
58
- };
59
- }
60
-
61
- describe("backfillFacts", () => {
62
- let tmp: string;
63
- let store: SqliteSessionStore;
64
- let factStore: SqliteFactStore;
65
- let statePath: string;
66
-
67
- beforeEach(() => {
68
- tmp = mkdtempSync(join(tmpdir(), "nlm-b5-"));
69
- store = new SqliteSessionStore({
70
- dbPath: join(tmp, "canonical.sqlite"),
71
- migrationsDir: MIGRATIONS_DIR,
72
- });
73
- factStore = new SqliteFactStore(store.rawDb());
74
- statePath = join(tmp, "backfill_facts.state");
75
- });
76
-
77
- afterEach(() => {
78
- store.close();
79
- rmSync(tmp, { recursive: true, force: true });
80
- });
81
-
82
- it("writes facts for sessions that have none, skips sessions with existing facts", async () => {
83
- // Two sessions need backfill; one already has a fact.
84
- store.insertSessionForTest(makeSession({
85
- id: "sess_old1", body: "BODY-OLD-1", startedAt: "2026-05-17T10:00:00Z",
86
- }));
87
- store.insertSessionForTest(makeSession({
88
- id: "sess_old2", body: "BODY-OLD-2", startedAt: "2026-05-17T11:00:00Z",
89
- }));
90
- store.insertSessionForTest(makeSession({
91
- id: "sess_done", body: "BODY-DONE", startedAt: "2026-05-17T12:00:00Z",
92
- }));
93
- // Pre-existing fact on sess_done — backfill should skip it.
94
- await factStore.insert({
95
- id: "f_pre", kind: "decision", subject: "x", predicate: "framework",
96
- value: "v", sourceSessionId: "sess_done", sourceQuote: null,
97
- createdAt: "2026-05-17T12:00:00Z", supersededBy: null, confidence: 0.9,
98
- });
99
-
100
- const classifier = new ScriptedClassifier(new Map([
101
- ["BODY-OLD-1", classifyResult([
102
- { kind: "decision", subject: "nlm-memory-ts", predicate: "framework", value: "Hono" },
103
- ])],
104
- ["BODY-OLD-2", classifyResult([
105
- { kind: "attribute", subject: "mac-pro", predicate: "endpoint", value: ":8080" },
106
- { kind: "attribute", subject: "mac-pro", predicate: "model", value: "qwen2.5-3b" },
107
- ])],
108
- ]));
109
-
110
- const report = await backfillFacts({
111
- store, factStore, classifier, statePath,
112
- });
113
-
114
- expect(report.total).toBe(2); // sess_done excluded by the NOT EXISTS clause
115
- expect(report.processed).toBe(2);
116
- expect(report.factsWritten).toBe(3);
117
- expect(classifier.calls).toHaveLength(2);
118
-
119
- expect(await factStore.listBySession("sess_old1")).toHaveLength(1);
120
- expect(await factStore.listBySession("sess_old2")).toHaveLength(2);
121
- expect(await factStore.listBySession("sess_done")).toHaveLength(1); // untouched
122
- });
123
-
124
- it("supersedence fires across backfill iterations (B.4 in the backfill path)", async () => {
125
- // Two sessions, both assert framework= for the same subject — newer wins.
126
- store.insertSessionForTest(makeSession({
127
- id: "sess_early", body: "BODY-EARLY", startedAt: "2026-05-17T10:00:00Z",
128
- }));
129
- store.insertSessionForTest(makeSession({
130
- id: "sess_late", body: "BODY-LATE", startedAt: "2026-05-18T10:00:00Z",
131
- }));
132
- const classifier = new ScriptedClassifier(new Map([
133
- ["BODY-EARLY", classifyResult([
134
- { kind: "decision", subject: "x", predicate: "framework", value: "Fastify" },
135
- ])],
136
- ["BODY-LATE", classifyResult([
137
- { kind: "decision", subject: "x", predicate: "framework", value: "Hono" },
138
- ])],
139
- ]));
140
-
141
- await backfillFacts({ store, factStore, classifier, statePath });
142
-
143
- const current = await factStore.findCurrent("x", "framework");
144
- expect(current?.value).toBe("Hono");
145
- expect(current?.sourceSessionId).toBe("sess_late");
146
-
147
- const chains = await factStore.getHistory("x", "framework");
148
- expect(chains[0]?.history.map((f) => f.value)).toEqual(["Hono", "Fastify"]);
149
- });
150
-
151
- it("dry-run reports counts without writing", async () => {
152
- store.insertSessionForTest(makeSession({
153
- id: "sess_dry", body: "BODY-DRY", startedAt: "2026-05-17T10:00:00Z",
154
- }));
155
- const classifier = new ScriptedClassifier(new Map([
156
- ["BODY-DRY", classifyResult([
157
- { kind: "decision", subject: "x", predicate: "framework", value: "v" },
158
- ])],
159
- ]));
160
-
161
- const report = await backfillFacts({
162
- store, factStore, classifier, statePath, dryRun: true,
163
- });
164
-
165
- expect(report.processed).toBe(1);
166
- expect(report.factsWritten).toBe(1);
167
- expect(await factStore.listBySession("sess_dry")).toHaveLength(0); // not written
168
- expect(existsSync(statePath)).toBe(false); // dry-run never touches state
169
- });
170
-
171
- it("state file is written after the run and used to skip done ids on reprocess re-runs", async () => {
172
- store.insertSessionForTest(makeSession({
173
- id: "sess_a", body: "BODY-A", startedAt: "2026-05-17T10:00:00Z",
174
- }));
175
- store.insertSessionForTest(makeSession({
176
- id: "sess_b", body: "BODY-B", startedAt: "2026-05-17T11:00:00Z",
177
- }));
178
- const classifier = new ScriptedClassifier(new Map([
179
- ["BODY-A", classifyResult([
180
- { kind: "decision", subject: "x", predicate: "framework", value: "v" },
181
- ])],
182
- ["BODY-B", classifyResult([
183
- { kind: "decision", subject: "y", predicate: "framework", value: "v" },
184
- ])],
185
- ]));
186
-
187
- const r1 = await backfillFacts({ store, factStore, classifier, statePath });
188
- expect(r1.processed).toBe(2);
189
- expect(JSON.parse(readFileSync(statePath, "utf8")).done.sort()).toEqual([
190
- "sess_a", "sess_b",
191
- ]);
192
-
193
- // Without reprocess, the SQL eligibility filter excludes both — the
194
- // happy-path "resume" is implicit (rows already have facts).
195
- const r2 = await backfillFacts({
196
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
197
- });
198
- expect(r2.total).toBe(0);
199
-
200
- // With reprocess, eligibility drops the NOT-EXISTS clause; the state
201
- // file is what keeps a resumed run from re-classifying done ids. Under
202
- // the post-fix semantics, state-file ids are filtered out BEFORE the
203
- // work queue is built, so `total` is 0 (empty work queue) and
204
- // `skippedAlreadyDone` reports the pre-filter count.
205
- const r3 = await backfillFacts({
206
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
207
- reprocess: true,
208
- });
209
- expect(r3.total).toBe(0);
210
- expect(r3.skippedAlreadyDone).toBe(2);
211
- expect(r3.processed).toBe(0);
212
- });
213
-
214
- it("--from skips sessions with id <= cutoff", async () => {
215
- store.insertSessionForTest(makeSession({
216
- id: "sess_aaa", body: "BODY-A", startedAt: "2026-05-17T10:00:00Z",
217
- }));
218
- store.insertSessionForTest(makeSession({
219
- id: "sess_zzz", body: "BODY-Z", startedAt: "2026-05-17T11:00:00Z",
220
- }));
221
- const classifier = new ScriptedClassifier(new Map([
222
- ["BODY-Z", classifyResult([
223
- { kind: "decision", subject: "z", predicate: "framework", value: "v" },
224
- ])],
225
- ]));
226
- const report = await backfillFacts({
227
- store, factStore, classifier, statePath, from: "sess_aaa",
228
- });
229
- expect(report.total).toBe(1);
230
- expect(classifier.calls).toEqual(["BODY-Z"]);
231
- });
232
-
233
- it("limit caps the batch size", async () => {
234
- for (let i = 0; i < 5; i++) {
235
- store.insertSessionForTest(makeSession({
236
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
237
- }));
238
- }
239
- const map = new Map<string, ClassifyResult>();
240
- for (let i = 0; i < 5; i++) {
241
- map.set(`BODY-${i}`, classifyResult([
242
- { kind: "decision", subject: `s${i}`, predicate: "framework", value: "v" },
243
- ]));
244
- }
245
- const classifier = new ScriptedClassifier(map);
246
- const report = await backfillFacts({
247
- store, factStore, classifier, statePath, limit: 2,
248
- });
249
- expect(report.total).toBe(2);
250
- expect(report.processed).toBe(2);
251
- expect(classifier.calls).toHaveLength(2);
252
- });
253
-
254
- it("limit counts processable sessions, not raw SQL rows (filters state-file done BEFORE limit)", async () => {
255
- // 5 sessions in the corpus. 3 are already done in the state file (e.g.
256
- // previously hit low-confidence). With --limit 2, the OLD behavior would
257
- // slice the first 2 SQL rows (both done) and process 0; the NEW behavior
258
- // filters out the 3 done ids and then processes the next 2 untouched
259
- // sessions — actually doing 2 sessions worth of work as the operator
260
- // expects.
261
- for (let i = 0; i < 5; i++) {
262
- store.insertSessionForTest(makeSession({
263
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
264
- }));
265
- }
266
- // Pre-populate state file as if sess_0, sess_1, sess_2 already done
267
- // (e.g. via prior low-confidence runs).
268
- const fs = await import("node:fs");
269
- fs.writeFileSync(statePath, JSON.stringify({ done: ["sess_0", "sess_1", "sess_2"] }));
270
-
271
- const classifier = new ScriptedClassifier(new Map([
272
- ["BODY-3", classifyResult([
273
- { kind: "decision", subject: "s3", predicate: "framework", value: "v" },
274
- ])],
275
- ["BODY-4", classifyResult([
276
- { kind: "decision", subject: "s4", predicate: "framework", value: "v" },
277
- ])],
278
- ]));
279
- const report = await backfillFacts({
280
- store, factStore, classifier, statePath, limit: 2,
281
- });
282
- expect(report.skippedAlreadyDone).toBe(3); // pre-filter count
283
- expect(report.total).toBe(2); // work queue after pre-filter
284
- expect(report.processed).toBe(2); // both processed
285
- expect(classifier.calls.sort()).toEqual(["BODY-3", "BODY-4"]);
286
- });
287
-
288
- it("low-confidence sessions get marked done so a re-run doesn't re-classify them", async () => {
289
- store.insertSessionForTest(makeSession({
290
- id: "sess_low", body: "BODY-LOW", startedAt: "2026-05-17T10:00:00Z",
291
- }));
292
- const classifier = new ScriptedClassifier(new Map([
293
- ["BODY-LOW", classifyResult(
294
- [{ kind: "decision", subject: "x", predicate: "framework", value: "v" }],
295
- 0.2,
296
- )],
297
- ]));
298
- const r1 = await backfillFacts({ store, factStore, classifier, statePath });
299
- expect(r1.skippedLowConfidence).toBe(1);
300
- expect(r1.factsWritten).toBe(0);
301
-
302
- // Re-run uses the state file to skip rather than retrying.
303
- const r2 = await backfillFacts({
304
- store, factStore, classifier: new ScriptedClassifier(new Map()), statePath,
305
- });
306
- expect(r2.skippedAlreadyDone).toBe(1);
307
- });
308
-
309
- it("stops the whole run when classifier reports LLMUnreachable (don't burn API)", async () => {
310
- for (let i = 0; i < 3; i++) {
311
- store.insertSessionForTest(makeSession({
312
- id: `sess_${i}`, body: `BODY-${i}`, startedAt: `2026-05-17T10:0${i}:00Z`,
313
- }));
314
- }
315
- const classifier = new ScriptedClassifier(
316
- new Map(),
317
- new Set(["BODY-0"]), // first session immediately errors
318
- );
319
- const report = await backfillFacts({ store, factStore, classifier, statePath });
320
- expect(report.classifyFailures).toBe(1);
321
- expect(report.processed).toBe(0);
322
- // Should NOT call the classifier on the remaining 2 sessions.
323
- expect(classifier.calls).toEqual(["BODY-0"]);
324
- });
325
-
326
- it("excludes sessions started at or after the script's cutoff (race with live ingest)", async () => {
327
- // Insert a session with a startedAt in the future relative to NOW.
328
- const future = new Date(Date.now() + 60 * 60 * 1000).toISOString();
329
- store.insertSessionForTest(makeSession({
330
- id: "sess_future", body: "BODY-FUTURE", startedAt: future,
331
- }));
332
- const classifier = new ScriptedClassifier(new Map());
333
- const report = await backfillFacts({ store, factStore, classifier, statePath });
334
- expect(report.total).toBe(0);
335
- });
336
-
337
- it("reprocess=true re-classifies sessions that already have facts", async () => {
338
- store.insertSessionForTest(makeSession({
339
- id: "sess_repro", body: "BODY-REPRO", startedAt: "2026-05-17T10:00:00Z",
340
- }));
341
- await factStore.insert({
342
- id: "f_existing", kind: "decision", subject: "x", predicate: "framework",
343
- value: "old", sourceSessionId: "sess_repro", sourceQuote: null,
344
- createdAt: "2026-05-17T10:00:00Z", supersededBy: null, confidence: 0.9,
345
- });
346
-
347
- const classifier = new ScriptedClassifier(new Map([
348
- ["BODY-REPRO", classifyResult([
349
- { kind: "decision", subject: "x", predicate: "framework", value: "new" },
350
- ])],
351
- ]));
352
- const report = await backfillFacts({
353
- store, factStore, classifier, statePath, reprocess: true,
354
- });
355
- expect(report.processed).toBe(1);
356
-
357
- // The DELETE+insertManyInTxn pattern in applyFactsInTxn wipes the old
358
- // fact (same source_session_id) and writes the new one.
359
- const all = await factStore.listBySession("sess_repro");
360
- expect(all.map((f) => f.value)).toEqual(["new"]);
361
- });
362
- });
@@ -1,111 +0,0 @@
1
- /**
2
- * POST /api/citation/explicit endpoint integration. Verifies that the
3
- * cite_session MCP tool's daemon POST path writes to the citation log
4
- * with kind "tool_use" and source "mcp_tool".
5
- */
6
-
7
- import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
8
- import { tmpdir } from "node:os";
9
- import { join, resolve } from "node:path";
10
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
11
- import type { Hono } from "hono";
12
- import { RecallService } from "../../src/core/recall/recall-service.js";
13
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
14
- import { createApp } from "../../src/http/app.js";
15
- import type { EmbedResult, LLMClient } from "../../src/ports/llm-client.js";
16
-
17
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
18
-
19
- class FixedEmbedder implements LLMClient {
20
- async embed(): Promise<EmbedResult> {
21
- return { vector: new Float32Array(768), model: "fixed-test" };
22
- }
23
- async classify(): Promise<never> {
24
- throw new Error("not used");
25
- }
26
- }
27
-
28
- describe("POST /api/citation/explicit", () => {
29
- let tmp: string;
30
- let store: SqliteSessionStore;
31
- let app: Hono;
32
- let citationLogPath: string;
33
-
34
- beforeEach(() => {
35
- tmp = mkdtempSync(join(tmpdir(), "nlm-citation-explicit-"));
36
- store = new SqliteSessionStore({
37
- dbPath: join(tmp, "canonical.sqlite"),
38
- migrationsDir: MIGRATIONS_DIR,
39
- });
40
- const recall = new RecallService({ store, llm: new FixedEmbedder() });
41
- citationLogPath = join(tmp, "citation-log.jsonl");
42
- app = createApp({ recall, store, liveStore: store, citationLogPath });
43
- });
44
-
45
- afterEach(() => {
46
- store.close();
47
- rmSync(tmp, { recursive: true, force: true });
48
- });
49
-
50
- it("logs a citation entry and returns logged:true", async () => {
51
- const res = await app.request("/api/citation/explicit", {
52
- method: "POST",
53
- headers: { "Content-Type": "application/json" },
54
- body: JSON.stringify({
55
- id: "cc_sub_a139f4ab7ca5aa909",
56
- conversation_id: "conv_test_001",
57
- }),
58
- });
59
- expect(res.status).toBe(200);
60
- const json = (await res.json()) as Record<string, unknown>;
61
- expect(json["logged"]).toBe(true);
62
- expect(json["id"]).toBe("cc_sub_a139f4ab7ca5aa909");
63
- expect(json["source"]).toBe("mcp_tool");
64
- });
65
-
66
- it("writes to the citation log with kind tool_use", async () => {
67
- await app.request("/api/citation/explicit", {
68
- method: "POST",
69
- headers: { "Content-Type": "application/json" },
70
- body: JSON.stringify({
71
- id: "cc_sub_a139f4ab7ca5aa909",
72
- reason: "Confirmed FTS5 decision.",
73
- }),
74
- });
75
- expect(existsSync(citationLogPath)).toBe(true);
76
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
77
- const entry = JSON.parse(lines[0]!) as Record<string, unknown>;
78
- expect(entry["cited_id"]).toBe("cc_sub_a139f4ab7ca5aa909");
79
- expect(entry["kind"]).toBe("tool_use");
80
- expect(entry["response_preview"]).toBe("Confirmed FTS5 decision.");
81
- });
82
-
83
- it("returns 400 when id is missing", async () => {
84
- const res = await app.request("/api/citation/explicit", {
85
- method: "POST",
86
- headers: { "Content-Type": "application/json" },
87
- body: JSON.stringify({ conversation_id: "conv_001" }),
88
- });
89
- expect(res.status).toBe(400);
90
- });
91
-
92
- it("returns 400 when body is not JSON", async () => {
93
- const res = await app.request("/api/citation/explicit", {
94
- method: "POST",
95
- headers: { "Content-Type": "text/plain" },
96
- body: "not json",
97
- });
98
- expect(res.status).toBe(400);
99
- });
100
-
101
- it("defaults conversation_id to mcp_tool when absent", async () => {
102
- await app.request("/api/citation/explicit", {
103
- method: "POST",
104
- headers: { "Content-Type": "application/json" },
105
- body: JSON.stringify({ id: "cc_sub_a139f4ab7ca5aa909" }),
106
- });
107
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
108
- const entry = JSON.parse(lines[0]!) as Record<string, unknown>;
109
- expect(entry["conversation_id"]).toBe("mcp_tool");
110
- });
111
- });
@@ -1,169 +0,0 @@
1
- /**
2
- * /api/recall/cite-event endpoint integration. Exercises the citation log
3
- * append + readback via Hono app.request() against a real RecallService.
4
- */
5
-
6
- import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
7
- import { tmpdir } from "node:os";
8
- import { join, resolve } from "node:path";
9
- import { afterEach, beforeEach, describe, expect, it } from "vitest";
10
- import type { Hono } from "hono";
11
- import { RecallService } from "../../src/core/recall/recall-service.js";
12
- import { SqliteSessionStore } from "../../src/core/storage/sqlite-session-store.js";
13
- import { createApp } from "../../src/http/app.js";
14
- import type { EmbedResult, LLMClient } from "../../src/ports/llm-client.js";
15
-
16
- const MIGRATIONS_DIR = resolve(__dirname, "../../migrations");
17
-
18
- class FixedEmbedder implements LLMClient {
19
- async embed(): Promise<EmbedResult> {
20
- return { vector: new Float32Array(768), model: "fixed-test" };
21
- }
22
- async classify(): Promise<never> {
23
- throw new Error("not used");
24
- }
25
- }
26
-
27
- describe("POST /api/recall/cite-event", () => {
28
- let tmp: string;
29
- let store: SqliteSessionStore;
30
- let app: Hono;
31
- let citationLogPath: string;
32
-
33
- beforeEach(() => {
34
- tmp = mkdtempSync(join(tmpdir(), "nlm-cite-"));
35
- store = new SqliteSessionStore({
36
- dbPath: join(tmp, "canonical.sqlite"),
37
- migrationsDir: MIGRATIONS_DIR,
38
- });
39
- const recall = new RecallService({ store, llm: new FixedEmbedder() });
40
- citationLogPath = join(tmp, "citation-log.jsonl");
41
- app = createApp({ recall, store, liveStore: store, citationLogPath });
42
- });
43
-
44
- afterEach(() => {
45
- store.close();
46
- rmSync(tmp, { recursive: true, force: true });
47
- });
48
-
49
- it("appends a citation entry and returns ok", async () => {
50
- const res = await app.request("/api/recall/cite-event", {
51
- method: "POST",
52
- headers: { "content-type": "application/json" },
53
- body: JSON.stringify({
54
- conversation_id: "conv-x",
55
- cited_id: "cc_sub_a139f4ab7ca5aa909",
56
- response_preview: "Per cc_sub_a139f4ab7ca5aa909 we chose FTS5.",
57
- }),
58
- });
59
- expect(res.status).toBe(200);
60
- const body = (await res.json()) as { ok: boolean };
61
- expect(body.ok).toBe(true);
62
-
63
- expect(existsSync(citationLogPath)).toBe(true);
64
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
65
- expect(lines).toHaveLength(1);
66
- const entry = JSON.parse(lines[0] ?? "{}") as Record<string, unknown>;
67
- expect(entry["conversation_id"]).toBe("conv-x");
68
- expect(entry["cited_id"]).toBe("cc_sub_a139f4ab7ca5aa909");
69
- expect(entry["response_preview"]).toBe(
70
- "Per cc_sub_a139f4ab7ca5aa909 we chose FTS5.",
71
- );
72
- expect(typeof entry["ts"]).toBe("string");
73
- });
74
-
75
- it("persists the kind field when provided (tool_use)", async () => {
76
- const res = await app.request("/api/recall/cite-event", {
77
- method: "POST",
78
- headers: { "content-type": "application/json" },
79
- body: JSON.stringify({
80
- conversation_id: "conv-mcp",
81
- cited_id: "cc_sub_a139f4ab7ca5aa909",
82
- kind: "tool_use",
83
- }),
84
- });
85
- expect(res.status).toBe(200);
86
- const lines = readFileSync(citationLogPath, "utf8").trim().split("\n");
87
- const entry = JSON.parse(lines[lines.length - 1] ?? "{}") as Record<string, unknown>;
88
- expect(entry["kind"]).toBe("tool_use");
89
- });
90
-
91
- it("rejects missing conversation_id", async () => {
92
- const res = await app.request("/api/recall/cite-event", {
93
- method: "POST",
94
- headers: { "content-type": "application/json" },
95
- body: JSON.stringify({ cited_id: "cc_sub_x" }),
96
- });
97
- expect(res.status).toBe(400);
98
- });
99
-
100
- it("rejects missing cited_id", async () => {
101
- const res = await app.request("/api/recall/cite-event", {
102
- method: "POST",
103
- headers: { "content-type": "application/json" },
104
- body: JSON.stringify({ conversation_id: "conv-x" }),
105
- });
106
- expect(res.status).toBe(400);
107
- });
108
-
109
- it("rejects non-JSON body", async () => {
110
- const res = await app.request("/api/recall/cite-event", {
111
- method: "POST",
112
- headers: { "content-type": "application/json" },
113
- body: "not json",
114
- });
115
- expect(res.status).toBe(400);
116
- });
117
-
118
- it("GET /api/recall/cite-stats aggregates appended citations", async () => {
119
- await app.request("/api/recall/cite-event", {
120
- method: "POST",
121
- headers: { "content-type": "application/json" },
122
- body: JSON.stringify({
123
- conversation_id: "c1",
124
- cited_id: "cc_sub_aaa111",
125
- }),
126
- });
127
- await app.request("/api/recall/cite-event", {
128
- method: "POST",
129
- headers: { "content-type": "application/json" },
130
- body: JSON.stringify({
131
- conversation_id: "c2",
132
- cited_id: "cc_sub_aaa111",
133
- }),
134
- });
135
- await app.request("/api/recall/cite-event", {
136
- method: "POST",
137
- headers: { "content-type": "application/json" },
138
- body: JSON.stringify({
139
- conversation_id: "c3",
140
- cited_id: "cc_sub_bbb222",
141
- }),
142
- });
143
-
144
- const res = await app.request("/api/recall/cite-stats?days=7");
145
- expect(res.status).toBe(200);
146
- const stats = (await res.json()) as {
147
- total: number;
148
- distinct_ids: number;
149
- top_ids: { id: string; count: number }[];
150
- log_present: boolean;
151
- };
152
- expect(stats.total).toBe(3);
153
- expect(stats.distinct_ids).toBe(2);
154
- expect(stats.log_present).toBe(true);
155
- expect(stats.top_ids[0]?.id).toBe("cc_sub_aaa111");
156
- expect(stats.top_ids[0]?.count).toBe(2);
157
- });
158
-
159
- it("GET /api/recall/cite-stats returns zero-totals when log is absent", async () => {
160
- const res = await app.request("/api/recall/cite-stats?days=14");
161
- expect(res.status).toBe(200);
162
- const stats = (await res.json()) as {
163
- total: number;
164
- log_present: boolean;
165
- };
166
- expect(stats.total).toBe(0);
167
- expect(stats.log_present).toBe(false);
168
- });
169
- });