claude-memory-layer 1.0.31 → 1.0.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (343) hide show
  1. package/README.md +9 -2
  2. package/dist/cli/index.js +1110 -72
  3. package/dist/cli/index.js.map +4 -4
  4. package/dist/core/index.js +414 -25
  5. package/dist/core/index.js.map +2 -2
  6. package/dist/hooks/post-tool-use.js +416 -27
  7. package/dist/hooks/post-tool-use.js.map +2 -2
  8. package/dist/hooks/semantic-daemon.js +416 -27
  9. package/dist/hooks/semantic-daemon.js.map +2 -2
  10. package/dist/hooks/session-end.js +416 -27
  11. package/dist/hooks/session-end.js.map +2 -2
  12. package/dist/hooks/session-start.js +416 -27
  13. package/dist/hooks/session-start.js.map +2 -2
  14. package/dist/hooks/stop.js +416 -27
  15. package/dist/hooks/stop.js.map +2 -2
  16. package/dist/hooks/user-prompt-submit.js +504 -34
  17. package/dist/hooks/user-prompt-submit.js.map +2 -2
  18. package/dist/index.js +416 -27
  19. package/dist/index.js.map +2 -2
  20. package/dist/mcp/index.js +407 -32
  21. package/dist/mcp/index.js.map +2 -2
  22. package/dist/server/api/index.js +850 -44
  23. package/dist/server/api/index.js.map +3 -3
  24. package/dist/server/index.js +1073 -64
  25. package/dist/server/index.js.map +3 -3
  26. package/dist/services/memory-service.js +416 -27
  27. package/dist/services/memory-service.js.map +2 -2
  28. package/dist/ui/assets/js/bootstrap.js +2 -0
  29. package/dist/ui/assets/js/overview.js +166 -3
  30. package/dist/ui/assets/js/state.js +3 -0
  31. package/dist/ui/index.html +20 -0
  32. package/dist/ui/style.css +193 -0
  33. package/package.json +15 -2
  34. package/scripts/postinstall-embedding-backend.cjs +16 -12
  35. package/AGENTS.md +0 -71
  36. package/CLAUDE.md +0 -30
  37. package/HANDOFF.md +0 -92
  38. package/Memo.txt +0 -558
  39. package/benchmarks/replay/anonymized-real-sessions.json +0 -48
  40. package/config/kpi-thresholds.json +0 -7
  41. package/context.md +0 -636
  42. package/docs/ARCHITECTURE_COMPARISON_AND_RECOMMENDATIONS.md +0 -627
  43. package/docs/HERMES_MEMORY_INGESTION_ANALYSIS.md +0 -440
  44. package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +0 -271
  45. package/docs/MEMORY_USEFULNESS_AUDIT.md +0 -371
  46. package/docs/MEMORY_USEFULNESS_AUDIT_RAW.json +0 -80
  47. package/docs/MEMSEARCH_PROJECT_STRUCTURE_ANALYSIS.md +0 -333
  48. package/docs/MEMU_ADOPTION.md +0 -40
  49. package/docs/OPERATIONS.md +0 -18
  50. package/docs/PRODUCT_VALIDATION_MATRIX.md +0 -82
  51. package/docs/PROJECT_STRUCTURE_ANALYSIS.md +0 -421
  52. package/docs/REFACTORING_MILESTONES_AND_ISSUES.md +0 -501
  53. package/docs/REFACTORING_PLAN_THIN_CORE.md +0 -414
  54. package/docs/REFERENCE_PROJECT_ANALYSES.md +0 -25
  55. package/docs/SUPERLOCALMEMORY_PROJECT_STRUCTURE_ANALYSIS.md +0 -452
  56. package/docs/TARGET_ARCHITECTURE_AND_FOLDER_STRUCTURE.md +0 -446
  57. package/docs/architecture/comparison-index.md +0 -47
  58. package/docs/reports/codex-real-data-validation-20260505T040447Z.md +0 -46
  59. package/plan.md +0 -1642
  60. package/scripts/build.ts +0 -159
  61. package/scripts/bump-patch-version.sh +0 -18
  62. package/scripts/delete-unknown-projects.js +0 -154
  63. package/scripts/fix-sync-gap.js +0 -32
  64. package/scripts/generate-session-qrels.ts +0 -126
  65. package/scripts/heartbeat-memory-orchestrator.sh +0 -28
  66. package/scripts/replay-retrieval-benchmark.ts +0 -69
  67. package/scripts/report-sync-gap.js +0 -26
  68. package/scripts/review-queue-auto-resolve.js +0 -21
  69. package/scripts/sync-gap-auto-heal.sh +0 -17
  70. package/spec.md +0 -624
  71. package/specs/20260207-dashboard-upgrade/context.md +0 -38
  72. package/specs/20260207-dashboard-upgrade/spec.md +0 -96
  73. package/specs/citations-system/context.md +0 -243
  74. package/specs/citations-system/plan.md +0 -495
  75. package/specs/citations-system/spec.md +0 -371
  76. package/specs/endless-mode/context.md +0 -305
  77. package/specs/endless-mode/plan.md +0 -620
  78. package/specs/endless-mode/spec.md +0 -455
  79. package/specs/entity-edge-model/context.md +0 -401
  80. package/specs/entity-edge-model/plan.md +0 -459
  81. package/specs/entity-edge-model/spec.md +0 -391
  82. package/specs/evidence-aligner-v2/context.md +0 -401
  83. package/specs/evidence-aligner-v2/plan.md +0 -303
  84. package/specs/evidence-aligner-v2/spec.md +0 -312
  85. package/specs/mcp-desktop-integration/context.md +0 -278
  86. package/specs/mcp-desktop-integration/plan.md +0 -550
  87. package/specs/mcp-desktop-integration/spec.md +0 -494
  88. package/specs/memory-utilization-improvements/context.md +0 -145
  89. package/specs/memory-utilization-improvements/plan.md +0 -361
  90. package/specs/memory-utilization-improvements/spec.md +0 -361
  91. package/specs/post-tool-use-hook/context.md +0 -319
  92. package/specs/post-tool-use-hook/plan.md +0 -469
  93. package/specs/post-tool-use-hook/spec.md +0 -364
  94. package/specs/private-tags/context.md +0 -288
  95. package/specs/private-tags/plan.md +0 -412
  96. package/specs/private-tags/spec.md +0 -345
  97. package/specs/progressive-disclosure/context.md +0 -346
  98. package/specs/progressive-disclosure/plan.md +0 -663
  99. package/specs/progressive-disclosure/spec.md +0 -415
  100. package/specs/selective-tool-observation/context.md +0 -100
  101. package/specs/selective-tool-observation/plan.md +0 -158
  102. package/specs/selective-tool-observation/spec.md +0 -127
  103. package/specs/task-entity-system/context.md +0 -297
  104. package/specs/task-entity-system/plan.md +0 -301
  105. package/specs/task-entity-system/spec.md +0 -314
  106. package/specs/thin-core-refactor/context.md +0 -275
  107. package/specs/thin-core-refactor/plan.md +0 -536
  108. package/specs/thin-core-refactor/spec.md +0 -465
  109. package/specs/vector-outbox-v2/context.md +0 -470
  110. package/specs/vector-outbox-v2/plan.md +0 -562
  111. package/specs/vector-outbox-v2/spec.md +0 -466
  112. package/specs/web-viewer-ui/context.md +0 -384
  113. package/specs/web-viewer-ui/plan.md +0 -797
  114. package/specs/web-viewer-ui/spec.md +0 -516
  115. package/src/adapters/claude/capture/index.ts +0 -3
  116. package/src/adapters/claude/context/index.ts +0 -3
  117. package/src/adapters/claude/hooks/index.ts +0 -21
  118. package/src/adapters/claude/hooks/post-tool-use.ts +0 -239
  119. package/src/adapters/claude/hooks/prompt-injection-policy.ts +0 -104
  120. package/src/adapters/claude/hooks/semantic-daemon-client.ts +0 -209
  121. package/src/adapters/claude/hooks/semantic-daemon.ts +0 -283
  122. package/src/adapters/claude/hooks/session-end.ts +0 -59
  123. package/src/adapters/claude/hooks/session-start.ts +0 -73
  124. package/src/adapters/claude/hooks/stop.ts +0 -128
  125. package/src/adapters/claude/hooks/user-prompt-submit.ts +0 -361
  126. package/src/adapters/claude/index.ts +0 -4
  127. package/src/adapters/claude/transcript/index.ts +0 -4
  128. package/src/adapters/claude/transcript/transcript-reader.ts +0 -57
  129. package/src/adapters/claude/transcript/turn-reconstructor.ts +0 -65
  130. package/src/apps/cli/claude-settings-hooks.ts +0 -138
  131. package/src/apps/cli/codex-import-runner.ts +0 -125
  132. package/src/apps/cli/codex-validation-output.ts +0 -95
  133. package/src/apps/cli/hermes-import-runner.ts +0 -130
  134. package/src/apps/cli/hermes-validation-output.ts +0 -91
  135. package/src/apps/cli/index.ts +0 -1735
  136. package/src/apps/cli/mcp-install.ts +0 -106
  137. package/src/apps/cli/retrieval-disclosure-output.ts +0 -196
  138. package/src/apps/dashboard/assets/js/bootstrap.js +0 -244
  139. package/src/apps/dashboard/assets/js/chat.js +0 -373
  140. package/src/apps/dashboard/assets/js/disclosure.js +0 -232
  141. package/src/apps/dashboard/assets/js/modals.js +0 -298
  142. package/src/apps/dashboard/assets/js/overview.js +0 -655
  143. package/src/apps/dashboard/assets/js/state.js +0 -72
  144. package/src/apps/dashboard/assets/js/views.js +0 -468
  145. package/src/apps/dashboard/index.html +0 -543
  146. package/src/apps/dashboard/index.ts +0 -3
  147. package/src/apps/dashboard/style.css +0 -1750
  148. package/src/apps/index.ts +0 -5
  149. package/src/apps/server/api/chat.ts +0 -244
  150. package/src/apps/server/api/citations.ts +0 -105
  151. package/src/apps/server/api/events.ts +0 -137
  152. package/src/apps/server/api/health.ts +0 -53
  153. package/src/apps/server/api/index.ts +0 -26
  154. package/src/apps/server/api/projects.ts +0 -74
  155. package/src/apps/server/api/search.ts +0 -184
  156. package/src/apps/server/api/sessions.ts +0 -115
  157. package/src/apps/server/api/stats.ts +0 -723
  158. package/src/apps/server/api/turns.ts +0 -143
  159. package/src/apps/server/api/utils.ts +0 -65
  160. package/src/apps/server/index.ts +0 -111
  161. package/src/cli/index.ts +0 -3
  162. package/src/cli/retrieval-disclosure-output.ts +0 -2
  163. package/src/compat/index.ts +0 -5
  164. package/src/core/canonical-key.ts +0 -186
  165. package/src/core/citation-generator.ts +0 -63
  166. package/src/core/consolidated-store.ts +0 -356
  167. package/src/core/consolidation-worker.ts +0 -493
  168. package/src/core/context-formatter.ts +0 -276
  169. package/src/core/continuity-manager.ts +0 -341
  170. package/src/core/db-wrapper.ts +0 -64
  171. package/src/core/derive/fact-deriver.ts +0 -170
  172. package/src/core/derive/index.ts +0 -2
  173. package/src/core/derive/summary-deriver.ts +0 -76
  174. package/src/core/edge-repo.ts +0 -333
  175. package/src/core/embedder.ts +0 -4
  176. package/src/core/engine/embedding-maintenance-service.ts +0 -187
  177. package/src/core/engine/endless-memory-services.ts +0 -4
  178. package/src/core/engine/index.ts +0 -19
  179. package/src/core/engine/memory-engine-services.ts +0 -170
  180. package/src/core/engine/memory-ingest-service.ts +0 -317
  181. package/src/core/engine/memory-query-service.ts +0 -173
  182. package/src/core/engine/memory-runtime-service.ts +0 -162
  183. package/src/core/engine/memory-service-composition.ts +0 -231
  184. package/src/core/engine/retrieval-analytics-service.ts +0 -181
  185. package/src/core/engine/retrieval-disclosure-service.ts +0 -420
  186. package/src/core/engine/retrieval-orchestrator.ts +0 -377
  187. package/src/core/engine/retrieval-services.ts +0 -176
  188. package/src/core/engine/shared-memory-services.ts +0 -4
  189. package/src/core/entity-repo.ts +0 -349
  190. package/src/core/event-store.ts +0 -779
  191. package/src/core/evidence-aligner.ts +0 -635
  192. package/src/core/external-market-context.ts +0 -582
  193. package/src/core/graduation-worker.ts +0 -171
  194. package/src/core/graduation.ts +0 -377
  195. package/src/core/index.ts +0 -64
  196. package/src/core/ingest-interceptor.ts +0 -80
  197. package/src/core/markdown-mirror.ts +0 -70
  198. package/src/core/matcher.ts +0 -208
  199. package/src/core/md-mirror.ts +0 -92
  200. package/src/core/metadata-extractor.ts +0 -203
  201. package/src/core/model/memory-fact.ts +0 -30
  202. package/src/core/model/memory-rule.ts +0 -14
  203. package/src/core/model/memory-summary.ts +0 -21
  204. package/src/core/model/raw-event.ts +0 -28
  205. package/src/core/model/retrieval-result.ts +0 -35
  206. package/src/core/mongo-sync-config.ts +0 -165
  207. package/src/core/mongo-sync-worker.ts +0 -381
  208. package/src/core/privacy/filter.ts +0 -190
  209. package/src/core/privacy/index.ts +0 -20
  210. package/src/core/privacy/tag-parser.ts +0 -145
  211. package/src/core/product-validation-matrix.ts +0 -314
  212. package/src/core/progressive-retriever.ts +0 -414
  213. package/src/core/registry/project-path.ts +0 -54
  214. package/src/core/registry/session-registry.ts +0 -69
  215. package/src/core/replay-evaluator.ts +0 -625
  216. package/src/core/retrieval-benchmark.ts +0 -117
  217. package/src/core/retrieval-quality.ts +0 -109
  218. package/src/core/retriever.ts +0 -800
  219. package/src/core/session-qrels.ts +0 -360
  220. package/src/core/shared-event-store.ts +0 -114
  221. package/src/core/shared-promoter.ts +0 -249
  222. package/src/core/shared-store.ts +0 -289
  223. package/src/core/shared-vector-store.ts +0 -203
  224. package/src/core/sqlite-event-store.ts +0 -1846
  225. package/src/core/sqlite-wrapper.ts +0 -116
  226. package/src/core/sync-worker.ts +0 -228
  227. package/src/core/tag-taxonomy.ts +0 -51
  228. package/src/core/task/blocker-resolver.ts +0 -333
  229. package/src/core/task/index.ts +0 -9
  230. package/src/core/task/task-matcher.ts +0 -240
  231. package/src/core/task/task-projector.ts +0 -358
  232. package/src/core/task/task-resolver.ts +0 -421
  233. package/src/core/turn-state.ts +0 -207
  234. package/src/core/types.ts +0 -952
  235. package/src/core/vector-outbox.ts +0 -299
  236. package/src/core/vector-store.ts +0 -231
  237. package/src/core/vector-worker.ts +0 -521
  238. package/src/core/working-set-store.ts +0 -257
  239. package/src/extensions/endless-memory/endless-memory-services.ts +0 -350
  240. package/src/extensions/endless-memory/index.ts +0 -1
  241. package/src/extensions/index.ts +0 -5
  242. package/src/extensions/mcp/handlers.ts +0 -960
  243. package/src/extensions/mcp/index.ts +0 -48
  244. package/src/extensions/mcp/tools.ts +0 -252
  245. package/src/extensions/shared-memory/index.ts +0 -1
  246. package/src/extensions/shared-memory/shared-memory-services.ts +0 -211
  247. package/src/extensions/vector/embedder.ts +0 -234
  248. package/src/extensions/vector/index.ts +0 -1
  249. package/src/hooks/post-tool-use.ts +0 -9
  250. package/src/hooks/semantic-daemon-client.ts +0 -1
  251. package/src/hooks/semantic-daemon.ts +0 -11
  252. package/src/hooks/session-end.ts +0 -9
  253. package/src/hooks/session-start.ts +0 -9
  254. package/src/hooks/stop.ts +0 -9
  255. package/src/hooks/user-prompt-submit.ts +0 -9
  256. package/src/index.ts +0 -13
  257. package/src/mcp/handlers.ts +0 -2
  258. package/src/mcp/index.ts +0 -4
  259. package/src/mcp/tools.ts +0 -2
  260. package/src/server/api/chat.ts +0 -2
  261. package/src/server/api/citations.ts +0 -2
  262. package/src/server/api/events.ts +0 -2
  263. package/src/server/api/health.ts +0 -2
  264. package/src/server/api/index.ts +0 -2
  265. package/src/server/api/projects.ts +0 -2
  266. package/src/server/api/search.ts +0 -2
  267. package/src/server/api/sessions.ts +0 -2
  268. package/src/server/api/stats.ts +0 -2
  269. package/src/server/api/turns.ts +0 -2
  270. package/src/server/api/utils.ts +0 -2
  271. package/src/server/index.ts +0 -2
  272. package/src/services/bootstrap-organizer.ts +0 -463
  273. package/src/services/codex-session-history-importer.ts +0 -966
  274. package/src/services/hermes-session-history-importer.ts +0 -733
  275. package/src/services/memory-service-config.ts +0 -36
  276. package/src/services/memory-service-registry.ts +0 -150
  277. package/src/services/memory-service.ts +0 -688
  278. package/src/services/session-history-importer.ts +0 -629
  279. package/tests/README.md +0 -23
  280. package/tests/adapters/claude/claude-semantic-daemon-adapter.test.ts +0 -54
  281. package/tests/adapters/claude/claude-transcript-reconstructor.test.ts +0 -98
  282. package/tests/adapters/claude-hook-prompt-injection-policy.test.ts +0 -99
  283. package/tests/apps/app-layer-boundary.test.ts +0 -48
  284. package/tests/apps/claude-settings-hooks.test.ts +0 -107
  285. package/tests/apps/cli-disclosure-output.test.ts +0 -212
  286. package/tests/apps/codex-import-runner.test.ts +0 -99
  287. package/tests/apps/codex-validation-output.test.ts +0 -100
  288. package/tests/apps/hermes-import-runner.test.ts +0 -99
  289. package/tests/apps/mcp-install-command.test.ts +0 -59
  290. package/tests/apps/package-build-entrypoints.test.ts +0 -30
  291. package/tests/apps/postinstall-embedding-backend.test.ts +0 -185
  292. package/tests/apps/search-api-disclosure.test.ts +0 -162
  293. package/tests/apps/stats-api-lightweight.test.ts +0 -67
  294. package/tests/apps/ui-disclosure-output.test.ts +0 -140
  295. package/tests/core/bootstrap-organizer.test.ts +0 -111
  296. package/tests/core/canonical-key.test.ts +0 -101
  297. package/tests/core/codex-session-history-importer-validation.test.ts +0 -185
  298. package/tests/core/consolidation-worker.test.ts +0 -75
  299. package/tests/core/embedding-maintenance-service.test.ts +0 -282
  300. package/tests/core/evidence-aligner.test.ts +0 -152
  301. package/tests/core/external-market-context.test.ts +0 -209
  302. package/tests/core/fact-deriver.test.ts +0 -79
  303. package/tests/core/hermes-session-history-importer-validation.test.ts +0 -609
  304. package/tests/core/ingest-interceptor.test.ts +0 -38
  305. package/tests/core/markdown-mirror.test.ts +0 -85
  306. package/tests/core/matcher.test.ts +0 -112
  307. package/tests/core/md-mirror.test.ts +0 -50
  308. package/tests/core/memory-engine-services.test.ts +0 -240
  309. package/tests/core/memory-ingest-service.test.ts +0 -296
  310. package/tests/core/memory-query-service.test.ts +0 -129
  311. package/tests/core/memory-runtime-service.test.ts +0 -201
  312. package/tests/core/memory-service-composition.test.ts +0 -192
  313. package/tests/core/memory-service-config.test.ts +0 -41
  314. package/tests/core/memory-service-facade.test.ts +0 -30
  315. package/tests/core/memory-service-registry.test.ts +0 -206
  316. package/tests/core/product-validation-matrix.test.ts +0 -61
  317. package/tests/core/project-registry.test.ts +0 -78
  318. package/tests/core/replay-evaluator.test.ts +0 -181
  319. package/tests/core/retrieval-analytics-service.test.ts +0 -210
  320. package/tests/core/retrieval-benchmark.test.ts +0 -93
  321. package/tests/core/retrieval-disclosure-service.test.ts +0 -264
  322. package/tests/core/retrieval-orchestrator.test.ts +0 -403
  323. package/tests/core/retrieval-quality.test.ts +0 -31
  324. package/tests/core/retrieval-services.test.ts +0 -185
  325. package/tests/core/retriever-fallback-chain.test.ts +0 -223
  326. package/tests/core/retriever-strategy-scope.test.ts +0 -164
  327. package/tests/core/retriever.memu-adoption.test.ts +0 -122
  328. package/tests/core/session-history-importer-filter.test.ts +0 -78
  329. package/tests/core/session-qrels.test.ts +0 -250
  330. package/tests/core/sqlite-event-store-replication.test.ts +0 -127
  331. package/tests/core/summary-deriver.test.ts +0 -66
  332. package/tests/extensions/embedder-warning-suppression.test.ts +0 -84
  333. package/tests/extensions/endless-memory-extension-boundary.test.ts +0 -17
  334. package/tests/extensions/endless-memory-services.test.ts +0 -325
  335. package/tests/extensions/mcp-context-tools.test.ts +0 -905
  336. package/tests/extensions/mcp-extension-boundary.test.ts +0 -21
  337. package/tests/extensions/mcp-package-build.test.ts +0 -22
  338. package/tests/extensions/mcp-project-aware-tools.test.ts +0 -102
  339. package/tests/extensions/shared-memory-extension-boundary.test.ts +0 -24
  340. package/tests/extensions/shared-memory-services.test.ts +0 -309
  341. package/tests/extensions/vector-extension-boundary.test.ts +0 -21
  342. package/tsconfig.json +0 -24
  343. package/vitest.config.ts +0 -15
@@ -1,1846 +0,0 @@
1
- /**
2
- * SQLite-based EventStore implementation
3
- * Primary store for hooks - WAL mode enables concurrent access
4
- */
5
-
6
- import { randomUUID } from 'crypto';
7
- import {
8
- MemoryEvent,
9
- MemoryEventInput,
10
- Session,
11
- AppendResult,
12
- OutboxItem
13
- } from './types.js';
14
- import { makeCanonicalKey, makeDedupeKey } from './canonical-key.js';
15
- import {
16
- createSQLiteDatabase,
17
- sqliteRun,
18
- sqliteAll,
19
- sqliteGet,
20
- sqliteClose,
21
- sqliteExec,
22
- toDateFromSQLite,
23
- toSQLiteTimestamp,
24
- type SQLiteDatabase,
25
- type SQLiteOptions
26
- } from './sqlite-wrapper.js';
27
- import { MarkdownMirror } from './markdown-mirror.js';
28
-
29
- export interface SQLiteEventStoreOptions extends SQLiteOptions {
30
- markdownMirrorRoot?: string;
31
- }
32
-
33
- export class SQLiteEventStore {
34
- private db: SQLiteDatabase;
35
- private initialized = false;
36
- private readonly readOnly: boolean;
37
- private readonly markdownMirror: MarkdownMirror | null;
38
-
39
- constructor(dbPath: string, options?: SQLiteEventStoreOptions) {
40
- this.readOnly = options?.readonly ?? false;
41
- this.db = createSQLiteDatabase(dbPath, {
42
- readonly: this.readOnly,
43
- walMode: !this.readOnly
44
- });
45
- this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot
46
- ? null
47
- : new MarkdownMirror(options.markdownMirrorRoot);
48
- }
49
-
50
- /**
51
- * Initialize database schema
52
- */
53
- async initialize(): Promise<void> {
54
- if (this.initialized) return;
55
-
56
- // In read-only mode, skip schema creation
57
- if (this.readOnly) {
58
- this.initialized = true;
59
- return;
60
- }
61
-
62
- // Create all tables in a single exec for efficiency
63
- sqliteExec(this.db, `
64
- -- L0 EventStore: Single Source of Truth (immutable, append-only)
65
- CREATE TABLE IF NOT EXISTS events (
66
- id TEXT PRIMARY KEY,
67
- event_type TEXT NOT NULL,
68
- session_id TEXT NOT NULL,
69
- timestamp TEXT NOT NULL,
70
- content TEXT NOT NULL,
71
- canonical_key TEXT NOT NULL,
72
- dedupe_key TEXT UNIQUE,
73
- metadata TEXT,
74
- access_count INTEGER DEFAULT 0,
75
- last_accessed_at TEXT
76
- );
77
-
78
- -- Dedup table for idempotency
79
- CREATE TABLE IF NOT EXISTS event_dedup (
80
- dedupe_key TEXT PRIMARY KEY,
81
- event_id TEXT NOT NULL,
82
- created_at TEXT DEFAULT (datetime('now'))
83
- );
84
-
85
- -- Session metadata
86
- CREATE TABLE IF NOT EXISTS sessions (
87
- id TEXT PRIMARY KEY,
88
- started_at TEXT NOT NULL,
89
- ended_at TEXT,
90
- project_path TEXT,
91
- summary TEXT,
92
- tags TEXT
93
- );
94
-
95
- -- Insights (derived data, rebuildable)
96
- CREATE TABLE IF NOT EXISTS insights (
97
- id TEXT PRIMARY KEY,
98
- insight_type TEXT NOT NULL,
99
- content TEXT NOT NULL,
100
- canonical_key TEXT NOT NULL,
101
- confidence REAL,
102
- source_events TEXT,
103
- created_at TEXT,
104
- last_updated TEXT
105
- );
106
-
107
- -- Embedding Outbox (Single-Writer Pattern)
108
- CREATE TABLE IF NOT EXISTS embedding_outbox (
109
- id TEXT PRIMARY KEY,
110
- event_id TEXT NOT NULL,
111
- content TEXT NOT NULL,
112
- status TEXT DEFAULT 'pending',
113
- retry_count INTEGER DEFAULT 0,
114
- created_at TEXT DEFAULT (datetime('now')),
115
- processed_at TEXT,
116
- error_message TEXT
117
- );
118
-
119
- -- Projection offset tracking
120
- CREATE TABLE IF NOT EXISTS projection_offsets (
121
- projection_name TEXT PRIMARY KEY,
122
- last_event_id TEXT,
123
- last_timestamp TEXT,
124
- updated_at TEXT DEFAULT (datetime('now'))
125
- );
126
-
127
- -- Memory level tracking
128
- CREATE TABLE IF NOT EXISTS memory_levels (
129
- event_id TEXT PRIMARY KEY,
130
- level TEXT NOT NULL DEFAULT 'L0',
131
- promoted_at TEXT DEFAULT (datetime('now'))
132
- );
133
-
134
- -- Entries (immutable memory units)
135
- CREATE TABLE IF NOT EXISTS entries (
136
- entry_id TEXT PRIMARY KEY,
137
- created_ts TEXT NOT NULL,
138
- entry_type TEXT NOT NULL,
139
- title TEXT NOT NULL,
140
- content_json TEXT NOT NULL,
141
- stage TEXT NOT NULL DEFAULT 'raw',
142
- status TEXT DEFAULT 'active',
143
- superseded_by TEXT,
144
- build_id TEXT,
145
- evidence_json TEXT,
146
- canonical_key TEXT,
147
- created_at TEXT DEFAULT (datetime('now'))
148
- );
149
-
150
- -- Entities (task/condition/artifact)
151
- CREATE TABLE IF NOT EXISTS entities (
152
- entity_id TEXT PRIMARY KEY,
153
- entity_type TEXT NOT NULL,
154
- canonical_key TEXT NOT NULL,
155
- title TEXT NOT NULL,
156
- stage TEXT NOT NULL DEFAULT 'raw',
157
- status TEXT NOT NULL DEFAULT 'active',
158
- current_json TEXT NOT NULL,
159
- title_norm TEXT,
160
- search_text TEXT,
161
- created_at TEXT DEFAULT (datetime('now')),
162
- updated_at TEXT DEFAULT (datetime('now'))
163
- );
164
-
165
- -- Entity aliases for canonical key lookup
166
- CREATE TABLE IF NOT EXISTS entity_aliases (
167
- entity_type TEXT NOT NULL,
168
- canonical_key TEXT NOT NULL,
169
- entity_id TEXT NOT NULL,
170
- is_primary INTEGER DEFAULT 0,
171
- created_at TEXT DEFAULT (datetime('now')),
172
- PRIMARY KEY(entity_type, canonical_key)
173
- );
174
-
175
- -- Edges (relationships between entries/entities)
176
- CREATE TABLE IF NOT EXISTS edges (
177
- edge_id TEXT PRIMARY KEY,
178
- src_type TEXT NOT NULL,
179
- src_id TEXT NOT NULL,
180
- rel_type TEXT NOT NULL,
181
- dst_type TEXT NOT NULL,
182
- dst_id TEXT NOT NULL,
183
- meta_json TEXT,
184
- created_at TEXT DEFAULT (datetime('now'))
185
- );
186
-
187
- -- Vector Outbox V2 Table
188
- CREATE TABLE IF NOT EXISTS vector_outbox (
189
- job_id TEXT PRIMARY KEY,
190
- item_kind TEXT NOT NULL,
191
- item_id TEXT NOT NULL,
192
- embedding_version TEXT NOT NULL,
193
- status TEXT NOT NULL DEFAULT 'pending',
194
- retry_count INTEGER DEFAULT 0,
195
- error TEXT,
196
- created_at TEXT DEFAULT (datetime('now')),
197
- updated_at TEXT DEFAULT (datetime('now')),
198
- UNIQUE(item_kind, item_id, embedding_version)
199
- );
200
-
201
- -- Build Runs
202
- CREATE TABLE IF NOT EXISTS build_runs (
203
- build_id TEXT PRIMARY KEY,
204
- started_at TEXT NOT NULL,
205
- finished_at TEXT,
206
- extractor_model TEXT NOT NULL,
207
- extractor_prompt_hash TEXT NOT NULL,
208
- embedder_model TEXT NOT NULL,
209
- embedding_version TEXT NOT NULL,
210
- idris_version TEXT NOT NULL,
211
- schema_version TEXT NOT NULL,
212
- status TEXT NOT NULL DEFAULT 'running',
213
- error TEXT
214
- );
215
-
216
- -- Pipeline Metrics
217
- CREATE TABLE IF NOT EXISTS pipeline_metrics (
218
- id TEXT PRIMARY KEY,
219
- ts TEXT NOT NULL,
220
- stage TEXT NOT NULL,
221
- latency_ms REAL NOT NULL,
222
- success INTEGER NOT NULL,
223
- error TEXT,
224
- session_id TEXT
225
- );
226
-
227
- -- Working Set table (active memory window)
228
- CREATE TABLE IF NOT EXISTS working_set (
229
- id TEXT PRIMARY KEY,
230
- event_id TEXT NOT NULL,
231
- added_at TEXT DEFAULT (datetime('now')),
232
- relevance_score REAL DEFAULT 1.0,
233
- topics TEXT,
234
- expires_at TEXT
235
- );
236
-
237
- -- Consolidated Memories table (long-term integrated memories)
238
- CREATE TABLE IF NOT EXISTS consolidated_memories (
239
- memory_id TEXT PRIMARY KEY,
240
- summary TEXT NOT NULL,
241
- topics TEXT,
242
- source_events TEXT,
243
- confidence REAL DEFAULT 0.5,
244
- created_at TEXT DEFAULT (datetime('now')),
245
- accessed_at TEXT,
246
- access_count INTEGER DEFAULT 0
247
- );
248
-
249
- -- Continuity Log table (tracks context transitions)
250
- CREATE TABLE IF NOT EXISTS continuity_log (
251
- log_id TEXT PRIMARY KEY,
252
- from_context_id TEXT,
253
- to_context_id TEXT,
254
- continuity_score REAL,
255
- transition_type TEXT,
256
- created_at TEXT DEFAULT (datetime('now'))
257
- );
258
-
259
- -- Consolidated Rules table (long-term stable memory)
260
- CREATE TABLE IF NOT EXISTS consolidated_rules (
261
- rule_id TEXT PRIMARY KEY,
262
- rule TEXT NOT NULL,
263
- topics TEXT,
264
- source_memory_ids TEXT,
265
- source_events TEXT,
266
- confidence REAL DEFAULT 0.5,
267
- created_at TEXT DEFAULT (datetime('now'))
268
- );
269
-
270
- -- Endless Mode Config table
271
- CREATE TABLE IF NOT EXISTS endless_config (
272
- key TEXT PRIMARY KEY,
273
- value TEXT,
274
- updated_at TEXT DEFAULT (datetime('now'))
275
- );
276
-
277
- -- Memory Helpfulness tracking
278
- CREATE TABLE IF NOT EXISTS memory_helpfulness (
279
- id TEXT PRIMARY KEY,
280
- event_id TEXT NOT NULL,
281
- session_id TEXT NOT NULL,
282
- retrieval_score REAL DEFAULT 0,
283
- query_preview TEXT,
284
- session_continued INTEGER DEFAULT 0,
285
- prompt_count_after INTEGER DEFAULT 0,
286
- tool_success_count INTEGER DEFAULT 0,
287
- tool_total_count INTEGER DEFAULT 0,
288
- was_reasked INTEGER DEFAULT 0,
289
- helpfulness_score REAL DEFAULT 0.5,
290
- created_at TEXT DEFAULT (datetime('now')),
291
- measured_at TEXT
292
- );
293
-
294
- -- Retrieval trace log (query -> candidates -> selected for context)
295
- CREATE TABLE IF NOT EXISTS retrieval_traces (
296
- trace_id TEXT PRIMARY KEY,
297
- session_id TEXT,
298
- project_hash TEXT,
299
- query_text TEXT NOT NULL,
300
- strategy TEXT,
301
- candidate_event_ids TEXT,
302
- selected_event_ids TEXT,
303
- candidate_details_json TEXT,
304
- selected_details_json TEXT,
305
- candidate_count INTEGER DEFAULT 0,
306
- selected_count INTEGER DEFAULT 0,
307
- confidence TEXT,
308
- fallback_trace TEXT,
309
- created_at TEXT DEFAULT (datetime('now'))
310
- );
311
-
312
- -- Sync position tracking (for SQLite -> DuckDB sync)
313
- CREATE TABLE IF NOT EXISTS sync_positions (
314
- target_name TEXT PRIMARY KEY,
315
- last_event_id TEXT,
316
- last_timestamp TEXT,
317
- updated_at TEXT DEFAULT (datetime('now'))
318
- );
319
-
320
- -- Create indexes
321
- CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
322
- CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp);
323
- CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
324
- CREATE INDEX IF NOT EXISTS idx_entries_stage ON entries(stage);
325
- CREATE INDEX IF NOT EXISTS idx_entries_canonical ON entries(canonical_key);
326
- CREATE INDEX IF NOT EXISTS idx_entities_type_key ON entities(entity_type, canonical_key);
327
- CREATE INDEX IF NOT EXISTS idx_entities_status ON entities(status);
328
- CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src_id, rel_type);
329
- CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst_id, rel_type);
330
- CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(rel_type);
331
- CREATE INDEX IF NOT EXISTS idx_outbox_status ON vector_outbox(status);
332
- CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at);
333
- CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
334
- CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
335
- CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
336
- CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
337
- CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
338
- CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
339
- CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
340
- CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
341
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
342
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
343
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
344
-
345
- -- FTS5 Full-Text Search for fast keyword search
346
- CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
347
- content,
348
- event_id UNINDEXED,
349
- tokenize='porter unicode61'
350
- );
351
-
352
- -- Triggers to keep FTS in sync with events table
353
- CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
354
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
355
- END;
356
-
357
- CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
358
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
359
- END;
360
-
361
- CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
362
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
363
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
364
- END;
365
- `);
366
-
367
-
368
- // Best-effort forward migration for retrieval trace detail column
369
- try {
370
- sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
371
- } catch {
372
- // column may already exist
373
- }
374
- try {
375
- sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
376
- } catch {
377
- // column may already exist
378
- }
379
-
380
- // Migrate existing events table to add new columns if they don't exist
381
- // Check if columns exist before trying to add them
382
- const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
383
- const columnNames = tableInfo.map((col: any) => col.name);
384
-
385
- if (!columnNames.includes('access_count')) {
386
- try {
387
- sqliteExec(this.db, `
388
- ALTER TABLE events ADD COLUMN access_count INTEGER DEFAULT 0;
389
- `);
390
- } catch (err: any) {
391
- console.error('Error adding access_count column:', err);
392
- }
393
- }
394
-
395
- if (!columnNames.includes('last_accessed_at')) {
396
- try {
397
- sqliteExec(this.db, `
398
- ALTER TABLE events ADD COLUMN last_accessed_at TEXT;
399
- `);
400
- } catch (err: any) {
401
- console.error('Error adding last_accessed_at column:', err);
402
- }
403
- }
404
-
405
- // Add turn_id column for grouping events within a conversation turn
406
- if (!columnNames.includes('turn_id')) {
407
- try {
408
- sqliteExec(this.db, `
409
- ALTER TABLE events ADD COLUMN turn_id TEXT;
410
- `);
411
- } catch (err: any) {
412
- console.error('Error adding turn_id column:', err);
413
- }
414
- }
415
-
416
- // Create indexes for new columns if they don't exist
417
- try {
418
- sqliteExec(this.db, `
419
- CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
420
- `);
421
- } catch (err: any) {
422
- // Index may already exist, ignore
423
- }
424
-
425
- try {
426
- sqliteExec(this.db, `
427
- CREATE INDEX IF NOT EXISTS idx_events_last_accessed ON events(last_accessed_at DESC);
428
- `);
429
- } catch (err: any) {
430
- // Index may already exist, ignore
431
- }
432
-
433
- try {
434
- sqliteExec(this.db, `
435
- CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
436
- `);
437
- } catch (err: any) {
438
- // Index may already exist, ignore
439
- }
440
-
441
- this.initialized = true;
442
- }
443
-
444
- /**
445
- * Append event to store (Append-only, Idempotent)
446
- */
447
- async append(input: MemoryEventInput): Promise<AppendResult> {
448
- await this.initialize();
449
-
450
- const canonicalKey = makeCanonicalKey(input.content);
451
- const dedupeKey = makeDedupeKey(input.content, input.sessionId);
452
-
453
- // Check for duplicate
454
- const existing = sqliteGet<{ event_id: string }>(
455
- this.db,
456
- `SELECT event_id FROM event_dedup WHERE dedupe_key = ?`,
457
- [dedupeKey]
458
- );
459
-
460
- if (existing) {
461
- return {
462
- success: true,
463
- eventId: existing.event_id,
464
- isDuplicate: true
465
- };
466
- }
467
-
468
- const id = randomUUID();
469
- const timestamp = toSQLiteTimestamp(input.timestamp);
470
-
471
- try {
472
- // Extract turnId from metadata if present
473
- const metadata = input.metadata || {};
474
- const turnId = (metadata.turnId as string) || null;
475
-
476
- // Use transaction for atomicity
477
- const insertEvent = this.db.prepare(`
478
- INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
479
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
480
- `);
481
-
482
- const insertDedup = this.db.prepare(`
483
- INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
484
- `);
485
-
486
- const insertLevel = this.db.prepare(`
487
- INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
488
- `);
489
-
490
- const transaction = this.db.transaction(() => {
491
- insertEvent.run(
492
- id,
493
- input.eventType,
494
- input.sessionId,
495
- timestamp,
496
- input.content,
497
- canonicalKey,
498
- dedupeKey,
499
- JSON.stringify(metadata),
500
- turnId
501
- );
502
- insertDedup.run(dedupeKey, id);
503
- insertLevel.run(id);
504
- });
505
-
506
- transaction();
507
-
508
- if (this.markdownMirror) {
509
- const event: MemoryEvent = {
510
- id,
511
- eventType: input.eventType,
512
- sessionId: input.sessionId,
513
- timestamp: input.timestamp,
514
- content: input.content,
515
- canonicalKey,
516
- dedupeKey,
517
- metadata
518
- };
519
- this.markdownMirror.append(event).catch((err) => {
520
- console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
521
- });
522
- }
523
-
524
- return { success: true, eventId: id, isDuplicate: false };
525
- } catch (error) {
526
- return {
527
- success: false,
528
- error: error instanceof Error ? error.message : String(error)
529
- };
530
- }
531
- }
532
-
533
- /**
534
- * Get session IDs that have events but no session_summary event.
535
- * Used to backfill summaries for sessions that ended without Stop hook.
536
- */
537
- async getSessionsWithoutSummary(currentSessionId: string, limit = 5): Promise<string[]> {
538
- await this.initialize();
539
- const rows = sqliteAll<{ session_id: string }>(
540
- this.db,
541
- `SELECT DISTINCT e.session_id
542
- FROM events e
543
- WHERE e.session_id != ?
544
- AND e.event_type != 'session_summary'
545
- AND e.session_id NOT IN (
546
- SELECT DISTINCT session_id FROM events WHERE event_type = 'session_summary'
547
- )
548
- GROUP BY e.session_id
549
- HAVING COUNT(*) >= 3
550
- ORDER BY MAX(e.timestamp) DESC
551
- LIMIT ?`,
552
- [currentSessionId, limit]
553
- );
554
- return rows.map((r) => r.session_id);
555
- }
556
-
557
- /**
558
- * Get events by session ID
559
- */
560
- async getSessionEvents(sessionId: string): Promise<MemoryEvent[]> {
561
- await this.initialize();
562
-
563
- const rows = sqliteAll<Record<string, unknown>>(
564
- this.db,
565
- `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
566
- [sessionId]
567
- );
568
-
569
- return rows.map(this.rowToEvent);
570
- }
571
-
572
- /**
573
- * Get recent events
574
- */
575
- async getRecentEvents(limit: number = 100): Promise<MemoryEvent[]> {
576
- await this.initialize();
577
-
578
- const rows = sqliteAll<Record<string, unknown>>(
579
- this.db,
580
- `SELECT * FROM events ORDER BY timestamp DESC LIMIT ?`,
581
- [limit]
582
- );
583
-
584
- return rows.map(this.rowToEvent);
585
- }
586
-
587
- /**
588
- * Get event by ID
589
- */
590
- async getEvent(id: string): Promise<MemoryEvent | null> {
591
- await this.initialize();
592
-
593
- const row = sqliteGet<Record<string, unknown>>(
594
- this.db,
595
- `SELECT * FROM events WHERE id = ?`,
596
- [id]
597
- );
598
-
599
- if (!row) return null;
600
- return this.rowToEvent(row);
601
- }
602
-
603
- /**
604
- * Get events since a timestamp (for sync)
605
- */
606
- async getEventsSince(timestamp: string, limit: number = 1000): Promise<MemoryEvent[]> {
607
- await this.initialize();
608
-
609
- const rows = sqliteAll<Record<string, unknown>>(
610
- this.db,
611
- `SELECT * FROM events WHERE timestamp > ? ORDER BY timestamp ASC LIMIT ?`,
612
- [timestamp, limit]
613
- );
614
-
615
- return rows.map(this.rowToEvent);
616
- }
617
-
618
- /**
619
- * Get events since a SQLite rowid (for robust incremental replication).
620
- * Rowid is monotonic for append-only tables, independent of client timestamps.
621
- */
622
- async getEventsSinceRowid(
623
- lastRowid: number,
624
- limit: number = 1000
625
- ): Promise<Array<{ rowid: number; event: MemoryEvent }>> {
626
- await this.initialize();
627
-
628
- const rows = sqliteAll<Record<string, unknown>>(
629
- this.db,
630
- `SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
631
- [lastRowid, limit]
632
- );
633
-
634
- return rows.map(row => ({
635
- rowid: row._rowid as number,
636
- event: this.rowToEvent(row)
637
- }));
638
- }
639
-
640
- /**
641
- * Import events with fixed IDs (used for cross-machine replication).
642
- * Idempotent: skips if event id or dedupeKey already exists.
643
- *
644
- * NOTE: This bypasses the append() id generation to preserve stable IDs.
645
- */
646
- async importEvents(events: MemoryEvent[]): Promise<{ inserted: number; skipped: number }> {
647
- if (events.length === 0) return { inserted: 0, skipped: 0 };
648
- if (this.readOnly) return { inserted: 0, skipped: events.length };
649
-
650
- await this.initialize();
651
-
652
- const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
653
- const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
654
-
655
- const insertEvent = this.db.prepare(`
656
- INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
657
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
658
- `);
659
-
660
- const insertDedup = this.db.prepare(`
661
- INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
662
- `);
663
-
664
- const insertLevel = this.db.prepare(`
665
- INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
666
- `);
667
-
668
- let inserted = 0;
669
- let skipped = 0;
670
- const insertedEvents: MemoryEvent[] = [];
671
-
672
- const tx = this.db.transaction((batch: MemoryEvent[]) => {
673
- for (const ev of batch) {
674
- // Skip if already present by id
675
- const existingById = getById.get(ev.id) as { id: string } | undefined;
676
- if (existingById) {
677
- skipped++;
678
- continue;
679
- }
680
-
681
- const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
682
- const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
683
-
684
- // Skip if already present by dedupe key
685
- const existingByDedupe = getByDedupe.get(dedupeKey) as { event_id: string } | undefined;
686
- if (existingByDedupe) {
687
- skipped++;
688
- continue;
689
- }
690
-
691
- const metadata = ev.metadata || {};
692
- const turnId = (metadata as any).turnId as string | undefined;
693
-
694
- insertEvent.run(
695
- ev.id,
696
- ev.eventType,
697
- ev.sessionId,
698
- toSQLiteTimestamp(ev.timestamp),
699
- ev.content,
700
- canonicalKey,
701
- dedupeKey,
702
- JSON.stringify(metadata),
703
- turnId ?? null
704
- );
705
-
706
- insertDedup.run(dedupeKey, ev.id);
707
- insertLevel.run(ev.id);
708
- inserted++;
709
- insertedEvents.push(ev);
710
- }
711
- });
712
-
713
- tx(events);
714
-
715
- if (this.markdownMirror && insertedEvents.length > 0) {
716
- for (const ev of insertedEvents) {
717
- this.markdownMirror.append(ev).catch((err) => {
718
- console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
719
- });
720
- }
721
- }
722
-
723
- return { inserted, skipped };
724
- }
725
-
726
- /**
727
- * Create or update session
728
- */
729
- async upsertSession(session: Partial<Session> & { id: string }): Promise<void> {
730
- await this.initialize();
731
-
732
- const existing = sqliteGet<{ id: string }>(
733
- this.db,
734
- `SELECT id FROM sessions WHERE id = ?`,
735
- [session.id]
736
- );
737
-
738
- if (!existing) {
739
- sqliteRun(
740
- this.db,
741
- `INSERT INTO sessions (id, started_at, project_path, tags)
742
- VALUES (?, ?, ?, ?)`,
743
- [
744
- session.id,
745
- toSQLiteTimestamp(session.startedAt || new Date()),
746
- session.projectPath || null,
747
- JSON.stringify(session.tags || [])
748
- ]
749
- );
750
- } else {
751
- const updates: string[] = [];
752
- const values: unknown[] = [];
753
-
754
- if (session.endedAt) {
755
- updates.push('ended_at = ?');
756
- values.push(toSQLiteTimestamp(session.endedAt));
757
- }
758
- if (session.summary) {
759
- updates.push('summary = ?');
760
- values.push(session.summary);
761
- }
762
- if (session.tags) {
763
- updates.push('tags = ?');
764
- values.push(JSON.stringify(session.tags));
765
- }
766
-
767
- if (updates.length > 0) {
768
- values.push(session.id);
769
- sqliteRun(
770
- this.db,
771
- `UPDATE sessions SET ${updates.join(', ')} WHERE id = ?`,
772
- values
773
- );
774
- }
775
- }
776
- }
777
-
778
- /**
779
- * Get session by ID
780
- */
781
- async getSession(id: string): Promise<Session | null> {
782
- await this.initialize();
783
-
784
- const row = sqliteGet<Record<string, unknown>>(
785
- this.db,
786
- `SELECT * FROM sessions WHERE id = ?`,
787
- [id]
788
- );
789
-
790
- if (!row) return null;
791
-
792
- return {
793
- id: row.id as string,
794
- startedAt: toDateFromSQLite(row.started_at),
795
- endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
796
- projectPath: row.project_path as string | undefined,
797
- summary: row.summary as string | undefined,
798
- tags: row.tags ? JSON.parse(row.tags as string) : undefined
799
- };
800
- }
801
-
802
- /**
803
- * Get all sessions
804
- */
805
- async getAllSessions(): Promise<Session[]> {
806
- await this.initialize();
807
-
808
- const rows = sqliteAll<Record<string, unknown>>(
809
- this.db,
810
- `SELECT * FROM sessions ORDER BY started_at DESC`
811
- );
812
-
813
- return rows.map(row => ({
814
- id: row.id as string,
815
- startedAt: toDateFromSQLite(row.started_at),
816
- endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
817
- projectPath: row.project_path as string | undefined,
818
- summary: row.summary as string | undefined,
819
- tags: row.tags ? JSON.parse(row.tags as string) : undefined
820
- }));
821
- }
822
-
823
- /**
824
- * Add to embedding outbox
825
- */
826
- async enqueueForEmbedding(eventId: string, content: string): Promise<string> {
827
- await this.initialize();
828
-
829
- const id = randomUUID();
830
- sqliteRun(
831
- this.db,
832
- `INSERT INTO embedding_outbox (id, event_id, content, status, retry_count)
833
- VALUES (?, ?, ?, 'pending', 0)`,
834
- [id, eventId, content]
835
- );
836
-
837
- return id;
838
- }
839
-
840
- /**
841
- * Get pending outbox items
842
- */
843
- async getPendingOutboxItems(limit: number = 32): Promise<OutboxItem[]> {
844
- await this.initialize();
845
-
846
- const pending = sqliteAll<Record<string, unknown>>(
847
- this.db,
848
- `SELECT * FROM embedding_outbox
849
- WHERE status = 'pending'
850
- ORDER BY created_at
851
- LIMIT ?`,
852
- [limit]
853
- );
854
-
855
- if (pending.length === 0) return [];
856
-
857
- // Update status to processing
858
- const ids = pending.map(r => r.id as string);
859
- const placeholders = ids.map(() => '?').join(',');
860
- sqliteRun(
861
- this.db,
862
- `UPDATE embedding_outbox SET status = 'processing' WHERE id IN (${placeholders})`,
863
- ids
864
- );
865
-
866
- return pending.map(row => ({
867
- id: row.id as string,
868
- eventId: row.event_id as string,
869
- content: row.content as string,
870
- status: 'processing' as const,
871
- retryCount: row.retry_count as number,
872
- createdAt: toDateFromSQLite(row.created_at),
873
- errorMessage: row.error_message as string | undefined
874
- }));
875
- }
876
-
877
- /**
878
- * Mark outbox items as done
879
- */
880
- async completeOutboxItems(ids: string[]): Promise<void> {
881
- if (ids.length === 0) return;
882
-
883
- const placeholders = ids.map(() => '?').join(',');
884
- sqliteRun(
885
- this.db,
886
- `DELETE FROM embedding_outbox WHERE id IN (${placeholders})`,
887
- ids
888
- );
889
- }
890
-
891
- /**
892
- * Clear embedding outbox (used for embedding model migration)
893
- */
894
- async clearEmbeddingOutbox(): Promise<void> {
895
- await this.initialize();
896
- sqliteRun(this.db, `DELETE FROM embedding_outbox`);
897
- }
898
-
899
- /**
900
- * Count total events
901
- */
902
- async countEvents(): Promise<number> {
903
- await this.initialize();
904
- const row = sqliteGet<{ count: number }>(this.db, `SELECT COUNT(*) as count FROM events`);
905
- return row?.count || 0;
906
- }
907
-
908
- /**
909
- * Get events page in timestamp ascending order (stable migration/reindex scans)
910
- */
911
- async getEventsPage(limit: number = 1000, offset: number = 0): Promise<MemoryEvent[]> {
912
- await this.initialize();
913
-
914
- const rows = sqliteAll<Record<string, unknown>>(
915
- this.db,
916
- `SELECT * FROM events ORDER BY timestamp ASC LIMIT ? OFFSET ?`,
917
- [limit, offset]
918
- );
919
-
920
- return rows.map(this.rowToEvent);
921
- }
922
-
923
- /**
924
- * Mark outbox items as failed
925
- */
926
- async failOutboxItems(ids: string[], error: string): Promise<void> {
927
- if (ids.length === 0) return;
928
-
929
- const placeholders = ids.map(() => '?').join(',');
930
- sqliteRun(
931
- this.db,
932
- `UPDATE embedding_outbox
933
- SET status = CASE WHEN retry_count >= 3 THEN 'failed' ELSE 'pending' END,
934
- retry_count = retry_count + 1,
935
- error_message = ?
936
- WHERE id IN (${placeholders})`,
937
- [error, ...ids]
938
- );
939
- }
940
-
941
-
942
- /**
943
- * Get embedding/vector outbox health statistics
944
- */
945
- async getOutboxStats(): Promise<{
946
- embedding: { pending: number; processing: number; failed: number; total: number };
947
- vector: { pending: number; processing: number; failed: number; total: number };
948
- }> {
949
- await this.initialize();
950
-
951
- const embeddingRows = sqliteAll<{ status: string; count: number }>(
952
- this.db,
953
- `SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
954
- );
955
- const vectorRows = sqliteAll<{ status: string; count: number }>(
956
- this.db,
957
- `SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
958
- );
959
-
960
- const fromRows = (rows: Array<{ status: string; count: number }>) => {
961
- const out = { pending: 0, processing: 0, failed: 0, total: 0 };
962
- for (const row of rows) {
963
- const key = row.status as 'pending' | 'processing' | 'failed' | 'done';
964
- if (key === 'pending' || key === 'processing' || key === 'failed') {
965
- out[key] += row.count;
966
- }
967
- out.total += row.count;
968
- }
969
- return out;
970
- };
971
-
972
- return {
973
- embedding: fromRows(embeddingRows),
974
- vector: fromRows(vectorRows)
975
- };
976
- }
977
-
978
- /**
979
- * Update memory level
980
- */
981
- async updateMemoryLevel(eventId: string, level: string): Promise<void> {
982
- await this.initialize();
983
-
984
- sqliteRun(
985
- this.db,
986
- `UPDATE memory_levels SET level = ?, promoted_at = datetime('now') WHERE event_id = ?`,
987
- [level, eventId]
988
- );
989
- }
990
-
991
- /**
992
- * Get memory level statistics
993
- */
994
- async getLevelStats(): Promise<Array<{ level: string; count: number }>> {
995
- await this.initialize();
996
-
997
- const rows = sqliteAll<{ level: string; count: number }>(
998
- this.db,
999
- `SELECT level, COUNT(*) as count FROM memory_levels GROUP BY level`
1000
- );
1001
-
1002
- return rows;
1003
- }
1004
-
1005
- /**
1006
- * Get events by memory level
1007
- */
1008
- async getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]> {
1009
- await this.initialize();
1010
-
1011
- const limit = options?.limit || 50;
1012
- const offset = options?.offset || 0;
1013
-
1014
- const rows = sqliteAll<Record<string, unknown>>(
1015
- this.db,
1016
- `SELECT e.* FROM events e
1017
- INNER JOIN memory_levels ml ON e.id = ml.event_id
1018
- WHERE ml.level = ?
1019
- ORDER BY e.timestamp DESC
1020
- LIMIT ? OFFSET ?`,
1021
- [level, limit, offset]
1022
- );
1023
-
1024
- return rows.map(row => this.rowToEvent(row));
1025
- }
1026
-
1027
- /**
1028
- * Get memory level for a specific event
1029
- */
1030
- async getEventLevel(eventId: string): Promise<string | null> {
1031
- await this.initialize();
1032
-
1033
- const row = sqliteGet<{ level: string }>(
1034
- this.db,
1035
- `SELECT level FROM memory_levels WHERE event_id = ?`,
1036
- [eventId]
1037
- );
1038
-
1039
- return row ? row.level : null;
1040
- }
1041
-
1042
- /**
1043
- * Get sync position for a target
1044
- */
1045
- async getSyncPosition(targetName: string): Promise<{ lastEventId: string | null; lastTimestamp: string | null }> {
1046
- await this.initialize();
1047
-
1048
- const row = sqliteGet<{ last_event_id: string | null; last_timestamp: string | null }>(
1049
- this.db,
1050
- `SELECT last_event_id, last_timestamp FROM sync_positions WHERE target_name = ?`,
1051
- [targetName]
1052
- );
1053
-
1054
- return {
1055
- lastEventId: row?.last_event_id ?? null,
1056
- lastTimestamp: row?.last_timestamp ?? null
1057
- };
1058
- }
1059
-
1060
- /**
1061
- * Update sync position for a target
1062
- */
1063
- async updateSyncPosition(targetName: string, lastEventId: string, lastTimestamp: string): Promise<void> {
1064
- await this.initialize();
1065
-
1066
- sqliteRun(
1067
- this.db,
1068
- `INSERT OR REPLACE INTO sync_positions (target_name, last_event_id, last_timestamp, updated_at)
1069
- VALUES (?, ?, ?, datetime('now'))`,
1070
- [targetName, lastEventId, lastTimestamp]
1071
- );
1072
- }
1073
-
1074
- /**
1075
- * Get config value for endless mode
1076
- */
1077
- async getEndlessConfig(key: string): Promise<unknown | null> {
1078
- await this.initialize();
1079
-
1080
- const row = sqliteGet<{ value: string }>(
1081
- this.db,
1082
- `SELECT value FROM endless_config WHERE key = ?`,
1083
- [key]
1084
- );
1085
-
1086
- if (!row) return null;
1087
- return JSON.parse(row.value);
1088
- }
1089
-
1090
- /**
1091
- * Set config value for endless mode
1092
- */
1093
- async setEndlessConfig(key: string, value: unknown): Promise<void> {
1094
- await this.initialize();
1095
-
1096
- sqliteRun(
1097
- this.db,
1098
- `INSERT OR REPLACE INTO endless_config (key, value, updated_at)
1099
- VALUES (?, ?, datetime('now'))`,
1100
- [key, JSON.stringify(value)]
1101
- );
1102
- }
1103
-
1104
- /**
1105
- * Increment access count for events
1106
- */
1107
- async incrementAccessCount(eventIds: string[]): Promise<void> {
1108
- if (eventIds.length === 0 || this.readOnly) return;
1109
-
1110
- await this.initialize();
1111
-
1112
- const placeholders = eventIds.map(() => '?').join(',');
1113
- const currentTime = toSQLiteTimestamp(new Date());
1114
-
1115
- sqliteRun(
1116
- this.db,
1117
- `UPDATE events
1118
- SET access_count = access_count + 1,
1119
- last_accessed_at = ?
1120
- WHERE id IN (${placeholders})`,
1121
- [currentTime, ...eventIds]
1122
- );
1123
- }
1124
-
1125
- /**
1126
- * Get most accessed memories (falls back to recent events if none accessed)
1127
- */
1128
- async getMostAccessed(limit: number = 10): Promise<MemoryEvent[]> {
1129
- await this.initialize();
1130
-
1131
- // First try events with access_count > 0
1132
- let rows = sqliteAll<Record<string, unknown>>(
1133
- this.db,
1134
- `SELECT * FROM events
1135
- WHERE access_count > 0
1136
- ORDER BY access_count DESC, last_accessed_at DESC
1137
- LIMIT ?`,
1138
- [limit]
1139
- );
1140
-
1141
- // Fallback: if no accessed events, show recent events
1142
- if (rows.length === 0) {
1143
- rows = sqliteAll<Record<string, unknown>>(
1144
- this.db,
1145
- `SELECT * FROM events
1146
- ORDER BY timestamp DESC
1147
- LIMIT ?`,
1148
- [limit]
1149
- );
1150
- }
1151
-
1152
- return rows.map(row => this.rowToEvent(row));
1153
- }
1154
-
1155
- /**
1156
- * Record a memory retrieval for helpfulness tracking
1157
- */
1158
- async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
1159
- if (this.readOnly) return;
1160
- await this.initialize();
1161
-
1162
- const id = randomUUID();
1163
- sqliteRun(
1164
- this.db,
1165
- `INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
1166
- VALUES (?, ?, ?, ?, ?, datetime('now'))`,
1167
- [id, eventId, sessionId, score, query.slice(0, 100)]
1168
- );
1169
- }
1170
-
1171
- /**
1172
- * Get session IDs that have unevaluated retrievals (measured_at IS NULL).
1173
- * Excludes the current session. Used to backfill sessions that ended without Stop hook.
1174
- */
1175
- async getUnevaluatedSessions(currentSessionId: string, limit = 5): Promise<string[]> {
1176
- await this.initialize();
1177
- const rows = sqliteAll<{ session_id: string }>(
1178
- this.db,
1179
- `SELECT DISTINCT session_id FROM memory_helpfulness
1180
- WHERE measured_at IS NULL AND session_id != ?
1181
- ORDER BY created_at DESC LIMIT ?`,
1182
- [currentSessionId, limit]
1183
- );
1184
- return rows.map((r) => r.session_id);
1185
- }
1186
-
1187
- /**
1188
- * Evaluate helpfulness for all retrievals in a session
1189
- * Called at session end - uses behavioral signals to compute score
1190
- */
1191
- async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
1192
- if (this.readOnly) return;
1193
- await this.initialize();
1194
-
1195
- // Get all retrieval records for this session
1196
- const retrievals = sqliteAll<Record<string, unknown>>(
1197
- this.db,
1198
- `SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
1199
- [sessionId]
1200
- );
1201
-
1202
- if (retrievals.length === 0) return;
1203
-
1204
- // Get session events to analyze behavior after retrieval
1205
- const sessionEvents = sqliteAll<Record<string, unknown>>(
1206
- this.db,
1207
- `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
1208
- [sessionId]
1209
- );
1210
-
1211
- const promptEvents = sessionEvents.filter((e: any) => e.event_type === 'user_prompt');
1212
- const toolEvents = sessionEvents.filter((e: any) => e.event_type === 'tool_observation');
1213
-
1214
- // Count successful vs failed tools
1215
- let toolSuccessCount = 0;
1216
- let toolTotalCount = toolEvents.length;
1217
- for (const t of toolEvents) {
1218
- try {
1219
- const content = JSON.parse(t.content as string);
1220
- if (content.success !== false) toolSuccessCount++;
1221
- } catch {
1222
- toolSuccessCount++; // Assume success if can't parse
1223
- }
1224
- }
1225
- const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
1226
-
1227
- for (const retrieval of retrievals) {
1228
- const retrievalTime = retrieval.created_at as string;
1229
-
1230
- // 1. Session continued after retrieval?
1231
- const eventsAfter = sessionEvents.filter((e: any) => e.timestamp > retrievalTime);
1232
- const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
1233
-
1234
- // 2. How many prompts came after?
1235
- const promptsAfter = promptEvents.filter((e: any) => e.timestamp > retrievalTime);
1236
- const promptCountAfter = promptsAfter.length;
1237
-
1238
- // 3. Was a similar query asked again? (simple word overlap check)
1239
- const queryWords = new Set((retrieval.query_preview as string || '').toLowerCase().split(/\s+/).filter(w => w.length > 2));
1240
- let wasReasked = 0;
1241
- for (const p of promptsAfter) {
1242
- const pWords = new Set((p.content as string).toLowerCase().split(/\s+/).filter((w: string) => w.length > 2));
1243
- let overlap = 0;
1244
- for (const w of queryWords) {
1245
- if (pWords.has(w)) overlap++;
1246
- }
1247
- if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
1248
- wasReasked = 1;
1249
- break;
1250
- }
1251
- }
1252
-
1253
- // Calculate helpfulness score
1254
- // Weights tuned for shopping-assistant-like corpora where sessions
1255
- // continue on the same topic (was_reasked was over-penalising normal conversation flow)
1256
- const retrievalScore = retrieval.retrieval_score as number || 0;
1257
- // More prompts after retrieval = memory was actually useful to the conversation
1258
- const promptNorm = Math.min(promptCountAfter / 2, 1.0);
1259
- const helpfulnessScore = (
1260
- 0.40 * Math.min(retrievalScore, 1.0) +
1261
- 0.30 * promptNorm +
1262
- 0.20 * toolSuccessRatio +
1263
- 0.10 * (sessionContinued ? 1.0 : 0.0)
1264
- );
1265
-
1266
- sqliteRun(
1267
- this.db,
1268
- `UPDATE memory_helpfulness
1269
- SET session_continued = ?, prompt_count_after = ?,
1270
- tool_success_count = ?, tool_total_count = ?,
1271
- was_reasked = ?, helpfulness_score = ?,
1272
- measured_at = datetime('now')
1273
- WHERE id = ?`,
1274
- [sessionContinued, promptCountAfter, toolSuccessCount, toolTotalCount,
1275
- wasReasked, helpfulnessScore, retrieval.id]
1276
- );
1277
- }
1278
- }
1279
-
1280
- /**
1281
- * Get most helpful memories ranked by helpfulness score
1282
- */
1283
- async getHelpfulMemories(limit: number = 10): Promise<Array<{
1284
- eventId: string;
1285
- summary: string;
1286
- helpfulnessScore: number;
1287
- accessCount: number;
1288
- evaluationCount: number;
1289
- }>> {
1290
- await this.initialize();
1291
-
1292
- const rows = sqliteAll<Record<string, unknown>>(
1293
- this.db,
1294
- `SELECT
1295
- mh.event_id,
1296
- AVG(mh.helpfulness_score) as avg_score,
1297
- COUNT(*) as eval_count,
1298
- e.content,
1299
- e.access_count
1300
- FROM memory_helpfulness mh
1301
- JOIN events e ON e.id = mh.event_id
1302
- WHERE mh.measured_at IS NOT NULL
1303
- GROUP BY mh.event_id
1304
- ORDER BY avg_score DESC
1305
- LIMIT ?`,
1306
- [limit]
1307
- );
1308
-
1309
- return rows.map(r => ({
1310
- eventId: r.event_id as string,
1311
- summary: (r.content as string).substring(0, 200) + ((r.content as string).length > 200 ? '...' : ''),
1312
- helpfulnessScore: Math.round((r.avg_score as number) * 100) / 100,
1313
- accessCount: (r.access_count as number) || 0,
1314
- evaluationCount: r.eval_count as number
1315
- }));
1316
- }
1317
-
1318
- /**
1319
- * Get helpfulness statistics for dashboard
1320
- */
1321
- async getHelpfulnessStats(): Promise<{
1322
- avgScore: number;
1323
- totalEvaluated: number;
1324
- totalRetrievals: number;
1325
- helpful: number;
1326
- neutral: number;
1327
- unhelpful: number;
1328
- }> {
1329
- await this.initialize();
1330
-
1331
- const stats = sqliteGet<Record<string, unknown>>(
1332
- this.db,
1333
- `SELECT
1334
- AVG(helpfulness_score) as avg_score,
1335
- COUNT(*) as total_evaluated,
1336
- SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
1337
- SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
1338
- SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
1339
- FROM memory_helpfulness
1340
- WHERE measured_at IS NOT NULL`
1341
- );
1342
-
1343
- const totalRow = sqliteGet<Record<string, unknown>>(
1344
- this.db,
1345
- `SELECT COUNT(*) as total FROM memory_helpfulness`
1346
- );
1347
-
1348
- return {
1349
- avgScore: Math.round(((stats?.avg_score as number) || 0) * 100) / 100,
1350
- totalEvaluated: (stats?.total_evaluated as number) || 0,
1351
- totalRetrievals: (totalRow?.total as number) || 0,
1352
- helpful: (stats?.helpful as number) || 0,
1353
- neutral: (stats?.neutral as number) || 0,
1354
- unhelpful: (stats?.unhelpful as number) || 0
1355
- };
1356
- }
1357
-
1358
- /**
1359
- * Fast keyword search using FTS5
1360
- * Returns events matching the search query, ranked by relevance
1361
- */
1362
- async keywordSearch(query: string, limit: number = 10): Promise<Array<{event: MemoryEvent; rank: number}>> {
1363
- await this.initialize();
1364
-
1365
- // Escape special FTS5 characters and prepare search terms
1366
- const searchTerms = query
1367
- .replace(/['"(){}[\]^~*?:\\/-]/g, ' ') // Remove special chars
1368
- .split(/\s+/)
1369
- .filter(term => term.length > 1) // Filter short terms
1370
- .map(term => `"${term}"*`) // Prefix matching
1371
- .join(' OR ');
1372
-
1373
- if (!searchTerms) {
1374
- return [];
1375
- }
1376
-
1377
- try {
1378
- const rows = sqliteAll<Record<string, unknown>>(
1379
- this.db,
1380
- `SELECT e.*, fts.rank
1381
- FROM events_fts fts
1382
- JOIN events e ON e.id = fts.event_id
1383
- WHERE events_fts MATCH ?
1384
- ORDER BY fts.rank
1385
- LIMIT ?`,
1386
- [searchTerms, limit]
1387
- );
1388
-
1389
- return rows.map(row => ({
1390
- event: this.rowToEvent(row),
1391
- rank: row.rank as number
1392
- }));
1393
- } catch (error: any) {
1394
- // FTS table might not exist yet (old database)
1395
- // Fallback to LIKE search
1396
- const likePattern = `%${query}%`;
1397
- const rows = sqliteAll<Record<string, unknown>>(
1398
- this.db,
1399
- `SELECT *, 0 as rank FROM events
1400
- WHERE content LIKE ?
1401
- ORDER BY timestamp DESC
1402
- LIMIT ?`,
1403
- [likePattern, limit]
1404
- );
1405
-
1406
- return rows.map(row => ({
1407
- event: this.rowToEvent(row),
1408
- rank: 0
1409
- }));
1410
- }
1411
- }
1412
-
1413
- /**
1414
- * Rebuild FTS index from existing events
1415
- * Call this once after upgrading to FTS5
1416
- */
1417
- async rebuildFtsIndex(): Promise<number> {
1418
- await this.initialize();
1419
-
1420
- // Get count of events to index
1421
- const countRow = sqliteGet<{count: number}>(this.db, 'SELECT COUNT(*) as count FROM events', []);
1422
- const totalEvents = countRow?.count ?? 0;
1423
-
1424
- // Clear and rebuild FTS index. Recreate the virtual table instead of
1425
- // issuing DELETE against it: older migrated FTS5 tables/triggers can fail
1426
- // with `no such column: T.event_id` when processing synthetic deletes.
1427
- sqliteExec(this.db, `
1428
- DROP TRIGGER IF EXISTS events_fts_insert;
1429
- DROP TRIGGER IF EXISTS events_fts_delete;
1430
- DROP TRIGGER IF EXISTS events_fts_update;
1431
- DROP TABLE IF EXISTS events_fts;
1432
-
1433
- CREATE VIRTUAL TABLE events_fts USING fts5(
1434
- content,
1435
- event_id UNINDEXED,
1436
- tokenize='porter unicode61'
1437
- );
1438
-
1439
- INSERT INTO events_fts(rowid, content, event_id)
1440
- SELECT rowid, content, id FROM events;
1441
-
1442
- CREATE TRIGGER events_fts_insert AFTER INSERT ON events BEGIN
1443
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1444
- END;
1445
-
1446
- CREATE TRIGGER events_fts_delete AFTER DELETE ON events BEGIN
1447
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1448
- END;
1449
-
1450
- CREATE TRIGGER events_fts_update AFTER UPDATE ON events BEGIN
1451
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1452
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1453
- END;
1454
- `);
1455
-
1456
- return totalEvents;
1457
- }
1458
-
1459
- /**
1460
- * Get database instance for direct access
1461
- */
1462
- getDatabase(): SQLiteDatabase {
1463
- return this.db;
1464
- }
1465
-
1466
-
1467
- async recordRetrievalTrace(input: {
1468
- sessionId?: string;
1469
- projectHash?: string;
1470
- queryText: string;
1471
- strategy?: string;
1472
- candidateEventIds: string[];
1473
- selectedEventIds: string[];
1474
- candidateDetails?: Array<{
1475
- eventId: string;
1476
- score: number;
1477
- semanticScore?: number;
1478
- lexicalScore?: number;
1479
- recencyScore?: number;
1480
- }>;
1481
- selectedDetails?: Array<{
1482
- eventId: string;
1483
- score: number;
1484
- semanticScore?: number;
1485
- lexicalScore?: number;
1486
- recencyScore?: number;
1487
- }>;
1488
- confidence?: string;
1489
- fallbackTrace?: string[];
1490
- }): Promise<void> {
1491
- await this.initialize();
1492
-
1493
- const traceId = randomUUID();
1494
- sqliteRun(
1495
- this.db,
1496
- `INSERT INTO retrieval_traces (
1497
- trace_id, session_id, project_hash, query_text, strategy,
1498
- candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
1499
- candidate_count, selected_count, confidence, fallback_trace
1500
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1501
- [
1502
- traceId,
1503
- input.sessionId || null,
1504
- input.projectHash || null,
1505
- input.queryText,
1506
- input.strategy || null,
1507
- JSON.stringify(input.candidateEventIds || []),
1508
- JSON.stringify(input.selectedEventIds || []),
1509
- JSON.stringify(input.candidateDetails || []),
1510
- JSON.stringify(input.selectedDetails || []),
1511
- (input.candidateEventIds || []).length,
1512
- (input.selectedEventIds || []).length,
1513
- input.confidence || null,
1514
- JSON.stringify(input.fallbackTrace || [])
1515
- ]
1516
- );
1517
- }
1518
-
1519
- async getRecentRetrievalTraces(limit: number = 50): Promise<Array<{
1520
- traceId: string;
1521
- sessionId?: string;
1522
- projectHash?: string;
1523
- queryText: string;
1524
- strategy?: string;
1525
- candidateEventIds: string[];
1526
- selectedEventIds: string[];
1527
- candidateDetails: Array<{
1528
- eventId: string;
1529
- score: number;
1530
- semanticScore?: number;
1531
- lexicalScore?: number;
1532
- recencyScore?: number;
1533
- }>;
1534
- selectedDetails: Array<{
1535
- eventId: string;
1536
- score: number;
1537
- semanticScore?: number;
1538
- lexicalScore?: number;
1539
- recencyScore?: number;
1540
- }>;
1541
- candidateCount: number;
1542
- selectedCount: number;
1543
- confidence?: string;
1544
- fallbackTrace: string[];
1545
- createdAt: Date;
1546
- }>> {
1547
- await this.initialize();
1548
-
1549
- try {
1550
- const rows = sqliteAll<Record<string, unknown>>(
1551
- this.db,
1552
- `SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
1553
- [limit]
1554
- );
1555
-
1556
- return rows.map((row) => ({
1557
- traceId: row.trace_id as string,
1558
- sessionId: (row.session_id as string) || undefined,
1559
- projectHash: (row.project_hash as string) || undefined,
1560
- queryText: row.query_text as string,
1561
- strategy: (row.strategy as string) || undefined,
1562
- candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids as string) : [],
1563
- selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids as string) : [],
1564
- candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json as string) : [],
1565
- selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json as string) : [],
1566
- candidateCount: Number(row.candidate_count || 0),
1567
- selectedCount: Number(row.selected_count || 0),
1568
- confidence: (row.confidence as string) || undefined,
1569
- fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace as string) : [],
1570
- createdAt: toDateFromSQLite(row.created_at),
1571
- }));
1572
- } catch (err: any) {
1573
- if (err?.message?.includes('no such table')) return [];
1574
- throw err;
1575
- }
1576
- }
1577
-
1578
- async getRetrievalTraceStats(): Promise<{
1579
- totalQueries: number;
1580
- avgCandidateCount: number;
1581
- avgSelectedCount: number;
1582
- selectionRate: number;
1583
- }> {
1584
- await this.initialize();
1585
-
1586
- try {
1587
- const row = sqliteGet<Record<string, unknown>>(
1588
- this.db,
1589
- `SELECT
1590
- COUNT(*) as total_queries,
1591
- AVG(candidate_count) as avg_candidate_count,
1592
- AVG(selected_count) as avg_selected_count,
1593
- CASE
1594
- WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
1595
- ELSE 0
1596
- END as selection_rate
1597
- FROM retrieval_traces`,
1598
- []
1599
- );
1600
-
1601
- return {
1602
- totalQueries: Number(row?.total_queries || 0),
1603
- avgCandidateCount: Number(row?.avg_candidate_count || 0),
1604
- avgSelectedCount: Number(row?.avg_selected_count || 0),
1605
- selectionRate: Number(row?.selection_rate || 0),
1606
- };
1607
- } catch (err: any) {
1608
- if (err?.message?.includes('no such table')) {
1609
- return { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 };
1610
- }
1611
- throw err;
1612
- }
1613
- }
1614
-
1615
- /**
1616
- * Close database connection
1617
- */
1618
- async close(): Promise<void> {
1619
- sqliteClose(this.db);
1620
- }
1621
-
1622
- /**
1623
- * Get events grouped by turn_id for a session
1624
- * Returns turns ordered by first event timestamp (newest first)
1625
- */
1626
- async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
1627
- turnId: string;
1628
- events: MemoryEvent[];
1629
- startedAt: Date;
1630
- promptPreview: string;
1631
- eventCount: number;
1632
- toolCount: number;
1633
- hasResponse: boolean;
1634
- }>> {
1635
- await this.initialize();
1636
-
1637
- const limit = options?.limit || 20;
1638
- const offset = options?.offset || 0;
1639
-
1640
- // Get distinct turn_ids for this session, ordered by first event timestamp
1641
- const turnRows = sqliteAll<{ turn_id: string; min_ts: string }>(
1642
- this.db,
1643
- `SELECT turn_id, MIN(timestamp) as min_ts
1644
- FROM events
1645
- WHERE session_id = ? AND turn_id IS NOT NULL
1646
- GROUP BY turn_id
1647
- ORDER BY min_ts DESC
1648
- LIMIT ? OFFSET ?`,
1649
- [sessionId, limit, offset]
1650
- );
1651
-
1652
- const turns: Array<{
1653
- turnId: string;
1654
- events: MemoryEvent[];
1655
- startedAt: Date;
1656
- promptPreview: string;
1657
- eventCount: number;
1658
- toolCount: number;
1659
- hasResponse: boolean;
1660
- }> = [];
1661
-
1662
- for (const turnRow of turnRows) {
1663
- const events = await this.getEventsByTurn(turnRow.turn_id);
1664
-
1665
- const promptEvent = events.find(e => e.eventType === 'user_prompt');
1666
- const toolEvents = events.filter(e => e.eventType === 'tool_observation');
1667
- const hasResponse = events.some(e => e.eventType === 'agent_response');
1668
-
1669
- turns.push({
1670
- turnId: turnRow.turn_id,
1671
- events,
1672
- startedAt: toDateFromSQLite(turnRow.min_ts),
1673
- promptPreview: promptEvent
1674
- ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? '...' : '')
1675
- : '(no prompt)',
1676
- eventCount: events.length,
1677
- toolCount: toolEvents.length,
1678
- hasResponse
1679
- });
1680
- }
1681
-
1682
- return turns;
1683
- }
1684
-
1685
- /**
1686
- * Get all events for a specific turn_id
1687
- */
1688
- async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
1689
- await this.initialize();
1690
-
1691
- const rows = sqliteAll<Record<string, unknown>>(
1692
- this.db,
1693
- `SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
1694
- [turnId]
1695
- );
1696
-
1697
- return rows.map(this.rowToEvent);
1698
- }
1699
-
1700
- /**
1701
- * Count total turns for a session
1702
- */
1703
- async countSessionTurns(sessionId: string): Promise<number> {
1704
- await this.initialize();
1705
-
1706
- const row = sqliteGet<{ count: number }>(
1707
- this.db,
1708
- `SELECT COUNT(DISTINCT turn_id) as count
1709
- FROM events
1710
- WHERE session_id = ? AND turn_id IS NOT NULL`,
1711
- [sessionId]
1712
- );
1713
-
1714
- return row?.count || 0;
1715
- }
1716
-
1717
- /**
1718
- * Migrate existing events: backfill turn_id for events that have turnId in metadata
1719
- * but no turn_id column value (for events stored before this migration)
1720
- */
1721
- async backfillTurnIds(): Promise<number> {
1722
- await this.initialize();
1723
-
1724
- // Find events with turnId in metadata JSON but no turn_id column value
1725
- const rows = sqliteAll<{ id: string; metadata: string }>(
1726
- this.db,
1727
- `SELECT id, metadata FROM events
1728
- WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
1729
- );
1730
-
1731
- let updated = 0;
1732
- for (const row of rows) {
1733
- try {
1734
- const metadata = JSON.parse(row.metadata);
1735
- if (metadata.turnId) {
1736
- sqliteRun(
1737
- this.db,
1738
- `UPDATE events SET turn_id = ? WHERE id = ?`,
1739
- [metadata.turnId, row.id]
1740
- );
1741
- updated++;
1742
- }
1743
- } catch {
1744
- // Skip rows with invalid JSON
1745
- }
1746
- }
1747
-
1748
- return updated;
1749
- }
1750
-
1751
- /**
1752
- * Delete all events for a session (for force reimport)
1753
- */
1754
- async deleteSessionEvents(sessionId: string): Promise<number> {
1755
- await this.initialize();
1756
-
1757
- // Get event IDs first for cascading deletes
1758
- const events = sqliteAll<{ id: string }>(
1759
- this.db,
1760
- `SELECT id FROM events WHERE session_id = ?`,
1761
- [sessionId]
1762
- );
1763
-
1764
- if (events.length === 0) return 0;
1765
-
1766
- const eventIds = events.map(e => e.id);
1767
- const placeholders = eventIds.map(() => '?').join(',');
1768
-
1769
- // Drop FTS triggers to prevent SQLITE_CORRUPT_VTAB during bulk delete
1770
- const ftsTriggersDropped: string[] = [];
1771
- for (const triggerName of ['events_fts_delete', 'events_fts_update', 'events_fts_insert']) {
1772
- try {
1773
- sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
1774
- ftsTriggersDropped.push(triggerName);
1775
- } catch {
1776
- // Trigger may not exist
1777
- }
1778
- }
1779
-
1780
- // Delete from related tables first (some may not exist depending on DB version)
1781
- for (const table of ['event_dedup', 'memory_levels', 'embedding_queue', 'embedding_outbox', 'vector_outbox']) {
1782
- try {
1783
- sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
1784
- } catch {
1785
- // Table may not exist
1786
- }
1787
- }
1788
-
1789
- // Delete events
1790
- const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
1791
-
1792
- // Rebuild FTS index if we dropped triggers
1793
- if (ftsTriggersDropped.length > 0) {
1794
- try {
1795
- // Rebuild FTS from remaining events
1796
- sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
1797
-
1798
- // Recreate triggers
1799
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
1800
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1801
- END`);
1802
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
1803
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1804
- END`);
1805
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
1806
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1807
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1808
- END`);
1809
- } catch {
1810
- // FTS rebuild failed - non-critical, will be rebuilt on next initialize
1811
- }
1812
- }
1813
-
1814
- return result.changes || 0;
1815
- }
1816
-
1817
- /**
1818
- * Convert database row to MemoryEvent
1819
- */
1820
- private rowToEvent(row: Record<string, unknown>): MemoryEvent {
1821
- const event: any = {
1822
- id: row.id as string,
1823
- eventType: row.event_type as 'user_prompt' | 'agent_response' | 'session_summary',
1824
- sessionId: row.session_id as string,
1825
- timestamp: toDateFromSQLite(row.timestamp),
1826
- content: row.content as string,
1827
- canonicalKey: row.canonical_key as string,
1828
- dedupeKey: row.dedupe_key as string,
1829
- metadata: row.metadata ? JSON.parse(row.metadata as string) : undefined
1830
- };
1831
-
1832
- // Include access tracking fields if present
1833
- if (row.access_count !== undefined) {
1834
- event.access_count = row.access_count;
1835
- }
1836
- if (row.last_accessed_at !== undefined) {
1837
- event.last_accessed_at = row.last_accessed_at;
1838
- }
1839
- // Include turn_id if present
1840
- if (row.turn_id !== undefined && row.turn_id !== null) {
1841
- event.turn_id = row.turn_id;
1842
- }
1843
-
1844
- return event;
1845
- }
1846
- }