claude-memory-layer 1.0.31 → 1.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (313) hide show
  1. package/README.md +9 -2
  2. package/dist/cli/index.js +1 -1
  3. package/package.json +11 -2
  4. package/scripts/postinstall-embedding-backend.cjs +16 -12
  5. package/AGENTS.md +0 -71
  6. package/CLAUDE.md +0 -30
  7. package/HANDOFF.md +0 -92
  8. package/Memo.txt +0 -558
  9. package/benchmarks/replay/anonymized-real-sessions.json +0 -48
  10. package/config/kpi-thresholds.json +0 -7
  11. package/context.md +0 -636
  12. package/docs/ARCHITECTURE_COMPARISON_AND_RECOMMENDATIONS.md +0 -627
  13. package/docs/HERMES_MEMORY_INGESTION_ANALYSIS.md +0 -440
  14. package/docs/MCP_MEMORY_SERVICE_COMPARATIVE_REVIEW.md +0 -271
  15. package/docs/MEMORY_USEFULNESS_AUDIT.md +0 -371
  16. package/docs/MEMORY_USEFULNESS_AUDIT_RAW.json +0 -80
  17. package/docs/MEMSEARCH_PROJECT_STRUCTURE_ANALYSIS.md +0 -333
  18. package/docs/MEMU_ADOPTION.md +0 -40
  19. package/docs/OPERATIONS.md +0 -18
  20. package/docs/PRODUCT_VALIDATION_MATRIX.md +0 -82
  21. package/docs/PROJECT_STRUCTURE_ANALYSIS.md +0 -421
  22. package/docs/REFACTORING_MILESTONES_AND_ISSUES.md +0 -501
  23. package/docs/REFACTORING_PLAN_THIN_CORE.md +0 -414
  24. package/docs/REFERENCE_PROJECT_ANALYSES.md +0 -25
  25. package/docs/SUPERLOCALMEMORY_PROJECT_STRUCTURE_ANALYSIS.md +0 -452
  26. package/docs/TARGET_ARCHITECTURE_AND_FOLDER_STRUCTURE.md +0 -446
  27. package/docs/architecture/comparison-index.md +0 -47
  28. package/docs/reports/codex-real-data-validation-20260505T040447Z.md +0 -46
  29. package/plan.md +0 -1642
  30. package/scripts/build.ts +0 -159
  31. package/scripts/bump-patch-version.sh +0 -18
  32. package/scripts/delete-unknown-projects.js +0 -154
  33. package/scripts/fix-sync-gap.js +0 -32
  34. package/scripts/generate-session-qrels.ts +0 -126
  35. package/scripts/heartbeat-memory-orchestrator.sh +0 -28
  36. package/scripts/replay-retrieval-benchmark.ts +0 -69
  37. package/scripts/report-sync-gap.js +0 -26
  38. package/scripts/review-queue-auto-resolve.js +0 -21
  39. package/scripts/sync-gap-auto-heal.sh +0 -17
  40. package/spec.md +0 -624
  41. package/specs/20260207-dashboard-upgrade/context.md +0 -38
  42. package/specs/20260207-dashboard-upgrade/spec.md +0 -96
  43. package/specs/citations-system/context.md +0 -243
  44. package/specs/citations-system/plan.md +0 -495
  45. package/specs/citations-system/spec.md +0 -371
  46. package/specs/endless-mode/context.md +0 -305
  47. package/specs/endless-mode/plan.md +0 -620
  48. package/specs/endless-mode/spec.md +0 -455
  49. package/specs/entity-edge-model/context.md +0 -401
  50. package/specs/entity-edge-model/plan.md +0 -459
  51. package/specs/entity-edge-model/spec.md +0 -391
  52. package/specs/evidence-aligner-v2/context.md +0 -401
  53. package/specs/evidence-aligner-v2/plan.md +0 -303
  54. package/specs/evidence-aligner-v2/spec.md +0 -312
  55. package/specs/mcp-desktop-integration/context.md +0 -278
  56. package/specs/mcp-desktop-integration/plan.md +0 -550
  57. package/specs/mcp-desktop-integration/spec.md +0 -494
  58. package/specs/memory-utilization-improvements/context.md +0 -145
  59. package/specs/memory-utilization-improvements/plan.md +0 -361
  60. package/specs/memory-utilization-improvements/spec.md +0 -361
  61. package/specs/post-tool-use-hook/context.md +0 -319
  62. package/specs/post-tool-use-hook/plan.md +0 -469
  63. package/specs/post-tool-use-hook/spec.md +0 -364
  64. package/specs/private-tags/context.md +0 -288
  65. package/specs/private-tags/plan.md +0 -412
  66. package/specs/private-tags/spec.md +0 -345
  67. package/specs/progressive-disclosure/context.md +0 -346
  68. package/specs/progressive-disclosure/plan.md +0 -663
  69. package/specs/progressive-disclosure/spec.md +0 -415
  70. package/specs/selective-tool-observation/context.md +0 -100
  71. package/specs/selective-tool-observation/plan.md +0 -158
  72. package/specs/selective-tool-observation/spec.md +0 -127
  73. package/specs/task-entity-system/context.md +0 -297
  74. package/specs/task-entity-system/plan.md +0 -301
  75. package/specs/task-entity-system/spec.md +0 -314
  76. package/specs/thin-core-refactor/context.md +0 -275
  77. package/specs/thin-core-refactor/plan.md +0 -536
  78. package/specs/thin-core-refactor/spec.md +0 -465
  79. package/specs/vector-outbox-v2/context.md +0 -470
  80. package/specs/vector-outbox-v2/plan.md +0 -562
  81. package/specs/vector-outbox-v2/spec.md +0 -466
  82. package/specs/web-viewer-ui/context.md +0 -384
  83. package/specs/web-viewer-ui/plan.md +0 -797
  84. package/specs/web-viewer-ui/spec.md +0 -516
  85. package/src/adapters/claude/capture/index.ts +0 -3
  86. package/src/adapters/claude/context/index.ts +0 -3
  87. package/src/adapters/claude/hooks/index.ts +0 -21
  88. package/src/adapters/claude/hooks/post-tool-use.ts +0 -239
  89. package/src/adapters/claude/hooks/prompt-injection-policy.ts +0 -104
  90. package/src/adapters/claude/hooks/semantic-daemon-client.ts +0 -209
  91. package/src/adapters/claude/hooks/semantic-daemon.ts +0 -283
  92. package/src/adapters/claude/hooks/session-end.ts +0 -59
  93. package/src/adapters/claude/hooks/session-start.ts +0 -73
  94. package/src/adapters/claude/hooks/stop.ts +0 -128
  95. package/src/adapters/claude/hooks/user-prompt-submit.ts +0 -361
  96. package/src/adapters/claude/index.ts +0 -4
  97. package/src/adapters/claude/transcript/index.ts +0 -4
  98. package/src/adapters/claude/transcript/transcript-reader.ts +0 -57
  99. package/src/adapters/claude/transcript/turn-reconstructor.ts +0 -65
  100. package/src/apps/cli/claude-settings-hooks.ts +0 -138
  101. package/src/apps/cli/codex-import-runner.ts +0 -125
  102. package/src/apps/cli/codex-validation-output.ts +0 -95
  103. package/src/apps/cli/hermes-import-runner.ts +0 -130
  104. package/src/apps/cli/hermes-validation-output.ts +0 -91
  105. package/src/apps/cli/index.ts +0 -1735
  106. package/src/apps/cli/mcp-install.ts +0 -106
  107. package/src/apps/cli/retrieval-disclosure-output.ts +0 -196
  108. package/src/apps/dashboard/assets/js/bootstrap.js +0 -244
  109. package/src/apps/dashboard/assets/js/chat.js +0 -373
  110. package/src/apps/dashboard/assets/js/disclosure.js +0 -232
  111. package/src/apps/dashboard/assets/js/modals.js +0 -298
  112. package/src/apps/dashboard/assets/js/overview.js +0 -655
  113. package/src/apps/dashboard/assets/js/state.js +0 -72
  114. package/src/apps/dashboard/assets/js/views.js +0 -468
  115. package/src/apps/dashboard/index.html +0 -543
  116. package/src/apps/dashboard/index.ts +0 -3
  117. package/src/apps/dashboard/style.css +0 -1750
  118. package/src/apps/index.ts +0 -5
  119. package/src/apps/server/api/chat.ts +0 -244
  120. package/src/apps/server/api/citations.ts +0 -105
  121. package/src/apps/server/api/events.ts +0 -137
  122. package/src/apps/server/api/health.ts +0 -53
  123. package/src/apps/server/api/index.ts +0 -26
  124. package/src/apps/server/api/projects.ts +0 -74
  125. package/src/apps/server/api/search.ts +0 -184
  126. package/src/apps/server/api/sessions.ts +0 -115
  127. package/src/apps/server/api/stats.ts +0 -723
  128. package/src/apps/server/api/turns.ts +0 -143
  129. package/src/apps/server/api/utils.ts +0 -65
  130. package/src/apps/server/index.ts +0 -111
  131. package/src/cli/index.ts +0 -3
  132. package/src/cli/retrieval-disclosure-output.ts +0 -2
  133. package/src/compat/index.ts +0 -5
  134. package/src/core/canonical-key.ts +0 -186
  135. package/src/core/citation-generator.ts +0 -63
  136. package/src/core/consolidated-store.ts +0 -356
  137. package/src/core/consolidation-worker.ts +0 -493
  138. package/src/core/context-formatter.ts +0 -276
  139. package/src/core/continuity-manager.ts +0 -341
  140. package/src/core/db-wrapper.ts +0 -64
  141. package/src/core/derive/fact-deriver.ts +0 -170
  142. package/src/core/derive/index.ts +0 -2
  143. package/src/core/derive/summary-deriver.ts +0 -76
  144. package/src/core/edge-repo.ts +0 -333
  145. package/src/core/embedder.ts +0 -4
  146. package/src/core/engine/embedding-maintenance-service.ts +0 -187
  147. package/src/core/engine/endless-memory-services.ts +0 -4
  148. package/src/core/engine/index.ts +0 -19
  149. package/src/core/engine/memory-engine-services.ts +0 -170
  150. package/src/core/engine/memory-ingest-service.ts +0 -317
  151. package/src/core/engine/memory-query-service.ts +0 -173
  152. package/src/core/engine/memory-runtime-service.ts +0 -162
  153. package/src/core/engine/memory-service-composition.ts +0 -231
  154. package/src/core/engine/retrieval-analytics-service.ts +0 -181
  155. package/src/core/engine/retrieval-disclosure-service.ts +0 -420
  156. package/src/core/engine/retrieval-orchestrator.ts +0 -377
  157. package/src/core/engine/retrieval-services.ts +0 -176
  158. package/src/core/engine/shared-memory-services.ts +0 -4
  159. package/src/core/entity-repo.ts +0 -349
  160. package/src/core/event-store.ts +0 -779
  161. package/src/core/evidence-aligner.ts +0 -635
  162. package/src/core/external-market-context.ts +0 -582
  163. package/src/core/graduation-worker.ts +0 -171
  164. package/src/core/graduation.ts +0 -377
  165. package/src/core/index.ts +0 -64
  166. package/src/core/ingest-interceptor.ts +0 -80
  167. package/src/core/markdown-mirror.ts +0 -70
  168. package/src/core/matcher.ts +0 -208
  169. package/src/core/md-mirror.ts +0 -92
  170. package/src/core/metadata-extractor.ts +0 -203
  171. package/src/core/model/memory-fact.ts +0 -30
  172. package/src/core/model/memory-rule.ts +0 -14
  173. package/src/core/model/memory-summary.ts +0 -21
  174. package/src/core/model/raw-event.ts +0 -28
  175. package/src/core/model/retrieval-result.ts +0 -35
  176. package/src/core/mongo-sync-config.ts +0 -165
  177. package/src/core/mongo-sync-worker.ts +0 -381
  178. package/src/core/privacy/filter.ts +0 -190
  179. package/src/core/privacy/index.ts +0 -20
  180. package/src/core/privacy/tag-parser.ts +0 -145
  181. package/src/core/product-validation-matrix.ts +0 -314
  182. package/src/core/progressive-retriever.ts +0 -414
  183. package/src/core/registry/project-path.ts +0 -54
  184. package/src/core/registry/session-registry.ts +0 -69
  185. package/src/core/replay-evaluator.ts +0 -625
  186. package/src/core/retrieval-benchmark.ts +0 -117
  187. package/src/core/retrieval-quality.ts +0 -109
  188. package/src/core/retriever.ts +0 -800
  189. package/src/core/session-qrels.ts +0 -360
  190. package/src/core/shared-event-store.ts +0 -114
  191. package/src/core/shared-promoter.ts +0 -249
  192. package/src/core/shared-store.ts +0 -289
  193. package/src/core/shared-vector-store.ts +0 -203
  194. package/src/core/sqlite-event-store.ts +0 -1846
  195. package/src/core/sqlite-wrapper.ts +0 -116
  196. package/src/core/sync-worker.ts +0 -228
  197. package/src/core/tag-taxonomy.ts +0 -51
  198. package/src/core/task/blocker-resolver.ts +0 -333
  199. package/src/core/task/index.ts +0 -9
  200. package/src/core/task/task-matcher.ts +0 -240
  201. package/src/core/task/task-projector.ts +0 -358
  202. package/src/core/task/task-resolver.ts +0 -421
  203. package/src/core/turn-state.ts +0 -207
  204. package/src/core/types.ts +0 -952
  205. package/src/core/vector-outbox.ts +0 -299
  206. package/src/core/vector-store.ts +0 -231
  207. package/src/core/vector-worker.ts +0 -521
  208. package/src/core/working-set-store.ts +0 -257
  209. package/src/extensions/endless-memory/endless-memory-services.ts +0 -350
  210. package/src/extensions/endless-memory/index.ts +0 -1
  211. package/src/extensions/index.ts +0 -5
  212. package/src/extensions/mcp/handlers.ts +0 -960
  213. package/src/extensions/mcp/index.ts +0 -48
  214. package/src/extensions/mcp/tools.ts +0 -252
  215. package/src/extensions/shared-memory/index.ts +0 -1
  216. package/src/extensions/shared-memory/shared-memory-services.ts +0 -211
  217. package/src/extensions/vector/embedder.ts +0 -234
  218. package/src/extensions/vector/index.ts +0 -1
  219. package/src/hooks/post-tool-use.ts +0 -9
  220. package/src/hooks/semantic-daemon-client.ts +0 -1
  221. package/src/hooks/semantic-daemon.ts +0 -11
  222. package/src/hooks/session-end.ts +0 -9
  223. package/src/hooks/session-start.ts +0 -9
  224. package/src/hooks/stop.ts +0 -9
  225. package/src/hooks/user-prompt-submit.ts +0 -9
  226. package/src/index.ts +0 -13
  227. package/src/mcp/handlers.ts +0 -2
  228. package/src/mcp/index.ts +0 -4
  229. package/src/mcp/tools.ts +0 -2
  230. package/src/server/api/chat.ts +0 -2
  231. package/src/server/api/citations.ts +0 -2
  232. package/src/server/api/events.ts +0 -2
  233. package/src/server/api/health.ts +0 -2
  234. package/src/server/api/index.ts +0 -2
  235. package/src/server/api/projects.ts +0 -2
  236. package/src/server/api/search.ts +0 -2
  237. package/src/server/api/sessions.ts +0 -2
  238. package/src/server/api/stats.ts +0 -2
  239. package/src/server/api/turns.ts +0 -2
  240. package/src/server/api/utils.ts +0 -2
  241. package/src/server/index.ts +0 -2
  242. package/src/services/bootstrap-organizer.ts +0 -463
  243. package/src/services/codex-session-history-importer.ts +0 -966
  244. package/src/services/hermes-session-history-importer.ts +0 -733
  245. package/src/services/memory-service-config.ts +0 -36
  246. package/src/services/memory-service-registry.ts +0 -150
  247. package/src/services/memory-service.ts +0 -688
  248. package/src/services/session-history-importer.ts +0 -629
  249. package/tests/README.md +0 -23
  250. package/tests/adapters/claude/claude-semantic-daemon-adapter.test.ts +0 -54
  251. package/tests/adapters/claude/claude-transcript-reconstructor.test.ts +0 -98
  252. package/tests/adapters/claude-hook-prompt-injection-policy.test.ts +0 -99
  253. package/tests/apps/app-layer-boundary.test.ts +0 -48
  254. package/tests/apps/claude-settings-hooks.test.ts +0 -107
  255. package/tests/apps/cli-disclosure-output.test.ts +0 -212
  256. package/tests/apps/codex-import-runner.test.ts +0 -99
  257. package/tests/apps/codex-validation-output.test.ts +0 -100
  258. package/tests/apps/hermes-import-runner.test.ts +0 -99
  259. package/tests/apps/mcp-install-command.test.ts +0 -59
  260. package/tests/apps/package-build-entrypoints.test.ts +0 -30
  261. package/tests/apps/postinstall-embedding-backend.test.ts +0 -185
  262. package/tests/apps/search-api-disclosure.test.ts +0 -162
  263. package/tests/apps/stats-api-lightweight.test.ts +0 -67
  264. package/tests/apps/ui-disclosure-output.test.ts +0 -140
  265. package/tests/core/bootstrap-organizer.test.ts +0 -111
  266. package/tests/core/canonical-key.test.ts +0 -101
  267. package/tests/core/codex-session-history-importer-validation.test.ts +0 -185
  268. package/tests/core/consolidation-worker.test.ts +0 -75
  269. package/tests/core/embedding-maintenance-service.test.ts +0 -282
  270. package/tests/core/evidence-aligner.test.ts +0 -152
  271. package/tests/core/external-market-context.test.ts +0 -209
  272. package/tests/core/fact-deriver.test.ts +0 -79
  273. package/tests/core/hermes-session-history-importer-validation.test.ts +0 -609
  274. package/tests/core/ingest-interceptor.test.ts +0 -38
  275. package/tests/core/markdown-mirror.test.ts +0 -85
  276. package/tests/core/matcher.test.ts +0 -112
  277. package/tests/core/md-mirror.test.ts +0 -50
  278. package/tests/core/memory-engine-services.test.ts +0 -240
  279. package/tests/core/memory-ingest-service.test.ts +0 -296
  280. package/tests/core/memory-query-service.test.ts +0 -129
  281. package/tests/core/memory-runtime-service.test.ts +0 -201
  282. package/tests/core/memory-service-composition.test.ts +0 -192
  283. package/tests/core/memory-service-config.test.ts +0 -41
  284. package/tests/core/memory-service-facade.test.ts +0 -30
  285. package/tests/core/memory-service-registry.test.ts +0 -206
  286. package/tests/core/product-validation-matrix.test.ts +0 -61
  287. package/tests/core/project-registry.test.ts +0 -78
  288. package/tests/core/replay-evaluator.test.ts +0 -181
  289. package/tests/core/retrieval-analytics-service.test.ts +0 -210
  290. package/tests/core/retrieval-benchmark.test.ts +0 -93
  291. package/tests/core/retrieval-disclosure-service.test.ts +0 -264
  292. package/tests/core/retrieval-orchestrator.test.ts +0 -403
  293. package/tests/core/retrieval-quality.test.ts +0 -31
  294. package/tests/core/retrieval-services.test.ts +0 -185
  295. package/tests/core/retriever-fallback-chain.test.ts +0 -223
  296. package/tests/core/retriever-strategy-scope.test.ts +0 -164
  297. package/tests/core/retriever.memu-adoption.test.ts +0 -122
  298. package/tests/core/session-history-importer-filter.test.ts +0 -78
  299. package/tests/core/session-qrels.test.ts +0 -250
  300. package/tests/core/sqlite-event-store-replication.test.ts +0 -127
  301. package/tests/core/summary-deriver.test.ts +0 -66
  302. package/tests/extensions/embedder-warning-suppression.test.ts +0 -84
  303. package/tests/extensions/endless-memory-extension-boundary.test.ts +0 -17
  304. package/tests/extensions/endless-memory-services.test.ts +0 -325
  305. package/tests/extensions/mcp-context-tools.test.ts +0 -905
  306. package/tests/extensions/mcp-extension-boundary.test.ts +0 -21
  307. package/tests/extensions/mcp-package-build.test.ts +0 -22
  308. package/tests/extensions/mcp-project-aware-tools.test.ts +0 -102
  309. package/tests/extensions/shared-memory-extension-boundary.test.ts +0 -24
  310. package/tests/extensions/shared-memory-services.test.ts +0 -309
  311. package/tests/extensions/vector-extension-boundary.test.ts +0 -21
  312. package/tsconfig.json +0 -24
  313. package/vitest.config.ts +0 -15
@@ -1,1846 +0,0 @@
1
- /**
2
- * SQLite-based EventStore implementation
3
- * Primary store for hooks - WAL mode enables concurrent access
4
- */
5
-
6
- import { randomUUID } from 'crypto';
7
- import {
8
- MemoryEvent,
9
- MemoryEventInput,
10
- Session,
11
- AppendResult,
12
- OutboxItem
13
- } from './types.js';
14
- import { makeCanonicalKey, makeDedupeKey } from './canonical-key.js';
15
- import {
16
- createSQLiteDatabase,
17
- sqliteRun,
18
- sqliteAll,
19
- sqliteGet,
20
- sqliteClose,
21
- sqliteExec,
22
- toDateFromSQLite,
23
- toSQLiteTimestamp,
24
- type SQLiteDatabase,
25
- type SQLiteOptions
26
- } from './sqlite-wrapper.js';
27
- import { MarkdownMirror } from './markdown-mirror.js';
28
-
29
- export interface SQLiteEventStoreOptions extends SQLiteOptions {
30
- markdownMirrorRoot?: string;
31
- }
32
-
33
- export class SQLiteEventStore {
34
- private db: SQLiteDatabase;
35
- private initialized = false;
36
- private readonly readOnly: boolean;
37
- private readonly markdownMirror: MarkdownMirror | null;
38
-
39
- constructor(dbPath: string, options?: SQLiteEventStoreOptions) {
40
- this.readOnly = options?.readonly ?? false;
41
- this.db = createSQLiteDatabase(dbPath, {
42
- readonly: this.readOnly,
43
- walMode: !this.readOnly
44
- });
45
- this.markdownMirror = this.readOnly || !options?.markdownMirrorRoot
46
- ? null
47
- : new MarkdownMirror(options.markdownMirrorRoot);
48
- }
49
-
50
- /**
51
- * Initialize database schema
52
- */
53
- async initialize(): Promise<void> {
54
- if (this.initialized) return;
55
-
56
- // In read-only mode, skip schema creation
57
- if (this.readOnly) {
58
- this.initialized = true;
59
- return;
60
- }
61
-
62
- // Create all tables in a single exec for efficiency
63
- sqliteExec(this.db, `
64
- -- L0 EventStore: Single Source of Truth (immutable, append-only)
65
- CREATE TABLE IF NOT EXISTS events (
66
- id TEXT PRIMARY KEY,
67
- event_type TEXT NOT NULL,
68
- session_id TEXT NOT NULL,
69
- timestamp TEXT NOT NULL,
70
- content TEXT NOT NULL,
71
- canonical_key TEXT NOT NULL,
72
- dedupe_key TEXT UNIQUE,
73
- metadata TEXT,
74
- access_count INTEGER DEFAULT 0,
75
- last_accessed_at TEXT
76
- );
77
-
78
- -- Dedup table for idempotency
79
- CREATE TABLE IF NOT EXISTS event_dedup (
80
- dedupe_key TEXT PRIMARY KEY,
81
- event_id TEXT NOT NULL,
82
- created_at TEXT DEFAULT (datetime('now'))
83
- );
84
-
85
- -- Session metadata
86
- CREATE TABLE IF NOT EXISTS sessions (
87
- id TEXT PRIMARY KEY,
88
- started_at TEXT NOT NULL,
89
- ended_at TEXT,
90
- project_path TEXT,
91
- summary TEXT,
92
- tags TEXT
93
- );
94
-
95
- -- Insights (derived data, rebuildable)
96
- CREATE TABLE IF NOT EXISTS insights (
97
- id TEXT PRIMARY KEY,
98
- insight_type TEXT NOT NULL,
99
- content TEXT NOT NULL,
100
- canonical_key TEXT NOT NULL,
101
- confidence REAL,
102
- source_events TEXT,
103
- created_at TEXT,
104
- last_updated TEXT
105
- );
106
-
107
- -- Embedding Outbox (Single-Writer Pattern)
108
- CREATE TABLE IF NOT EXISTS embedding_outbox (
109
- id TEXT PRIMARY KEY,
110
- event_id TEXT NOT NULL,
111
- content TEXT NOT NULL,
112
- status TEXT DEFAULT 'pending',
113
- retry_count INTEGER DEFAULT 0,
114
- created_at TEXT DEFAULT (datetime('now')),
115
- processed_at TEXT,
116
- error_message TEXT
117
- );
118
-
119
- -- Projection offset tracking
120
- CREATE TABLE IF NOT EXISTS projection_offsets (
121
- projection_name TEXT PRIMARY KEY,
122
- last_event_id TEXT,
123
- last_timestamp TEXT,
124
- updated_at TEXT DEFAULT (datetime('now'))
125
- );
126
-
127
- -- Memory level tracking
128
- CREATE TABLE IF NOT EXISTS memory_levels (
129
- event_id TEXT PRIMARY KEY,
130
- level TEXT NOT NULL DEFAULT 'L0',
131
- promoted_at TEXT DEFAULT (datetime('now'))
132
- );
133
-
134
- -- Entries (immutable memory units)
135
- CREATE TABLE IF NOT EXISTS entries (
136
- entry_id TEXT PRIMARY KEY,
137
- created_ts TEXT NOT NULL,
138
- entry_type TEXT NOT NULL,
139
- title TEXT NOT NULL,
140
- content_json TEXT NOT NULL,
141
- stage TEXT NOT NULL DEFAULT 'raw',
142
- status TEXT DEFAULT 'active',
143
- superseded_by TEXT,
144
- build_id TEXT,
145
- evidence_json TEXT,
146
- canonical_key TEXT,
147
- created_at TEXT DEFAULT (datetime('now'))
148
- );
149
-
150
- -- Entities (task/condition/artifact)
151
- CREATE TABLE IF NOT EXISTS entities (
152
- entity_id TEXT PRIMARY KEY,
153
- entity_type TEXT NOT NULL,
154
- canonical_key TEXT NOT NULL,
155
- title TEXT NOT NULL,
156
- stage TEXT NOT NULL DEFAULT 'raw',
157
- status TEXT NOT NULL DEFAULT 'active',
158
- current_json TEXT NOT NULL,
159
- title_norm TEXT,
160
- search_text TEXT,
161
- created_at TEXT DEFAULT (datetime('now')),
162
- updated_at TEXT DEFAULT (datetime('now'))
163
- );
164
-
165
- -- Entity aliases for canonical key lookup
166
- CREATE TABLE IF NOT EXISTS entity_aliases (
167
- entity_type TEXT NOT NULL,
168
- canonical_key TEXT NOT NULL,
169
- entity_id TEXT NOT NULL,
170
- is_primary INTEGER DEFAULT 0,
171
- created_at TEXT DEFAULT (datetime('now')),
172
- PRIMARY KEY(entity_type, canonical_key)
173
- );
174
-
175
- -- Edges (relationships between entries/entities)
176
- CREATE TABLE IF NOT EXISTS edges (
177
- edge_id TEXT PRIMARY KEY,
178
- src_type TEXT NOT NULL,
179
- src_id TEXT NOT NULL,
180
- rel_type TEXT NOT NULL,
181
- dst_type TEXT NOT NULL,
182
- dst_id TEXT NOT NULL,
183
- meta_json TEXT,
184
- created_at TEXT DEFAULT (datetime('now'))
185
- );
186
-
187
- -- Vector Outbox V2 Table
188
- CREATE TABLE IF NOT EXISTS vector_outbox (
189
- job_id TEXT PRIMARY KEY,
190
- item_kind TEXT NOT NULL,
191
- item_id TEXT NOT NULL,
192
- embedding_version TEXT NOT NULL,
193
- status TEXT NOT NULL DEFAULT 'pending',
194
- retry_count INTEGER DEFAULT 0,
195
- error TEXT,
196
- created_at TEXT DEFAULT (datetime('now')),
197
- updated_at TEXT DEFAULT (datetime('now')),
198
- UNIQUE(item_kind, item_id, embedding_version)
199
- );
200
-
201
- -- Build Runs
202
- CREATE TABLE IF NOT EXISTS build_runs (
203
- build_id TEXT PRIMARY KEY,
204
- started_at TEXT NOT NULL,
205
- finished_at TEXT,
206
- extractor_model TEXT NOT NULL,
207
- extractor_prompt_hash TEXT NOT NULL,
208
- embedder_model TEXT NOT NULL,
209
- embedding_version TEXT NOT NULL,
210
- idris_version TEXT NOT NULL,
211
- schema_version TEXT NOT NULL,
212
- status TEXT NOT NULL DEFAULT 'running',
213
- error TEXT
214
- );
215
-
216
- -- Pipeline Metrics
217
- CREATE TABLE IF NOT EXISTS pipeline_metrics (
218
- id TEXT PRIMARY KEY,
219
- ts TEXT NOT NULL,
220
- stage TEXT NOT NULL,
221
- latency_ms REAL NOT NULL,
222
- success INTEGER NOT NULL,
223
- error TEXT,
224
- session_id TEXT
225
- );
226
-
227
- -- Working Set table (active memory window)
228
- CREATE TABLE IF NOT EXISTS working_set (
229
- id TEXT PRIMARY KEY,
230
- event_id TEXT NOT NULL,
231
- added_at TEXT DEFAULT (datetime('now')),
232
- relevance_score REAL DEFAULT 1.0,
233
- topics TEXT,
234
- expires_at TEXT
235
- );
236
-
237
- -- Consolidated Memories table (long-term integrated memories)
238
- CREATE TABLE IF NOT EXISTS consolidated_memories (
239
- memory_id TEXT PRIMARY KEY,
240
- summary TEXT NOT NULL,
241
- topics TEXT,
242
- source_events TEXT,
243
- confidence REAL DEFAULT 0.5,
244
- created_at TEXT DEFAULT (datetime('now')),
245
- accessed_at TEXT,
246
- access_count INTEGER DEFAULT 0
247
- );
248
-
249
- -- Continuity Log table (tracks context transitions)
250
- CREATE TABLE IF NOT EXISTS continuity_log (
251
- log_id TEXT PRIMARY KEY,
252
- from_context_id TEXT,
253
- to_context_id TEXT,
254
- continuity_score REAL,
255
- transition_type TEXT,
256
- created_at TEXT DEFAULT (datetime('now'))
257
- );
258
-
259
- -- Consolidated Rules table (long-term stable memory)
260
- CREATE TABLE IF NOT EXISTS consolidated_rules (
261
- rule_id TEXT PRIMARY KEY,
262
- rule TEXT NOT NULL,
263
- topics TEXT,
264
- source_memory_ids TEXT,
265
- source_events TEXT,
266
- confidence REAL DEFAULT 0.5,
267
- created_at TEXT DEFAULT (datetime('now'))
268
- );
269
-
270
- -- Endless Mode Config table
271
- CREATE TABLE IF NOT EXISTS endless_config (
272
- key TEXT PRIMARY KEY,
273
- value TEXT,
274
- updated_at TEXT DEFAULT (datetime('now'))
275
- );
276
-
277
- -- Memory Helpfulness tracking
278
- CREATE TABLE IF NOT EXISTS memory_helpfulness (
279
- id TEXT PRIMARY KEY,
280
- event_id TEXT NOT NULL,
281
- session_id TEXT NOT NULL,
282
- retrieval_score REAL DEFAULT 0,
283
- query_preview TEXT,
284
- session_continued INTEGER DEFAULT 0,
285
- prompt_count_after INTEGER DEFAULT 0,
286
- tool_success_count INTEGER DEFAULT 0,
287
- tool_total_count INTEGER DEFAULT 0,
288
- was_reasked INTEGER DEFAULT 0,
289
- helpfulness_score REAL DEFAULT 0.5,
290
- created_at TEXT DEFAULT (datetime('now')),
291
- measured_at TEXT
292
- );
293
-
294
- -- Retrieval trace log (query -> candidates -> selected for context)
295
- CREATE TABLE IF NOT EXISTS retrieval_traces (
296
- trace_id TEXT PRIMARY KEY,
297
- session_id TEXT,
298
- project_hash TEXT,
299
- query_text TEXT NOT NULL,
300
- strategy TEXT,
301
- candidate_event_ids TEXT,
302
- selected_event_ids TEXT,
303
- candidate_details_json TEXT,
304
- selected_details_json TEXT,
305
- candidate_count INTEGER DEFAULT 0,
306
- selected_count INTEGER DEFAULT 0,
307
- confidence TEXT,
308
- fallback_trace TEXT,
309
- created_at TEXT DEFAULT (datetime('now'))
310
- );
311
-
312
- -- Sync position tracking (for SQLite -> DuckDB sync)
313
- CREATE TABLE IF NOT EXISTS sync_positions (
314
- target_name TEXT PRIMARY KEY,
315
- last_event_id TEXT,
316
- last_timestamp TEXT,
317
- updated_at TEXT DEFAULT (datetime('now'))
318
- );
319
-
320
- -- Create indexes
321
- CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
322
- CREATE INDEX IF NOT EXISTS idx_events_timestamp ON events(timestamp);
323
- CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
324
- CREATE INDEX IF NOT EXISTS idx_entries_stage ON entries(stage);
325
- CREATE INDEX IF NOT EXISTS idx_entries_canonical ON entries(canonical_key);
326
- CREATE INDEX IF NOT EXISTS idx_entities_type_key ON entities(entity_type, canonical_key);
327
- CREATE INDEX IF NOT EXISTS idx_entities_status ON entities(status);
328
- CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src_id, rel_type);
329
- CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst_id, rel_type);
330
- CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(rel_type);
331
- CREATE INDEX IF NOT EXISTS idx_outbox_status ON vector_outbox(status);
332
- CREATE INDEX IF NOT EXISTS idx_working_set_expires ON working_set(expires_at);
333
- CREATE INDEX IF NOT EXISTS idx_working_set_relevance ON working_set(relevance_score);
334
- CREATE INDEX IF NOT EXISTS idx_consolidated_confidence ON consolidated_memories(confidence);
335
- CREATE INDEX IF NOT EXISTS idx_continuity_created ON continuity_log(created_at);
336
- CREATE INDEX IF NOT EXISTS idx_consolidated_rules_confidence ON consolidated_rules(confidence);
337
- CREATE INDEX IF NOT EXISTS idx_embedding_outbox_status ON embedding_outbox(status);
338
- CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
339
- CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
340
- CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
341
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
342
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
343
- CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
344
-
345
- -- FTS5 Full-Text Search for fast keyword search
346
- CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
347
- content,
348
- event_id UNINDEXED,
349
- tokenize='porter unicode61'
350
- );
351
-
352
- -- Triggers to keep FTS in sync with events table
353
- CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
354
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
355
- END;
356
-
357
- CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
358
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
359
- END;
360
-
361
- CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
362
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
363
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
364
- END;
365
- `);
366
-
367
-
368
- // Best-effort forward migration for retrieval trace detail column
369
- try {
370
- sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN selected_details_json TEXT;`);
371
- } catch {
372
- // column may already exist
373
- }
374
- try {
375
- sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
376
- } catch {
377
- // column may already exist
378
- }
379
-
380
- // Migrate existing events table to add new columns if they don't exist
381
- // Check if columns exist before trying to add them
382
- const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
383
- const columnNames = tableInfo.map((col: any) => col.name);
384
-
385
- if (!columnNames.includes('access_count')) {
386
- try {
387
- sqliteExec(this.db, `
388
- ALTER TABLE events ADD COLUMN access_count INTEGER DEFAULT 0;
389
- `);
390
- } catch (err: any) {
391
- console.error('Error adding access_count column:', err);
392
- }
393
- }
394
-
395
- if (!columnNames.includes('last_accessed_at')) {
396
- try {
397
- sqliteExec(this.db, `
398
- ALTER TABLE events ADD COLUMN last_accessed_at TEXT;
399
- `);
400
- } catch (err: any) {
401
- console.error('Error adding last_accessed_at column:', err);
402
- }
403
- }
404
-
405
- // Add turn_id column for grouping events within a conversation turn
406
- if (!columnNames.includes('turn_id')) {
407
- try {
408
- sqliteExec(this.db, `
409
- ALTER TABLE events ADD COLUMN turn_id TEXT;
410
- `);
411
- } catch (err: any) {
412
- console.error('Error adding turn_id column:', err);
413
- }
414
- }
415
-
416
- // Create indexes for new columns if they don't exist
417
- try {
418
- sqliteExec(this.db, `
419
- CREATE INDEX IF NOT EXISTS idx_events_access_count ON events(access_count DESC);
420
- `);
421
- } catch (err: any) {
422
- // Index may already exist, ignore
423
- }
424
-
425
- try {
426
- sqliteExec(this.db, `
427
- CREATE INDEX IF NOT EXISTS idx_events_last_accessed ON events(last_accessed_at DESC);
428
- `);
429
- } catch (err: any) {
430
- // Index may already exist, ignore
431
- }
432
-
433
- try {
434
- sqliteExec(this.db, `
435
- CREATE INDEX IF NOT EXISTS idx_events_turn_id ON events(turn_id);
436
- `);
437
- } catch (err: any) {
438
- // Index may already exist, ignore
439
- }
440
-
441
- this.initialized = true;
442
- }
443
-
444
- /**
445
- * Append event to store (Append-only, Idempotent)
446
- */
447
- async append(input: MemoryEventInput): Promise<AppendResult> {
448
- await this.initialize();
449
-
450
- const canonicalKey = makeCanonicalKey(input.content);
451
- const dedupeKey = makeDedupeKey(input.content, input.sessionId);
452
-
453
- // Check for duplicate
454
- const existing = sqliteGet<{ event_id: string }>(
455
- this.db,
456
- `SELECT event_id FROM event_dedup WHERE dedupe_key = ?`,
457
- [dedupeKey]
458
- );
459
-
460
- if (existing) {
461
- return {
462
- success: true,
463
- eventId: existing.event_id,
464
- isDuplicate: true
465
- };
466
- }
467
-
468
- const id = randomUUID();
469
- const timestamp = toSQLiteTimestamp(input.timestamp);
470
-
471
- try {
472
- // Extract turnId from metadata if present
473
- const metadata = input.metadata || {};
474
- const turnId = (metadata.turnId as string) || null;
475
-
476
- // Use transaction for atomicity
477
- const insertEvent = this.db.prepare(`
478
- INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
479
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
480
- `);
481
-
482
- const insertDedup = this.db.prepare(`
483
- INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
484
- `);
485
-
486
- const insertLevel = this.db.prepare(`
487
- INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
488
- `);
489
-
490
- const transaction = this.db.transaction(() => {
491
- insertEvent.run(
492
- id,
493
- input.eventType,
494
- input.sessionId,
495
- timestamp,
496
- input.content,
497
- canonicalKey,
498
- dedupeKey,
499
- JSON.stringify(metadata),
500
- turnId
501
- );
502
- insertDedup.run(dedupeKey, id);
503
- insertLevel.run(id);
504
- });
505
-
506
- transaction();
507
-
508
- if (this.markdownMirror) {
509
- const event: MemoryEvent = {
510
- id,
511
- eventType: input.eventType,
512
- sessionId: input.sessionId,
513
- timestamp: input.timestamp,
514
- content: input.content,
515
- canonicalKey,
516
- dedupeKey,
517
- metadata
518
- };
519
- this.markdownMirror.append(event).catch((err) => {
520
- console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
521
- });
522
- }
523
-
524
- return { success: true, eventId: id, isDuplicate: false };
525
- } catch (error) {
526
- return {
527
- success: false,
528
- error: error instanceof Error ? error.message : String(error)
529
- };
530
- }
531
- }
532
-
533
- /**
534
- * Get session IDs that have events but no session_summary event.
535
- * Used to backfill summaries for sessions that ended without Stop hook.
536
- */
537
- async getSessionsWithoutSummary(currentSessionId: string, limit = 5): Promise<string[]> {
538
- await this.initialize();
539
- const rows = sqliteAll<{ session_id: string }>(
540
- this.db,
541
- `SELECT DISTINCT e.session_id
542
- FROM events e
543
- WHERE e.session_id != ?
544
- AND e.event_type != 'session_summary'
545
- AND e.session_id NOT IN (
546
- SELECT DISTINCT session_id FROM events WHERE event_type = 'session_summary'
547
- )
548
- GROUP BY e.session_id
549
- HAVING COUNT(*) >= 3
550
- ORDER BY MAX(e.timestamp) DESC
551
- LIMIT ?`,
552
- [currentSessionId, limit]
553
- );
554
- return rows.map((r) => r.session_id);
555
- }
556
-
557
- /**
558
- * Get events by session ID
559
- */
560
- async getSessionEvents(sessionId: string): Promise<MemoryEvent[]> {
561
- await this.initialize();
562
-
563
- const rows = sqliteAll<Record<string, unknown>>(
564
- this.db,
565
- `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
566
- [sessionId]
567
- );
568
-
569
- return rows.map(this.rowToEvent);
570
- }
571
-
572
- /**
573
- * Get recent events
574
- */
575
- async getRecentEvents(limit: number = 100): Promise<MemoryEvent[]> {
576
- await this.initialize();
577
-
578
- const rows = sqliteAll<Record<string, unknown>>(
579
- this.db,
580
- `SELECT * FROM events ORDER BY timestamp DESC LIMIT ?`,
581
- [limit]
582
- );
583
-
584
- return rows.map(this.rowToEvent);
585
- }
586
-
587
- /**
588
- * Get event by ID
589
- */
590
- async getEvent(id: string): Promise<MemoryEvent | null> {
591
- await this.initialize();
592
-
593
- const row = sqliteGet<Record<string, unknown>>(
594
- this.db,
595
- `SELECT * FROM events WHERE id = ?`,
596
- [id]
597
- );
598
-
599
- if (!row) return null;
600
- return this.rowToEvent(row);
601
- }
602
-
603
- /**
604
- * Get events since a timestamp (for sync)
605
- */
606
- async getEventsSince(timestamp: string, limit: number = 1000): Promise<MemoryEvent[]> {
607
- await this.initialize();
608
-
609
- const rows = sqliteAll<Record<string, unknown>>(
610
- this.db,
611
- `SELECT * FROM events WHERE timestamp > ? ORDER BY timestamp ASC LIMIT ?`,
612
- [timestamp, limit]
613
- );
614
-
615
- return rows.map(this.rowToEvent);
616
- }
617
-
618
- /**
619
- * Get events since a SQLite rowid (for robust incremental replication).
620
- * Rowid is monotonic for append-only tables, independent of client timestamps.
621
- */
622
- async getEventsSinceRowid(
623
- lastRowid: number,
624
- limit: number = 1000
625
- ): Promise<Array<{ rowid: number; event: MemoryEvent }>> {
626
- await this.initialize();
627
-
628
- const rows = sqliteAll<Record<string, unknown>>(
629
- this.db,
630
- `SELECT rowid as _rowid, * FROM events WHERE rowid > ? ORDER BY rowid ASC LIMIT ?`,
631
- [lastRowid, limit]
632
- );
633
-
634
- return rows.map(row => ({
635
- rowid: row._rowid as number,
636
- event: this.rowToEvent(row)
637
- }));
638
- }
639
-
640
- /**
641
- * Import events with fixed IDs (used for cross-machine replication).
642
- * Idempotent: skips if event id or dedupeKey already exists.
643
- *
644
- * NOTE: This bypasses the append() id generation to preserve stable IDs.
645
- */
646
- async importEvents(events: MemoryEvent[]): Promise<{ inserted: number; skipped: number }> {
647
- if (events.length === 0) return { inserted: 0, skipped: 0 };
648
- if (this.readOnly) return { inserted: 0, skipped: events.length };
649
-
650
- await this.initialize();
651
-
652
- const getById = this.db.prepare(`SELECT id FROM events WHERE id = ?`);
653
- const getByDedupe = this.db.prepare(`SELECT event_id FROM event_dedup WHERE dedupe_key = ?`);
654
-
655
- const insertEvent = this.db.prepare(`
656
- INSERT INTO events (id, event_type, session_id, timestamp, content, canonical_key, dedupe_key, metadata, turn_id)
657
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
658
- `);
659
-
660
- const insertDedup = this.db.prepare(`
661
- INSERT INTO event_dedup (dedupe_key, event_id) VALUES (?, ?)
662
- `);
663
-
664
- const insertLevel = this.db.prepare(`
665
- INSERT INTO memory_levels (event_id, level) VALUES (?, 'L0')
666
- `);
667
-
668
- let inserted = 0;
669
- let skipped = 0;
670
- const insertedEvents: MemoryEvent[] = [];
671
-
672
- const tx = this.db.transaction((batch: MemoryEvent[]) => {
673
- for (const ev of batch) {
674
- // Skip if already present by id
675
- const existingById = getById.get(ev.id) as { id: string } | undefined;
676
- if (existingById) {
677
- skipped++;
678
- continue;
679
- }
680
-
681
- const canonicalKey = ev.canonicalKey || makeCanonicalKey(ev.content);
682
- const dedupeKey = ev.dedupeKey || makeDedupeKey(ev.content, ev.sessionId);
683
-
684
- // Skip if already present by dedupe key
685
- const existingByDedupe = getByDedupe.get(dedupeKey) as { event_id: string } | undefined;
686
- if (existingByDedupe) {
687
- skipped++;
688
- continue;
689
- }
690
-
691
- const metadata = ev.metadata || {};
692
- const turnId = (metadata as any).turnId as string | undefined;
693
-
694
- insertEvent.run(
695
- ev.id,
696
- ev.eventType,
697
- ev.sessionId,
698
- toSQLiteTimestamp(ev.timestamp),
699
- ev.content,
700
- canonicalKey,
701
- dedupeKey,
702
- JSON.stringify(metadata),
703
- turnId ?? null
704
- );
705
-
706
- insertDedup.run(dedupeKey, ev.id);
707
- insertLevel.run(ev.id);
708
- inserted++;
709
- insertedEvents.push(ev);
710
- }
711
- });
712
-
713
- tx(events);
714
-
715
- if (this.markdownMirror && insertedEvents.length > 0) {
716
- for (const ev of insertedEvents) {
717
- this.markdownMirror.append(ev).catch((err) => {
718
- console.warn('[SQLiteEventStore] markdown mirror append failed:', err);
719
- });
720
- }
721
- }
722
-
723
- return { inserted, skipped };
724
- }
725
-
726
- /**
727
- * Create or update session
728
- */
729
- async upsertSession(session: Partial<Session> & { id: string }): Promise<void> {
730
- await this.initialize();
731
-
732
- const existing = sqliteGet<{ id: string }>(
733
- this.db,
734
- `SELECT id FROM sessions WHERE id = ?`,
735
- [session.id]
736
- );
737
-
738
- if (!existing) {
739
- sqliteRun(
740
- this.db,
741
- `INSERT INTO sessions (id, started_at, project_path, tags)
742
- VALUES (?, ?, ?, ?)`,
743
- [
744
- session.id,
745
- toSQLiteTimestamp(session.startedAt || new Date()),
746
- session.projectPath || null,
747
- JSON.stringify(session.tags || [])
748
- ]
749
- );
750
- } else {
751
- const updates: string[] = [];
752
- const values: unknown[] = [];
753
-
754
- if (session.endedAt) {
755
- updates.push('ended_at = ?');
756
- values.push(toSQLiteTimestamp(session.endedAt));
757
- }
758
- if (session.summary) {
759
- updates.push('summary = ?');
760
- values.push(session.summary);
761
- }
762
- if (session.tags) {
763
- updates.push('tags = ?');
764
- values.push(JSON.stringify(session.tags));
765
- }
766
-
767
- if (updates.length > 0) {
768
- values.push(session.id);
769
- sqliteRun(
770
- this.db,
771
- `UPDATE sessions SET ${updates.join(', ')} WHERE id = ?`,
772
- values
773
- );
774
- }
775
- }
776
- }
777
-
778
- /**
779
- * Get session by ID
780
- */
781
- async getSession(id: string): Promise<Session | null> {
782
- await this.initialize();
783
-
784
- const row = sqliteGet<Record<string, unknown>>(
785
- this.db,
786
- `SELECT * FROM sessions WHERE id = ?`,
787
- [id]
788
- );
789
-
790
- if (!row) return null;
791
-
792
- return {
793
- id: row.id as string,
794
- startedAt: toDateFromSQLite(row.started_at),
795
- endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
796
- projectPath: row.project_path as string | undefined,
797
- summary: row.summary as string | undefined,
798
- tags: row.tags ? JSON.parse(row.tags as string) : undefined
799
- };
800
- }
801
-
802
- /**
803
- * Get all sessions
804
- */
805
- async getAllSessions(): Promise<Session[]> {
806
- await this.initialize();
807
-
808
- const rows = sqliteAll<Record<string, unknown>>(
809
- this.db,
810
- `SELECT * FROM sessions ORDER BY started_at DESC`
811
- );
812
-
813
- return rows.map(row => ({
814
- id: row.id as string,
815
- startedAt: toDateFromSQLite(row.started_at),
816
- endedAt: row.ended_at ? toDateFromSQLite(row.ended_at) : undefined,
817
- projectPath: row.project_path as string | undefined,
818
- summary: row.summary as string | undefined,
819
- tags: row.tags ? JSON.parse(row.tags as string) : undefined
820
- }));
821
- }
822
-
823
- /**
824
- * Add to embedding outbox
825
- */
826
- async enqueueForEmbedding(eventId: string, content: string): Promise<string> {
827
- await this.initialize();
828
-
829
- const id = randomUUID();
830
- sqliteRun(
831
- this.db,
832
- `INSERT INTO embedding_outbox (id, event_id, content, status, retry_count)
833
- VALUES (?, ?, ?, 'pending', 0)`,
834
- [id, eventId, content]
835
- );
836
-
837
- return id;
838
- }
839
-
840
- /**
841
- * Get pending outbox items
842
- */
843
- async getPendingOutboxItems(limit: number = 32): Promise<OutboxItem[]> {
844
- await this.initialize();
845
-
846
- const pending = sqliteAll<Record<string, unknown>>(
847
- this.db,
848
- `SELECT * FROM embedding_outbox
849
- WHERE status = 'pending'
850
- ORDER BY created_at
851
- LIMIT ?`,
852
- [limit]
853
- );
854
-
855
- if (pending.length === 0) return [];
856
-
857
- // Update status to processing
858
- const ids = pending.map(r => r.id as string);
859
- const placeholders = ids.map(() => '?').join(',');
860
- sqliteRun(
861
- this.db,
862
- `UPDATE embedding_outbox SET status = 'processing' WHERE id IN (${placeholders})`,
863
- ids
864
- );
865
-
866
- return pending.map(row => ({
867
- id: row.id as string,
868
- eventId: row.event_id as string,
869
- content: row.content as string,
870
- status: 'processing' as const,
871
- retryCount: row.retry_count as number,
872
- createdAt: toDateFromSQLite(row.created_at),
873
- errorMessage: row.error_message as string | undefined
874
- }));
875
- }
876
-
877
- /**
878
- * Mark outbox items as done
879
- */
880
- async completeOutboxItems(ids: string[]): Promise<void> {
881
- if (ids.length === 0) return;
882
-
883
- const placeholders = ids.map(() => '?').join(',');
884
- sqliteRun(
885
- this.db,
886
- `DELETE FROM embedding_outbox WHERE id IN (${placeholders})`,
887
- ids
888
- );
889
- }
890
-
891
- /**
892
- * Clear embedding outbox (used for embedding model migration)
893
- */
894
- async clearEmbeddingOutbox(): Promise<void> {
895
- await this.initialize();
896
- sqliteRun(this.db, `DELETE FROM embedding_outbox`);
897
- }
898
-
899
- /**
900
- * Count total events
901
- */
902
- async countEvents(): Promise<number> {
903
- await this.initialize();
904
- const row = sqliteGet<{ count: number }>(this.db, `SELECT COUNT(*) as count FROM events`);
905
- return row?.count || 0;
906
- }
907
-
908
- /**
909
- * Get events page in timestamp ascending order (stable migration/reindex scans)
910
- */
911
- async getEventsPage(limit: number = 1000, offset: number = 0): Promise<MemoryEvent[]> {
912
- await this.initialize();
913
-
914
- const rows = sqliteAll<Record<string, unknown>>(
915
- this.db,
916
- `SELECT * FROM events ORDER BY timestamp ASC LIMIT ? OFFSET ?`,
917
- [limit, offset]
918
- );
919
-
920
- return rows.map(this.rowToEvent);
921
- }
922
-
923
- /**
924
- * Mark outbox items as failed
925
- */
926
- async failOutboxItems(ids: string[], error: string): Promise<void> {
927
- if (ids.length === 0) return;
928
-
929
- const placeholders = ids.map(() => '?').join(',');
930
- sqliteRun(
931
- this.db,
932
- `UPDATE embedding_outbox
933
- SET status = CASE WHEN retry_count >= 3 THEN 'failed' ELSE 'pending' END,
934
- retry_count = retry_count + 1,
935
- error_message = ?
936
- WHERE id IN (${placeholders})`,
937
- [error, ...ids]
938
- );
939
- }
940
-
941
-
942
- /**
943
- * Get embedding/vector outbox health statistics
944
- */
945
- async getOutboxStats(): Promise<{
946
- embedding: { pending: number; processing: number; failed: number; total: number };
947
- vector: { pending: number; processing: number; failed: number; total: number };
948
- }> {
949
- await this.initialize();
950
-
951
- const embeddingRows = sqliteAll<{ status: string; count: number }>(
952
- this.db,
953
- `SELECT status, COUNT(*) as count FROM embedding_outbox GROUP BY status`
954
- );
955
- const vectorRows = sqliteAll<{ status: string; count: number }>(
956
- this.db,
957
- `SELECT status, COUNT(*) as count FROM vector_outbox GROUP BY status`
958
- );
959
-
960
- const fromRows = (rows: Array<{ status: string; count: number }>) => {
961
- const out = { pending: 0, processing: 0, failed: 0, total: 0 };
962
- for (const row of rows) {
963
- const key = row.status as 'pending' | 'processing' | 'failed' | 'done';
964
- if (key === 'pending' || key === 'processing' || key === 'failed') {
965
- out[key] += row.count;
966
- }
967
- out.total += row.count;
968
- }
969
- return out;
970
- };
971
-
972
- return {
973
- embedding: fromRows(embeddingRows),
974
- vector: fromRows(vectorRows)
975
- };
976
- }
977
-
978
- /**
979
- * Update memory level
980
- */
981
- async updateMemoryLevel(eventId: string, level: string): Promise<void> {
982
- await this.initialize();
983
-
984
- sqliteRun(
985
- this.db,
986
- `UPDATE memory_levels SET level = ?, promoted_at = datetime('now') WHERE event_id = ?`,
987
- [level, eventId]
988
- );
989
- }
990
-
991
- /**
992
- * Get memory level statistics
993
- */
994
- async getLevelStats(): Promise<Array<{ level: string; count: number }>> {
995
- await this.initialize();
996
-
997
- const rows = sqliteAll<{ level: string; count: number }>(
998
- this.db,
999
- `SELECT level, COUNT(*) as count FROM memory_levels GROUP BY level`
1000
- );
1001
-
1002
- return rows;
1003
- }
1004
-
1005
- /**
1006
- * Get events by memory level
1007
- */
1008
- async getEventsByLevel(level: string, options?: { limit?: number; offset?: number }): Promise<MemoryEvent[]> {
1009
- await this.initialize();
1010
-
1011
- const limit = options?.limit || 50;
1012
- const offset = options?.offset || 0;
1013
-
1014
- const rows = sqliteAll<Record<string, unknown>>(
1015
- this.db,
1016
- `SELECT e.* FROM events e
1017
- INNER JOIN memory_levels ml ON e.id = ml.event_id
1018
- WHERE ml.level = ?
1019
- ORDER BY e.timestamp DESC
1020
- LIMIT ? OFFSET ?`,
1021
- [level, limit, offset]
1022
- );
1023
-
1024
- return rows.map(row => this.rowToEvent(row));
1025
- }
1026
-
1027
- /**
1028
- * Get memory level for a specific event
1029
- */
1030
- async getEventLevel(eventId: string): Promise<string | null> {
1031
- await this.initialize();
1032
-
1033
- const row = sqliteGet<{ level: string }>(
1034
- this.db,
1035
- `SELECT level FROM memory_levels WHERE event_id = ?`,
1036
- [eventId]
1037
- );
1038
-
1039
- return row ? row.level : null;
1040
- }
1041
-
1042
- /**
1043
- * Get sync position for a target
1044
- */
1045
- async getSyncPosition(targetName: string): Promise<{ lastEventId: string | null; lastTimestamp: string | null }> {
1046
- await this.initialize();
1047
-
1048
- const row = sqliteGet<{ last_event_id: string | null; last_timestamp: string | null }>(
1049
- this.db,
1050
- `SELECT last_event_id, last_timestamp FROM sync_positions WHERE target_name = ?`,
1051
- [targetName]
1052
- );
1053
-
1054
- return {
1055
- lastEventId: row?.last_event_id ?? null,
1056
- lastTimestamp: row?.last_timestamp ?? null
1057
- };
1058
- }
1059
-
1060
- /**
1061
- * Update sync position for a target
1062
- */
1063
- async updateSyncPosition(targetName: string, lastEventId: string, lastTimestamp: string): Promise<void> {
1064
- await this.initialize();
1065
-
1066
- sqliteRun(
1067
- this.db,
1068
- `INSERT OR REPLACE INTO sync_positions (target_name, last_event_id, last_timestamp, updated_at)
1069
- VALUES (?, ?, ?, datetime('now'))`,
1070
- [targetName, lastEventId, lastTimestamp]
1071
- );
1072
- }
1073
-
1074
- /**
1075
- * Get config value for endless mode
1076
- */
1077
- async getEndlessConfig(key: string): Promise<unknown | null> {
1078
- await this.initialize();
1079
-
1080
- const row = sqliteGet<{ value: string }>(
1081
- this.db,
1082
- `SELECT value FROM endless_config WHERE key = ?`,
1083
- [key]
1084
- );
1085
-
1086
- if (!row) return null;
1087
- return JSON.parse(row.value);
1088
- }
1089
-
1090
- /**
1091
- * Set config value for endless mode
1092
- */
1093
- async setEndlessConfig(key: string, value: unknown): Promise<void> {
1094
- await this.initialize();
1095
-
1096
- sqliteRun(
1097
- this.db,
1098
- `INSERT OR REPLACE INTO endless_config (key, value, updated_at)
1099
- VALUES (?, ?, datetime('now'))`,
1100
- [key, JSON.stringify(value)]
1101
- );
1102
- }
1103
-
1104
- /**
1105
- * Increment access count for events
1106
- */
1107
- async incrementAccessCount(eventIds: string[]): Promise<void> {
1108
- if (eventIds.length === 0 || this.readOnly) return;
1109
-
1110
- await this.initialize();
1111
-
1112
- const placeholders = eventIds.map(() => '?').join(',');
1113
- const currentTime = toSQLiteTimestamp(new Date());
1114
-
1115
- sqliteRun(
1116
- this.db,
1117
- `UPDATE events
1118
- SET access_count = access_count + 1,
1119
- last_accessed_at = ?
1120
- WHERE id IN (${placeholders})`,
1121
- [currentTime, ...eventIds]
1122
- );
1123
- }
1124
-
1125
- /**
1126
- * Get most accessed memories (falls back to recent events if none accessed)
1127
- */
1128
- async getMostAccessed(limit: number = 10): Promise<MemoryEvent[]> {
1129
- await this.initialize();
1130
-
1131
- // First try events with access_count > 0
1132
- let rows = sqliteAll<Record<string, unknown>>(
1133
- this.db,
1134
- `SELECT * FROM events
1135
- WHERE access_count > 0
1136
- ORDER BY access_count DESC, last_accessed_at DESC
1137
- LIMIT ?`,
1138
- [limit]
1139
- );
1140
-
1141
- // Fallback: if no accessed events, show recent events
1142
- if (rows.length === 0) {
1143
- rows = sqliteAll<Record<string, unknown>>(
1144
- this.db,
1145
- `SELECT * FROM events
1146
- ORDER BY timestamp DESC
1147
- LIMIT ?`,
1148
- [limit]
1149
- );
1150
- }
1151
-
1152
- return rows.map(row => this.rowToEvent(row));
1153
- }
1154
-
1155
- /**
1156
- * Record a memory retrieval for helpfulness tracking
1157
- */
1158
- async recordRetrieval(eventId: string, sessionId: string, score: number, query: string): Promise<void> {
1159
- if (this.readOnly) return;
1160
- await this.initialize();
1161
-
1162
- const id = randomUUID();
1163
- sqliteRun(
1164
- this.db,
1165
- `INSERT INTO memory_helpfulness (id, event_id, session_id, retrieval_score, query_preview, created_at)
1166
- VALUES (?, ?, ?, ?, ?, datetime('now'))`,
1167
- [id, eventId, sessionId, score, query.slice(0, 100)]
1168
- );
1169
- }
1170
-
1171
- /**
1172
- * Get session IDs that have unevaluated retrievals (measured_at IS NULL).
1173
- * Excludes the current session. Used to backfill sessions that ended without Stop hook.
1174
- */
1175
- async getUnevaluatedSessions(currentSessionId: string, limit = 5): Promise<string[]> {
1176
- await this.initialize();
1177
- const rows = sqliteAll<{ session_id: string }>(
1178
- this.db,
1179
- `SELECT DISTINCT session_id FROM memory_helpfulness
1180
- WHERE measured_at IS NULL AND session_id != ?
1181
- ORDER BY created_at DESC LIMIT ?`,
1182
- [currentSessionId, limit]
1183
- );
1184
- return rows.map((r) => r.session_id);
1185
- }
1186
-
1187
- /**
1188
- * Evaluate helpfulness for all retrievals in a session
1189
- * Called at session end - uses behavioral signals to compute score
1190
- */
1191
- async evaluateSessionHelpfulness(sessionId: string): Promise<void> {
1192
- if (this.readOnly) return;
1193
- await this.initialize();
1194
-
1195
- // Get all retrieval records for this session
1196
- const retrievals = sqliteAll<Record<string, unknown>>(
1197
- this.db,
1198
- `SELECT * FROM memory_helpfulness WHERE session_id = ? AND measured_at IS NULL`,
1199
- [sessionId]
1200
- );
1201
-
1202
- if (retrievals.length === 0) return;
1203
-
1204
- // Get session events to analyze behavior after retrieval
1205
- const sessionEvents = sqliteAll<Record<string, unknown>>(
1206
- this.db,
1207
- `SELECT * FROM events WHERE session_id = ? ORDER BY timestamp ASC`,
1208
- [sessionId]
1209
- );
1210
-
1211
- const promptEvents = sessionEvents.filter((e: any) => e.event_type === 'user_prompt');
1212
- const toolEvents = sessionEvents.filter((e: any) => e.event_type === 'tool_observation');
1213
-
1214
- // Count successful vs failed tools
1215
- let toolSuccessCount = 0;
1216
- let toolTotalCount = toolEvents.length;
1217
- for (const t of toolEvents) {
1218
- try {
1219
- const content = JSON.parse(t.content as string);
1220
- if (content.success !== false) toolSuccessCount++;
1221
- } catch {
1222
- toolSuccessCount++; // Assume success if can't parse
1223
- }
1224
- }
1225
- const toolSuccessRatio = toolTotalCount > 0 ? toolSuccessCount / toolTotalCount : 0.5;
1226
-
1227
- for (const retrieval of retrievals) {
1228
- const retrievalTime = retrieval.created_at as string;
1229
-
1230
- // 1. Session continued after retrieval?
1231
- const eventsAfter = sessionEvents.filter((e: any) => e.timestamp > retrievalTime);
1232
- const sessionContinued = eventsAfter.length > 0 ? 1 : 0;
1233
-
1234
- // 2. How many prompts came after?
1235
- const promptsAfter = promptEvents.filter((e: any) => e.timestamp > retrievalTime);
1236
- const promptCountAfter = promptsAfter.length;
1237
-
1238
- // 3. Was a similar query asked again? (simple word overlap check)
1239
- const queryWords = new Set((retrieval.query_preview as string || '').toLowerCase().split(/\s+/).filter(w => w.length > 2));
1240
- let wasReasked = 0;
1241
- for (const p of promptsAfter) {
1242
- const pWords = new Set((p.content as string).toLowerCase().split(/\s+/).filter((w: string) => w.length > 2));
1243
- let overlap = 0;
1244
- for (const w of queryWords) {
1245
- if (pWords.has(w)) overlap++;
1246
- }
1247
- if (queryWords.size > 0 && overlap / queryWords.size > 0.5) {
1248
- wasReasked = 1;
1249
- break;
1250
- }
1251
- }
1252
-
1253
- // Calculate helpfulness score
1254
- // Weights tuned for shopping-assistant-like corpora where sessions
1255
- // continue on the same topic (was_reasked was over-penalising normal conversation flow)
1256
- const retrievalScore = retrieval.retrieval_score as number || 0;
1257
- // More prompts after retrieval = memory was actually useful to the conversation
1258
- const promptNorm = Math.min(promptCountAfter / 2, 1.0);
1259
- const helpfulnessScore = (
1260
- 0.40 * Math.min(retrievalScore, 1.0) +
1261
- 0.30 * promptNorm +
1262
- 0.20 * toolSuccessRatio +
1263
- 0.10 * (sessionContinued ? 1.0 : 0.0)
1264
- );
1265
-
1266
- sqliteRun(
1267
- this.db,
1268
- `UPDATE memory_helpfulness
1269
- SET session_continued = ?, prompt_count_after = ?,
1270
- tool_success_count = ?, tool_total_count = ?,
1271
- was_reasked = ?, helpfulness_score = ?,
1272
- measured_at = datetime('now')
1273
- WHERE id = ?`,
1274
- [sessionContinued, promptCountAfter, toolSuccessCount, toolTotalCount,
1275
- wasReasked, helpfulnessScore, retrieval.id]
1276
- );
1277
- }
1278
- }
1279
-
1280
- /**
1281
- * Get most helpful memories ranked by helpfulness score
1282
- */
1283
- async getHelpfulMemories(limit: number = 10): Promise<Array<{
1284
- eventId: string;
1285
- summary: string;
1286
- helpfulnessScore: number;
1287
- accessCount: number;
1288
- evaluationCount: number;
1289
- }>> {
1290
- await this.initialize();
1291
-
1292
- const rows = sqliteAll<Record<string, unknown>>(
1293
- this.db,
1294
- `SELECT
1295
- mh.event_id,
1296
- AVG(mh.helpfulness_score) as avg_score,
1297
- COUNT(*) as eval_count,
1298
- e.content,
1299
- e.access_count
1300
- FROM memory_helpfulness mh
1301
- JOIN events e ON e.id = mh.event_id
1302
- WHERE mh.measured_at IS NOT NULL
1303
- GROUP BY mh.event_id
1304
- ORDER BY avg_score DESC
1305
- LIMIT ?`,
1306
- [limit]
1307
- );
1308
-
1309
- return rows.map(r => ({
1310
- eventId: r.event_id as string,
1311
- summary: (r.content as string).substring(0, 200) + ((r.content as string).length > 200 ? '...' : ''),
1312
- helpfulnessScore: Math.round((r.avg_score as number) * 100) / 100,
1313
- accessCount: (r.access_count as number) || 0,
1314
- evaluationCount: r.eval_count as number
1315
- }));
1316
- }
1317
-
1318
- /**
1319
- * Get helpfulness statistics for dashboard
1320
- */
1321
- async getHelpfulnessStats(): Promise<{
1322
- avgScore: number;
1323
- totalEvaluated: number;
1324
- totalRetrievals: number;
1325
- helpful: number;
1326
- neutral: number;
1327
- unhelpful: number;
1328
- }> {
1329
- await this.initialize();
1330
-
1331
- const stats = sqliteGet<Record<string, unknown>>(
1332
- this.db,
1333
- `SELECT
1334
- AVG(helpfulness_score) as avg_score,
1335
- COUNT(*) as total_evaluated,
1336
- SUM(CASE WHEN helpfulness_score >= 0.7 THEN 1 ELSE 0 END) as helpful,
1337
- SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
1338
- SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
1339
- FROM memory_helpfulness
1340
- WHERE measured_at IS NOT NULL`
1341
- );
1342
-
1343
- const totalRow = sqliteGet<Record<string, unknown>>(
1344
- this.db,
1345
- `SELECT COUNT(*) as total FROM memory_helpfulness`
1346
- );
1347
-
1348
- return {
1349
- avgScore: Math.round(((stats?.avg_score as number) || 0) * 100) / 100,
1350
- totalEvaluated: (stats?.total_evaluated as number) || 0,
1351
- totalRetrievals: (totalRow?.total as number) || 0,
1352
- helpful: (stats?.helpful as number) || 0,
1353
- neutral: (stats?.neutral as number) || 0,
1354
- unhelpful: (stats?.unhelpful as number) || 0
1355
- };
1356
- }
1357
-
1358
- /**
1359
- * Fast keyword search using FTS5
1360
- * Returns events matching the search query, ranked by relevance
1361
- */
1362
- async keywordSearch(query: string, limit: number = 10): Promise<Array<{event: MemoryEvent; rank: number}>> {
1363
- await this.initialize();
1364
-
1365
- // Escape special FTS5 characters and prepare search terms
1366
- const searchTerms = query
1367
- .replace(/['"(){}[\]^~*?:\\/-]/g, ' ') // Remove special chars
1368
- .split(/\s+/)
1369
- .filter(term => term.length > 1) // Filter short terms
1370
- .map(term => `"${term}"*`) // Prefix matching
1371
- .join(' OR ');
1372
-
1373
- if (!searchTerms) {
1374
- return [];
1375
- }
1376
-
1377
- try {
1378
- const rows = sqliteAll<Record<string, unknown>>(
1379
- this.db,
1380
- `SELECT e.*, fts.rank
1381
- FROM events_fts fts
1382
- JOIN events e ON e.id = fts.event_id
1383
- WHERE events_fts MATCH ?
1384
- ORDER BY fts.rank
1385
- LIMIT ?`,
1386
- [searchTerms, limit]
1387
- );
1388
-
1389
- return rows.map(row => ({
1390
- event: this.rowToEvent(row),
1391
- rank: row.rank as number
1392
- }));
1393
- } catch (error: any) {
1394
- // FTS table might not exist yet (old database)
1395
- // Fallback to LIKE search
1396
- const likePattern = `%${query}%`;
1397
- const rows = sqliteAll<Record<string, unknown>>(
1398
- this.db,
1399
- `SELECT *, 0 as rank FROM events
1400
- WHERE content LIKE ?
1401
- ORDER BY timestamp DESC
1402
- LIMIT ?`,
1403
- [likePattern, limit]
1404
- );
1405
-
1406
- return rows.map(row => ({
1407
- event: this.rowToEvent(row),
1408
- rank: 0
1409
- }));
1410
- }
1411
- }
1412
-
1413
- /**
1414
- * Rebuild FTS index from existing events
1415
- * Call this once after upgrading to FTS5
1416
- */
1417
- async rebuildFtsIndex(): Promise<number> {
1418
- await this.initialize();
1419
-
1420
- // Get count of events to index
1421
- const countRow = sqliteGet<{count: number}>(this.db, 'SELECT COUNT(*) as count FROM events', []);
1422
- const totalEvents = countRow?.count ?? 0;
1423
-
1424
- // Clear and rebuild FTS index. Recreate the virtual table instead of
1425
- // issuing DELETE against it: older migrated FTS5 tables/triggers can fail
1426
- // with `no such column: T.event_id` when processing synthetic deletes.
1427
- sqliteExec(this.db, `
1428
- DROP TRIGGER IF EXISTS events_fts_insert;
1429
- DROP TRIGGER IF EXISTS events_fts_delete;
1430
- DROP TRIGGER IF EXISTS events_fts_update;
1431
- DROP TABLE IF EXISTS events_fts;
1432
-
1433
- CREATE VIRTUAL TABLE events_fts USING fts5(
1434
- content,
1435
- event_id UNINDEXED,
1436
- tokenize='porter unicode61'
1437
- );
1438
-
1439
- INSERT INTO events_fts(rowid, content, event_id)
1440
- SELECT rowid, content, id FROM events;
1441
-
1442
- CREATE TRIGGER events_fts_insert AFTER INSERT ON events BEGIN
1443
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1444
- END;
1445
-
1446
- CREATE TRIGGER events_fts_delete AFTER DELETE ON events BEGIN
1447
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1448
- END;
1449
-
1450
- CREATE TRIGGER events_fts_update AFTER UPDATE ON events BEGIN
1451
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1452
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1453
- END;
1454
- `);
1455
-
1456
- return totalEvents;
1457
- }
1458
-
1459
- /**
1460
- * Get database instance for direct access
1461
- */
1462
- getDatabase(): SQLiteDatabase {
1463
- return this.db;
1464
- }
1465
-
1466
-
1467
- async recordRetrievalTrace(input: {
1468
- sessionId?: string;
1469
- projectHash?: string;
1470
- queryText: string;
1471
- strategy?: string;
1472
- candidateEventIds: string[];
1473
- selectedEventIds: string[];
1474
- candidateDetails?: Array<{
1475
- eventId: string;
1476
- score: number;
1477
- semanticScore?: number;
1478
- lexicalScore?: number;
1479
- recencyScore?: number;
1480
- }>;
1481
- selectedDetails?: Array<{
1482
- eventId: string;
1483
- score: number;
1484
- semanticScore?: number;
1485
- lexicalScore?: number;
1486
- recencyScore?: number;
1487
- }>;
1488
- confidence?: string;
1489
- fallbackTrace?: string[];
1490
- }): Promise<void> {
1491
- await this.initialize();
1492
-
1493
- const traceId = randomUUID();
1494
- sqliteRun(
1495
- this.db,
1496
- `INSERT INTO retrieval_traces (
1497
- trace_id, session_id, project_hash, query_text, strategy,
1498
- candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
1499
- candidate_count, selected_count, confidence, fallback_trace
1500
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1501
- [
1502
- traceId,
1503
- input.sessionId || null,
1504
- input.projectHash || null,
1505
- input.queryText,
1506
- input.strategy || null,
1507
- JSON.stringify(input.candidateEventIds || []),
1508
- JSON.stringify(input.selectedEventIds || []),
1509
- JSON.stringify(input.candidateDetails || []),
1510
- JSON.stringify(input.selectedDetails || []),
1511
- (input.candidateEventIds || []).length,
1512
- (input.selectedEventIds || []).length,
1513
- input.confidence || null,
1514
- JSON.stringify(input.fallbackTrace || [])
1515
- ]
1516
- );
1517
- }
1518
-
1519
- async getRecentRetrievalTraces(limit: number = 50): Promise<Array<{
1520
- traceId: string;
1521
- sessionId?: string;
1522
- projectHash?: string;
1523
- queryText: string;
1524
- strategy?: string;
1525
- candidateEventIds: string[];
1526
- selectedEventIds: string[];
1527
- candidateDetails: Array<{
1528
- eventId: string;
1529
- score: number;
1530
- semanticScore?: number;
1531
- lexicalScore?: number;
1532
- recencyScore?: number;
1533
- }>;
1534
- selectedDetails: Array<{
1535
- eventId: string;
1536
- score: number;
1537
- semanticScore?: number;
1538
- lexicalScore?: number;
1539
- recencyScore?: number;
1540
- }>;
1541
- candidateCount: number;
1542
- selectedCount: number;
1543
- confidence?: string;
1544
- fallbackTrace: string[];
1545
- createdAt: Date;
1546
- }>> {
1547
- await this.initialize();
1548
-
1549
- try {
1550
- const rows = sqliteAll<Record<string, unknown>>(
1551
- this.db,
1552
- `SELECT * FROM retrieval_traces ORDER BY created_at DESC LIMIT ?`,
1553
- [limit]
1554
- );
1555
-
1556
- return rows.map((row) => ({
1557
- traceId: row.trace_id as string,
1558
- sessionId: (row.session_id as string) || undefined,
1559
- projectHash: (row.project_hash as string) || undefined,
1560
- queryText: row.query_text as string,
1561
- strategy: (row.strategy as string) || undefined,
1562
- candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids as string) : [],
1563
- selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids as string) : [],
1564
- candidateDetails: row.candidate_details_json ? JSON.parse(row.candidate_details_json as string) : [],
1565
- selectedDetails: row.selected_details_json ? JSON.parse(row.selected_details_json as string) : [],
1566
- candidateCount: Number(row.candidate_count || 0),
1567
- selectedCount: Number(row.selected_count || 0),
1568
- confidence: (row.confidence as string) || undefined,
1569
- fallbackTrace: row.fallback_trace ? JSON.parse(row.fallback_trace as string) : [],
1570
- createdAt: toDateFromSQLite(row.created_at),
1571
- }));
1572
- } catch (err: any) {
1573
- if (err?.message?.includes('no such table')) return [];
1574
- throw err;
1575
- }
1576
- }
1577
-
1578
- async getRetrievalTraceStats(): Promise<{
1579
- totalQueries: number;
1580
- avgCandidateCount: number;
1581
- avgSelectedCount: number;
1582
- selectionRate: number;
1583
- }> {
1584
- await this.initialize();
1585
-
1586
- try {
1587
- const row = sqliteGet<Record<string, unknown>>(
1588
- this.db,
1589
- `SELECT
1590
- COUNT(*) as total_queries,
1591
- AVG(candidate_count) as avg_candidate_count,
1592
- AVG(selected_count) as avg_selected_count,
1593
- CASE
1594
- WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
1595
- ELSE 0
1596
- END as selection_rate
1597
- FROM retrieval_traces`,
1598
- []
1599
- );
1600
-
1601
- return {
1602
- totalQueries: Number(row?.total_queries || 0),
1603
- avgCandidateCount: Number(row?.avg_candidate_count || 0),
1604
- avgSelectedCount: Number(row?.avg_selected_count || 0),
1605
- selectionRate: Number(row?.selection_rate || 0),
1606
- };
1607
- } catch (err: any) {
1608
- if (err?.message?.includes('no such table')) {
1609
- return { totalQueries: 0, avgCandidateCount: 0, avgSelectedCount: 0, selectionRate: 0 };
1610
- }
1611
- throw err;
1612
- }
1613
- }
1614
-
1615
- /**
1616
- * Close database connection
1617
- */
1618
- async close(): Promise<void> {
1619
- sqliteClose(this.db);
1620
- }
1621
-
1622
- /**
1623
- * Get events grouped by turn_id for a session
1624
- * Returns turns ordered by first event timestamp (newest first)
1625
- */
1626
- async getSessionTurns(sessionId: string, options?: { limit?: number; offset?: number }): Promise<Array<{
1627
- turnId: string;
1628
- events: MemoryEvent[];
1629
- startedAt: Date;
1630
- promptPreview: string;
1631
- eventCount: number;
1632
- toolCount: number;
1633
- hasResponse: boolean;
1634
- }>> {
1635
- await this.initialize();
1636
-
1637
- const limit = options?.limit || 20;
1638
- const offset = options?.offset || 0;
1639
-
1640
- // Get distinct turn_ids for this session, ordered by first event timestamp
1641
- const turnRows = sqliteAll<{ turn_id: string; min_ts: string }>(
1642
- this.db,
1643
- `SELECT turn_id, MIN(timestamp) as min_ts
1644
- FROM events
1645
- WHERE session_id = ? AND turn_id IS NOT NULL
1646
- GROUP BY turn_id
1647
- ORDER BY min_ts DESC
1648
- LIMIT ? OFFSET ?`,
1649
- [sessionId, limit, offset]
1650
- );
1651
-
1652
- const turns: Array<{
1653
- turnId: string;
1654
- events: MemoryEvent[];
1655
- startedAt: Date;
1656
- promptPreview: string;
1657
- eventCount: number;
1658
- toolCount: number;
1659
- hasResponse: boolean;
1660
- }> = [];
1661
-
1662
- for (const turnRow of turnRows) {
1663
- const events = await this.getEventsByTurn(turnRow.turn_id);
1664
-
1665
- const promptEvent = events.find(e => e.eventType === 'user_prompt');
1666
- const toolEvents = events.filter(e => e.eventType === 'tool_observation');
1667
- const hasResponse = events.some(e => e.eventType === 'agent_response');
1668
-
1669
- turns.push({
1670
- turnId: turnRow.turn_id,
1671
- events,
1672
- startedAt: toDateFromSQLite(turnRow.min_ts),
1673
- promptPreview: promptEvent
1674
- ? promptEvent.content.slice(0, 200) + (promptEvent.content.length > 200 ? '...' : '')
1675
- : '(no prompt)',
1676
- eventCount: events.length,
1677
- toolCount: toolEvents.length,
1678
- hasResponse
1679
- });
1680
- }
1681
-
1682
- return turns;
1683
- }
1684
-
1685
- /**
1686
- * Get all events for a specific turn_id
1687
- */
1688
- async getEventsByTurn(turnId: string): Promise<MemoryEvent[]> {
1689
- await this.initialize();
1690
-
1691
- const rows = sqliteAll<Record<string, unknown>>(
1692
- this.db,
1693
- `SELECT * FROM events WHERE turn_id = ? ORDER BY timestamp ASC`,
1694
- [turnId]
1695
- );
1696
-
1697
- return rows.map(this.rowToEvent);
1698
- }
1699
-
1700
- /**
1701
- * Count total turns for a session
1702
- */
1703
- async countSessionTurns(sessionId: string): Promise<number> {
1704
- await this.initialize();
1705
-
1706
- const row = sqliteGet<{ count: number }>(
1707
- this.db,
1708
- `SELECT COUNT(DISTINCT turn_id) as count
1709
- FROM events
1710
- WHERE session_id = ? AND turn_id IS NOT NULL`,
1711
- [sessionId]
1712
- );
1713
-
1714
- return row?.count || 0;
1715
- }
1716
-
1717
- /**
1718
- * Migrate existing events: backfill turn_id for events that have turnId in metadata
1719
- * but no turn_id column value (for events stored before this migration)
1720
- */
1721
- async backfillTurnIds(): Promise<number> {
1722
- await this.initialize();
1723
-
1724
- // Find events with turnId in metadata JSON but no turn_id column value
1725
- const rows = sqliteAll<{ id: string; metadata: string }>(
1726
- this.db,
1727
- `SELECT id, metadata FROM events
1728
- WHERE turn_id IS NULL AND metadata IS NOT NULL AND metadata LIKE '%turnId%'`
1729
- );
1730
-
1731
- let updated = 0;
1732
- for (const row of rows) {
1733
- try {
1734
- const metadata = JSON.parse(row.metadata);
1735
- if (metadata.turnId) {
1736
- sqliteRun(
1737
- this.db,
1738
- `UPDATE events SET turn_id = ? WHERE id = ?`,
1739
- [metadata.turnId, row.id]
1740
- );
1741
- updated++;
1742
- }
1743
- } catch {
1744
- // Skip rows with invalid JSON
1745
- }
1746
- }
1747
-
1748
- return updated;
1749
- }
1750
-
1751
- /**
1752
- * Delete all events for a session (for force reimport)
1753
- */
1754
- async deleteSessionEvents(sessionId: string): Promise<number> {
1755
- await this.initialize();
1756
-
1757
- // Get event IDs first for cascading deletes
1758
- const events = sqliteAll<{ id: string }>(
1759
- this.db,
1760
- `SELECT id FROM events WHERE session_id = ?`,
1761
- [sessionId]
1762
- );
1763
-
1764
- if (events.length === 0) return 0;
1765
-
1766
- const eventIds = events.map(e => e.id);
1767
- const placeholders = eventIds.map(() => '?').join(',');
1768
-
1769
- // Drop FTS triggers to prevent SQLITE_CORRUPT_VTAB during bulk delete
1770
- const ftsTriggersDropped: string[] = [];
1771
- for (const triggerName of ['events_fts_delete', 'events_fts_update', 'events_fts_insert']) {
1772
- try {
1773
- sqliteRun(this.db, `DROP TRIGGER IF EXISTS ${triggerName}`);
1774
- ftsTriggersDropped.push(triggerName);
1775
- } catch {
1776
- // Trigger may not exist
1777
- }
1778
- }
1779
-
1780
- // Delete from related tables first (some may not exist depending on DB version)
1781
- for (const table of ['event_dedup', 'memory_levels', 'embedding_queue', 'embedding_outbox', 'vector_outbox']) {
1782
- try {
1783
- sqliteRun(this.db, `DELETE FROM ${table} WHERE event_id IN (${placeholders})`, eventIds);
1784
- } catch {
1785
- // Table may not exist
1786
- }
1787
- }
1788
-
1789
- // Delete events
1790
- const result = sqliteRun(this.db, `DELETE FROM events WHERE session_id = ?`, [sessionId]);
1791
-
1792
- // Rebuild FTS index if we dropped triggers
1793
- if (ftsTriggersDropped.length > 0) {
1794
- try {
1795
- // Rebuild FTS from remaining events
1796
- sqliteRun(this.db, `INSERT INTO events_fts(events_fts) VALUES('rebuild')`);
1797
-
1798
- // Recreate triggers
1799
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_insert AFTER INSERT ON events BEGIN
1800
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1801
- END`);
1802
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_delete AFTER DELETE ON events BEGIN
1803
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1804
- END`);
1805
- sqliteRun(this.db, `CREATE TRIGGER IF NOT EXISTS events_fts_update AFTER UPDATE ON events BEGIN
1806
- DELETE FROM events_fts WHERE rowid = OLD.rowid;
1807
- INSERT INTO events_fts(rowid, content, event_id) VALUES (NEW.rowid, NEW.content, NEW.id);
1808
- END`);
1809
- } catch {
1810
- // FTS rebuild failed - non-critical, will be rebuilt on next initialize
1811
- }
1812
- }
1813
-
1814
- return result.changes || 0;
1815
- }
1816
-
1817
- /**
1818
- * Convert database row to MemoryEvent
1819
- */
1820
- private rowToEvent(row: Record<string, unknown>): MemoryEvent {
1821
- const event: any = {
1822
- id: row.id as string,
1823
- eventType: row.event_type as 'user_prompt' | 'agent_response' | 'session_summary',
1824
- sessionId: row.session_id as string,
1825
- timestamp: toDateFromSQLite(row.timestamp),
1826
- content: row.content as string,
1827
- canonicalKey: row.canonical_key as string,
1828
- dedupeKey: row.dedupe_key as string,
1829
- metadata: row.metadata ? JSON.parse(row.metadata as string) : undefined
1830
- };
1831
-
1832
- // Include access tracking fields if present
1833
- if (row.access_count !== undefined) {
1834
- event.access_count = row.access_count;
1835
- }
1836
- if (row.last_accessed_at !== undefined) {
1837
- event.last_accessed_at = row.last_accessed_at;
1838
- }
1839
- // Include turn_id if present
1840
- if (row.turn_id !== undefined && row.turn_id !== null) {
1841
- event.turn_id = row.turn_id;
1842
- }
1843
-
1844
- return event;
1845
- }
1846
- }