@bryti/agent 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/Dockerfile +27 -0
  2. package/README.md +77 -50
  3. package/config.example.yml +265 -0
  4. package/dist/active-hours.d.ts +23 -0
  5. package/dist/active-hours.d.ts.map +1 -0
  6. package/dist/active-hours.js +68 -0
  7. package/dist/active-hours.js.map +1 -0
  8. package/dist/agent.d.ts +84 -0
  9. package/dist/agent.d.ts.map +1 -0
  10. package/dist/agent.js +383 -0
  11. package/dist/agent.js.map +1 -0
  12. package/dist/channels/markdown/ir.d.ts +79 -0
  13. package/dist/channels/markdown/ir.d.ts.map +1 -0
  14. package/dist/channels/markdown/ir.js +824 -0
  15. package/dist/channels/markdown/ir.js.map +1 -0
  16. package/dist/channels/markdown/render.d.ts +35 -0
  17. package/dist/channels/markdown/render.d.ts.map +1 -0
  18. package/dist/channels/markdown/render.js +178 -0
  19. package/dist/channels/markdown/render.js.map +1 -0
  20. package/dist/channels/telegram-network-errors.d.ts +27 -0
  21. package/dist/channels/telegram-network-errors.d.ts.map +1 -0
  22. package/dist/channels/telegram-network-errors.js +156 -0
  23. package/dist/channels/telegram-network-errors.js.map +1 -0
  24. package/dist/channels/telegram.d.ts +76 -0
  25. package/dist/channels/telegram.d.ts.map +1 -0
  26. package/dist/channels/telegram.js +814 -0
  27. package/dist/channels/telegram.js.map +1 -0
  28. package/dist/channels/types.d.ts +59 -0
  29. package/dist/channels/types.d.ts.map +1 -0
  30. package/dist/channels/types.js +9 -0
  31. package/dist/channels/types.js.map +1 -0
  32. package/dist/channels/whatsapp.d.ts +45 -0
  33. package/dist/channels/whatsapp.d.ts.map +1 -0
  34. package/dist/channels/whatsapp.js +310 -0
  35. package/dist/channels/whatsapp.js.map +1 -0
  36. package/dist/cli.d.ts +13 -0
  37. package/dist/cli.d.ts.map +1 -0
  38. package/dist/cli.js +635 -0
  39. package/dist/cli.js.map +1 -0
  40. package/dist/commands.d.ts +35 -0
  41. package/dist/commands.d.ts.map +1 -0
  42. package/dist/commands.js +113 -0
  43. package/dist/commands.js.map +1 -0
  44. package/dist/compaction/history.d.ts +17 -0
  45. package/dist/compaction/history.d.ts.map +1 -0
  46. package/dist/compaction/history.js +35 -0
  47. package/dist/compaction/history.js.map +1 -0
  48. package/dist/compaction/index.d.ts +3 -0
  49. package/dist/compaction/index.d.ts.map +1 -0
  50. package/dist/compaction/index.js +3 -0
  51. package/dist/compaction/index.js.map +1 -0
  52. package/dist/compaction/proactive.d.ts +25 -0
  53. package/dist/compaction/proactive.d.ts.map +1 -0
  54. package/dist/compaction/proactive.js +87 -0
  55. package/dist/compaction/proactive.js.map +1 -0
  56. package/dist/compaction/transcript-repair.d.ts +55 -0
  57. package/dist/compaction/transcript-repair.d.ts.map +1 -0
  58. package/dist/compaction/transcript-repair.js +215 -0
  59. package/dist/compaction/transcript-repair.js.map +1 -0
  60. package/dist/config.d.ts +128 -0
  61. package/dist/config.d.ts.map +1 -0
  62. package/dist/config.js +317 -0
  63. package/dist/config.js.map +1 -0
  64. package/dist/crash-recovery.d.ts +23 -0
  65. package/dist/crash-recovery.d.ts.map +1 -0
  66. package/dist/crash-recovery.js +96 -0
  67. package/dist/crash-recovery.js.map +1 -0
  68. package/dist/defaults/extensions/EXTENSIONS.md +158 -0
  69. package/dist/defaults/extensions/documents-hedgedoc.ts +153 -0
  70. package/dist/history.d.ts +31 -0
  71. package/dist/history.d.ts.map +1 -0
  72. package/dist/history.js +49 -0
  73. package/dist/history.js.map +1 -0
  74. package/dist/index.d.ts +19 -0
  75. package/dist/index.d.ts.map +1 -0
  76. package/dist/index.js +673 -0
  77. package/dist/index.js.map +1 -0
  78. package/dist/logger.d.ts +39 -0
  79. package/dist/logger.d.ts.map +1 -0
  80. package/dist/logger.js +143 -0
  81. package/dist/logger.js.map +1 -0
  82. package/dist/memory/conversation-search.d.ts +15 -0
  83. package/dist/memory/conversation-search.d.ts.map +1 -0
  84. package/dist/memory/conversation-search.js +60 -0
  85. package/dist/memory/conversation-search.js.map +1 -0
  86. package/dist/memory/core-memory.d.ts +28 -0
  87. package/dist/memory/core-memory.d.ts.map +1 -0
  88. package/dist/memory/core-memory.js +102 -0
  89. package/dist/memory/core-memory.js.map +1 -0
  90. package/dist/memory/embeddings.d.ts +44 -0
  91. package/dist/memory/embeddings.d.ts.map +1 -0
  92. package/dist/memory/embeddings.js +139 -0
  93. package/dist/memory/embeddings.js.map +1 -0
  94. package/dist/memory/search.d.ts +49 -0
  95. package/dist/memory/search.d.ts.map +1 -0
  96. package/dist/memory/search.js +97 -0
  97. package/dist/memory/search.js.map +1 -0
  98. package/dist/memory/store.d.ts +32 -0
  99. package/dist/memory/store.d.ts.map +1 -0
  100. package/dist/memory/store.js +205 -0
  101. package/dist/memory/store.js.map +1 -0
  102. package/dist/message-queue.d.ts +73 -0
  103. package/dist/message-queue.d.ts.map +1 -0
  104. package/dist/message-queue.js +188 -0
  105. package/dist/message-queue.js.map +1 -0
  106. package/dist/model-infra.d.ts +64 -0
  107. package/dist/model-infra.d.ts.map +1 -0
  108. package/dist/model-infra.js +202 -0
  109. package/dist/model-infra.js.map +1 -0
  110. package/dist/projection/format.d.ts +10 -0
  111. package/dist/projection/format.d.ts.map +1 -0
  112. package/dist/projection/format.js +30 -0
  113. package/dist/projection/format.js.map +1 -0
  114. package/dist/projection/index.d.ts +11 -0
  115. package/dist/projection/index.d.ts.map +1 -0
  116. package/dist/projection/index.js +9 -0
  117. package/dist/projection/index.js.map +1 -0
  118. package/dist/projection/reflection.d.ts +94 -0
  119. package/dist/projection/reflection.d.ts.map +1 -0
  120. package/dist/projection/reflection.js +334 -0
  121. package/dist/projection/reflection.js.map +1 -0
  122. package/dist/projection/store.d.ts +144 -0
  123. package/dist/projection/store.d.ts.map +1 -0
  124. package/dist/projection/store.js +519 -0
  125. package/dist/projection/store.js.map +1 -0
  126. package/dist/projection/tools.d.ts +11 -0
  127. package/dist/projection/tools.d.ts.map +1 -0
  128. package/dist/projection/tools.js +237 -0
  129. package/dist/projection/tools.js.map +1 -0
  130. package/dist/scheduler.d.ts +36 -0
  131. package/dist/scheduler.d.ts.map +1 -0
  132. package/dist/scheduler.js +286 -0
  133. package/dist/scheduler.js.map +1 -0
  134. package/dist/system-prompt.d.ts +41 -0
  135. package/dist/system-prompt.d.ts.map +1 -0
  136. package/dist/system-prompt.js +162 -0
  137. package/dist/system-prompt.js.map +1 -0
  138. package/dist/time.d.ts +52 -0
  139. package/dist/time.d.ts.map +1 -0
  140. package/dist/time.js +138 -0
  141. package/dist/time.js.map +1 -0
  142. package/dist/tools/archival-memory-tool.d.ts +8 -0
  143. package/dist/tools/archival-memory-tool.d.ts.map +1 -0
  144. package/dist/tools/archival-memory-tool.js +68 -0
  145. package/dist/tools/archival-memory-tool.js.map +1 -0
  146. package/dist/tools/conversation-search-tool.d.ts +6 -0
  147. package/dist/tools/conversation-search-tool.d.ts.map +1 -0
  148. package/dist/tools/conversation-search-tool.js +28 -0
  149. package/dist/tools/conversation-search-tool.js.map +1 -0
  150. package/dist/tools/core-memory-tool.d.ts +7 -0
  151. package/dist/tools/core-memory-tool.d.ts.map +1 -0
  152. package/dist/tools/core-memory-tool.js +59 -0
  153. package/dist/tools/core-memory-tool.js.map +1 -0
  154. package/dist/tools/fetch-url.d.ts +15 -0
  155. package/dist/tools/fetch-url.d.ts.map +1 -0
  156. package/dist/tools/fetch-url.js +76 -0
  157. package/dist/tools/fetch-url.js.map +1 -0
  158. package/dist/tools/files.d.ts +10 -0
  159. package/dist/tools/files.d.ts.map +1 -0
  160. package/dist/tools/files.js +127 -0
  161. package/dist/tools/files.js.map +1 -0
  162. package/dist/tools/index.d.ts +17 -0
  163. package/dist/tools/index.d.ts.map +1 -0
  164. package/dist/tools/index.js +118 -0
  165. package/dist/tools/index.js.map +1 -0
  166. package/dist/tools/result.d.ts +21 -0
  167. package/dist/tools/result.d.ts.map +1 -0
  168. package/dist/tools/result.js +36 -0
  169. package/dist/tools/result.js.map +1 -0
  170. package/dist/tools/skill-install.d.ts +17 -0
  171. package/dist/tools/skill-install.d.ts.map +1 -0
  172. package/dist/tools/skill-install.js +148 -0
  173. package/dist/tools/skill-install.js.map +1 -0
  174. package/dist/tools/web-search.d.ts +42 -0
  175. package/dist/tools/web-search.d.ts.map +1 -0
  176. package/dist/tools/web-search.js +237 -0
  177. package/dist/tools/web-search.js.map +1 -0
  178. package/dist/trust/guardrail.d.ts +60 -0
  179. package/dist/trust/guardrail.d.ts.map +1 -0
  180. package/dist/trust/guardrail.js +171 -0
  181. package/dist/trust/guardrail.js.map +1 -0
  182. package/dist/trust/index.d.ts +12 -0
  183. package/dist/trust/index.d.ts.map +1 -0
  184. package/dist/trust/index.js +12 -0
  185. package/dist/trust/index.js.map +1 -0
  186. package/dist/trust/store.d.ts +118 -0
  187. package/dist/trust/store.d.ts.map +1 -0
  188. package/dist/trust/store.js +209 -0
  189. package/dist/trust/store.js.map +1 -0
  190. package/dist/trust/wrapper.d.ts +36 -0
  191. package/dist/trust/wrapper.d.ts.map +1 -0
  192. package/dist/trust/wrapper.js +142 -0
  193. package/dist/trust/wrapper.js.map +1 -0
  194. package/dist/usage.d.ts +53 -0
  195. package/dist/usage.d.ts.map +1 -0
  196. package/dist/usage.js +124 -0
  197. package/dist/usage.js.map +1 -0
  198. package/dist/util/math.d.ts +9 -0
  199. package/dist/util/math.d.ts.map +1 -0
  200. package/dist/util/math.js +22 -0
  201. package/dist/util/math.js.map +1 -0
  202. package/dist/util/ssrf.d.ts +21 -0
  203. package/dist/util/ssrf.d.ts.map +1 -0
  204. package/dist/util/ssrf.js +77 -0
  205. package/dist/util/ssrf.js.map +1 -0
  206. package/dist/workers/index.d.ts +8 -0
  207. package/dist/workers/index.d.ts.map +1 -0
  208. package/dist/workers/index.js +7 -0
  209. package/dist/workers/index.js.map +1 -0
  210. package/dist/workers/registry.d.ts +53 -0
  211. package/dist/workers/registry.d.ts.map +1 -0
  212. package/dist/workers/registry.js +38 -0
  213. package/dist/workers/registry.js.map +1 -0
  214. package/dist/workers/scoped-tools.d.ts +21 -0
  215. package/dist/workers/scoped-tools.d.ts.map +1 -0
  216. package/dist/workers/scoped-tools.js +111 -0
  217. package/dist/workers/scoped-tools.js.map +1 -0
  218. package/dist/workers/spawn.d.ts +62 -0
  219. package/dist/workers/spawn.d.ts.map +1 -0
  220. package/dist/workers/spawn.js +314 -0
  221. package/dist/workers/spawn.js.map +1 -0
  222. package/dist/workers/tools.d.ts +26 -0
  223. package/dist/workers/tools.d.ts.map +1 -0
  224. package/dist/workers/tools.js +380 -0
  225. package/dist/workers/tools.js.map +1 -0
  226. package/docker-compose.yml +72 -0
  227. package/package.json +16 -1
  228. package/run.sh +27 -0
@@ -0,0 +1,139 @@
1
+ /**
2
+ * Local embeddings via node-llama-cpp.
3
+ *
4
+ * Singleton pattern: one Llama instance, one model, one embedding context,
5
+ * shared across all calls for the lifetime of the process. The model weighs
6
+ * 300MB+, so loading it per-call or per-request is not viable. Instead,
7
+ * getEmbeddingContext() initialises everything on first call and caches the
8
+ * result; every subsequent call returns the cached context immediately.
9
+ *
10
+ * Model files live in <dataDir>/.models/.
11
+ */
12
+ import { getLlama, LlamaLogLevel, resolveModelFile } from "node-llama-cpp";
13
+ // Hugging Face URI in node-llama-cpp's "hf:<owner>/<repo>/<file>" format.
14
+ // On first use, node-llama-cpp resolves this automatically: it locates (or
15
+ // downloads) the file and caches it in the modelsDir supplied at init time.
16
+ const EMBEDDING_MODEL_URI = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
17
+ let llamaInstance = null;
18
+ let llamaModel = null;
19
+ let embeddingContext = null;
20
+ // In-flight initialisation guard. Without this, concurrent calls to
21
+ // getEmbeddingContext() before the first load completes would each kick off a
22
+ // separate model load. Storing the Promise here means every concurrent caller
23
+ // awaits the same in-progress load instead of starting a new one.
24
+ let initPromise = null;
25
+ /**
26
+ * Initialize and return the embedding context, loading the model on first call.
27
+ * Subsequent calls return the cached context.
28
+ */
29
+ async function getEmbeddingContext(modelsDir) {
30
+ if (embeddingContext !== null) {
31
+ return embeddingContext;
32
+ }
33
+ if (initPromise !== null) {
34
+ return initPromise;
35
+ }
36
+ initPromise = (async () => {
37
+ const llama = await getLlama({
38
+ gpu: "auto",
39
+ logger(level, message) {
40
+ // 'special_eos_id is not in special_eog_ids' is a benign metadata quirk
41
+ // in the embedding model's GGUF file: the end-of-sequence token id is
42
+ // not listed in the end-of-generation set. It has no effect on embedding
43
+ // quality and is safe to ignore.
44
+ if (level === LlamaLogLevel.warn && message.includes("special_eos_id is not in special_eog_ids")) {
45
+ return;
46
+ }
47
+ if (level === LlamaLogLevel.error || level === LlamaLogLevel.fatal) {
48
+ console.error("[llama]", message);
49
+ }
50
+ },
51
+ });
52
+ llamaInstance = llama;
53
+ const modelPath = await resolveModelFile(EMBEDDING_MODEL_URI, {
54
+ directory: modelsDir,
55
+ cli: false,
56
+ onProgress({ totalSize, downloadedSize }) {
57
+ const pct = totalSize > 0 ? Math.round((downloadedSize / totalSize) * 100) : 0;
58
+ process.stdout.write(`\rDownloading embedding model: ${pct}%`);
59
+ },
60
+ });
61
+ process.stdout.write("\n");
62
+ const model = await llama.loadModel({ modelPath });
63
+ llamaModel = model;
64
+ const ctx = await model.createEmbeddingContext();
65
+ embeddingContext = ctx;
66
+ return ctx;
67
+ })();
68
+ return initPromise;
69
+ }
70
+ /**
71
+ * Generate an embedding for a single text.
72
+ *
73
+ * @param text Input text (must be non-empty)
74
+ * @param modelsDir Directory to store/load the model (defaults to node-llama-cpp global dir)
75
+ */
76
+ export async function embed(text, modelsDir) {
77
+ if (!text.trim()) {
78
+ throw new Error("Embedding input is empty");
79
+ }
80
+ const ctx = await getEmbeddingContext(modelsDir);
81
+ const result = await ctx.getEmbeddingFor(text);
82
+ return Array.from(result.vector);
83
+ }
84
+ /**
85
+ * Generate embeddings for multiple texts. Sequential; the model is fast
86
+ * enough on CPU that batching provides no meaningful advantage.
87
+ */
88
+ export async function embedBatch(texts, modelsDir) {
89
+ if (texts.length === 0) {
90
+ return [];
91
+ }
92
+ for (const text of texts) {
93
+ if (!text.trim()) {
94
+ throw new Error("Embedding input is empty");
95
+ }
96
+ }
97
+ const results = [];
98
+ for (const text of texts) {
99
+ results.push(await embed(text, modelsDir));
100
+ }
101
+ return results;
102
+ }
103
+ /**
104
+ * Pre-load the embedding model at startup.
105
+ *
106
+ * Calling this eagerly means a missing or corrupt model file surfaces as a
107
+ * startup error rather than failing silently on the first user message that
108
+ * triggers a memory operation. Also amortises the cold-start download/load
109
+ * time before any user is waiting.
110
+ *
111
+ * @param modelsDir Directory to store/load the model
112
+ */
113
+ export async function warmupEmbeddings(modelsDir) {
114
+ await getEmbeddingContext(modelsDir);
115
+ }
116
+ /**
117
+ * Release the embedding context, model, and Llama instance.
118
+ *
119
+ * Call this on graceful shutdown. node-llama-cpp allocates native (non-GC)
120
+ * resources for model weights and inference threads; skipping dispose leaves
121
+ * those resources live until the OS reclaims them, and can prevent the Node
122
+ * process from exiting cleanly.
123
+ */
124
+ export async function disposeEmbeddings() {
125
+ if (embeddingContext) {
126
+ await embeddingContext.dispose();
127
+ embeddingContext = null;
128
+ }
129
+ if (llamaModel) {
130
+ await llamaModel.dispose();
131
+ llamaModel = null;
132
+ }
133
+ if (llamaInstance) {
134
+ await llamaInstance.dispose();
135
+ llamaInstance = null;
136
+ }
137
+ initPromise = null;
138
+ }
139
+ //# sourceMappingURL=embeddings.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/memory/embeddings.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAG3E,0EAA0E;AAC1E,2EAA2E;AAC3E,4EAA4E;AAC5E,MAAM,mBAAmB,GACvB,oEAAoE,CAAC;AAEvE,IAAI,aAAa,GAAiB,IAAI,CAAC;AACvC,IAAI,UAAU,GAAsB,IAAI,CAAC;AACzC,IAAI,gBAAgB,GAAiC,IAAI,CAAC;AAC1D,oEAAoE;AACpE,8EAA8E;AAC9E,8EAA8E;AAC9E,kEAAkE;AAClE,IAAI,WAAW,GAA0C,IAAI,CAAC;AAE9D;;;GAGG;AACH,KAAK,UAAU,mBAAmB,CAAC,SAAkB;IACnD,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QAC9B,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;QACxB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC;YAC3B,GAAG,EAAE,MAAM;YACX,MAAM,CAAC,KAAK,EAAE,OAAO;gBACnB,wEAAwE;gBACxE,sEAAsE;gBACtE,yEAAyE;gBACzE,iCAAiC;gBACjC,IAAI,KAAK,KAAK,aAAa,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC,EAAE,CAAC;oBACjG,OAAO;gBACT,CAAC;gBACD,IAAI,KAAK,KAAK,aAAa,CAAC,KAAK,IAAI,KAAK,KAAK,aAAa,CAAC,KAAK,EAAE,CAAC;oBACnE,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,aAAa,GAAG,KAAK,CAAC;QAEtB,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,mBAAmB,EAAE;YAC5D,SAAS,EAAE,SAAS;YACpB,GAAG,EAAE,KAAK;YACV,UAAU,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE;gBACtC,MAAM,GAAG,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,GAAG,GAAG,CAAC,CAAC;YACjE,CAAC;SACF,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;QACnD,UAAU,GAAG,KAAK,CAAC;QACnB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,sBAAsB,EAAE,CAAC;QAEjD,gBAAgB,GAAG,GAAG,CAAC;QACvB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAY,EAAE,SAAkB;IAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe,EAAE,SAAkB;IAClE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAkB;IACvD,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC;QACjC,gBAAgB,GAAG,IAAI,CAAC;IAC1B,CAAC;IACD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;QAC3B,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;QAC9B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,WAAW,GAAG,IAAI,CAAC;AACrB,CAAC"}
@@ -0,0 +1,49 @@
1
+ /**
2
+ * Hybrid search: keyword (FTS5) + vector, fused with Reciprocal Rank Fusion.
3
+ *
4
+ * RRF formula: score(doc) = w_vec * 1/(k + rank_vec) + w_kw * 1/(k + rank_kw)
5
+ * Defaults: 0.7 vector, 0.3 keyword, k = 60.
6
+ *
7
+ * Why RRF instead of a weighted average of raw scores?
8
+ * FTS5 BM25 scores and cosine similarity values live on completely different
9
+ * scales, so adding them directly produces garbage. RRF converts each result
10
+ * list to ranks first, making the fusion scale-invariant: it doesn't matter
11
+ * what units either scorer uses.
12
+ */
13
+ import type { MemoryStore, ScoredResult } from "./store.js";
14
+ export interface SearchResult extends ScoredResult {
15
+ /** Combined score from both methods. */
16
+ combinedScore: number;
17
+ /** Which methods matched (for debugging/display). */
18
+ matchedBy: ("keyword" | "vector")[];
19
+ }
20
+ export interface HybridSearchOptions {
21
+ /** Weight for vector search results (default: 0.7) */
22
+ vectorWeight?: number;
23
+ /** Weight for keyword search results (default: 0.3) */
24
+ keywordWeight?: number;
25
+ /**
26
+ * Fusion smoothing constant k (default: 60).
27
+ * Standard RRF value from the original paper. Lower values amplify the
28
+ * advantage of top-ranked documents; 60 is empirically good across most
29
+ * retrieval tasks and avoids over-rewarding a single strong signal.
30
+ */
31
+ k?: number;
32
+ /** Maximum results to return (default: 5) */
33
+ limit?: number;
34
+ }
35
+ /**
36
+ * Create a hybrid search function over the given memory store.
37
+ *
38
+ * Returns a closure that accepts a query string. The closure calls `embed` on
39
+ * every invocation to produce a fresh query embedding; embeddings are not
40
+ * cached here. Callers that need caching should wrap `embed` before passing
41
+ * it in.
42
+ *
43
+ * @param store The memory store to search against (provides keyword + vector).
44
+ * @param embed Embedding function — called once per query, not cached.
45
+ * @param options Weights, k, and result limit overrides.
46
+ * @returns An async function `(query) => SearchResult[]` ready for use.
47
+ */
48
+ export declare function createHybridSearch(store: MemoryStore, embed: (text: string) => Promise<number[]>, options?: HybridSearchOptions): (query: string) => Promise<SearchResult[]>;
49
+ //# sourceMappingURL=search.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/memory/search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE5D,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,wCAAwC;IACxC,aAAa,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,SAAS,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,mBAAmB;IAClC,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AASD;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,EAC1C,OAAO,GAAE,mBAAwB,GAChC,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAyE5C"}
@@ -0,0 +1,97 @@
1
+ /**
2
+ * Hybrid search: keyword (FTS5) + vector, fused with Reciprocal Rank Fusion.
3
+ *
4
+ * RRF formula: score(doc) = w_vec * 1/(k + rank_vec) + w_kw * 1/(k + rank_kw)
5
+ * Defaults: 0.7 vector, 0.3 keyword, k = 60.
6
+ *
7
+ * Why RRF instead of a weighted average of raw scores?
8
+ * FTS5 BM25 scores and cosine similarity values live on completely different
9
+ * scales, so adding them directly produces garbage. RRF converts each result
10
+ * list to ranks first, making the fusion scale-invariant: it doesn't matter
11
+ * what units either scorer uses.
12
+ */
13
+ const DEFAULT_OPTIONS = {
14
+ vectorWeight: 0.7,
15
+ keywordWeight: 0.3,
16
+ k: 60,
17
+ limit: 5,
18
+ };
19
+ /**
20
+ * Create a hybrid search function over the given memory store.
21
+ *
22
+ * Returns a closure that accepts a query string. The closure calls `embed` on
23
+ * every invocation to produce a fresh query embedding; embeddings are not
24
+ * cached here. Callers that need caching should wrap `embed` before passing
25
+ * it in.
26
+ *
27
+ * @param store The memory store to search against (provides keyword + vector).
28
+ * @param embed Embedding function — called once per query, not cached.
29
+ * @param options Weights, k, and result limit overrides.
30
+ * @returns An async function `(query) => SearchResult[]` ready for use.
31
+ */
32
+ export function createHybridSearch(store, embed, options = {}) {
33
+ const opts = { ...DEFAULT_OPTIONS, ...options };
34
+ return async function hybridSearch(query) {
35
+ // Handle empty query
36
+ if (!query.trim()) {
37
+ return [];
38
+ }
39
+ // Get embedding for query
40
+ const queryEmbedding = await embed(query);
41
+ // Run both searches in parallel
42
+ const [keywordResults, vectorResults] = await Promise.all([
43
+ store.searchKeyword(query, opts.limit * 2), // Get more to account for dedup
44
+ store.searchVector(queryEmbedding, opts.limit * 2),
45
+ ]);
46
+ // Handle empty results
47
+ if (keywordResults.length === 0 && vectorResults.length === 0) {
48
+ return [];
49
+ }
50
+ // Both searches can return the same document. Merge by id: the first time
51
+ // a document appears its entry is created; the second time its score is
52
+ // accumulated and the matching method is appended to matchedBy.
53
+ const resultsMap = new Map();
54
+ // Process keyword results — applies the RRF term: w_kw * 1/(k + rank_kw)
55
+ for (let i = 0; i < keywordResults.length; i++) {
56
+ const result = keywordResults[i];
57
+ const rank = i + 1; // RRF ranks are 1-based
58
+ const score = opts.keywordWeight * (1 / (opts.k + rank));
59
+ if (resultsMap.has(result.id)) {
60
+ const existing = resultsMap.get(result.id);
61
+ existing.combinedScore += score;
62
+ existing.matchedBy.push("keyword");
63
+ }
64
+ else {
65
+ resultsMap.set(result.id, {
66
+ ...result,
67
+ combinedScore: score,
68
+ matchedBy: ["keyword"],
69
+ });
70
+ }
71
+ }
72
+ // Process vector results — applies the RRF term: w_vec * 1/(k + rank_vec)
73
+ for (let i = 0; i < vectorResults.length; i++) {
74
+ const result = vectorResults[i];
75
+ const rank = i + 1; // RRF ranks are 1-based
76
+ const score = opts.vectorWeight * (1 / (opts.k + rank));
77
+ if (resultsMap.has(result.id)) {
78
+ const existing = resultsMap.get(result.id);
79
+ existing.combinedScore += score;
80
+ existing.matchedBy.push("vector");
81
+ }
82
+ else {
83
+ resultsMap.set(result.id, {
84
+ ...result,
85
+ combinedScore: score,
86
+ matchedBy: ["vector"],
87
+ });
88
+ }
89
+ }
90
+ // Convert to array and sort by combined score
91
+ const mergedResults = Array.from(resultsMap.values())
92
+ .sort((a, b) => b.combinedScore - a.combinedScore)
93
+ .slice(0, opts.limit);
94
+ return mergedResults;
95
+ };
96
+ }
97
+ //# sourceMappingURL=search.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/memory/search.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AA2BH,MAAM,eAAe,GAAkC;IACrD,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,GAAG;IAClB,CAAC,EAAE,EAAE;IACL,KAAK,EAAE,CAAC;CACT,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAkB,EAClB,KAA0C,EAC1C,UAA+B,EAAE;IAEjC,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAEhD,OAAO,KAAK,UAAU,YAAY,CAAC,KAAa;QAC9C,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,0BAA0B;QAC1B,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1C,gCAAgC;QAChC,MAAM,CAAC,cAAc,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACxD,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,gCAAgC;YAC5E,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;SACnD,CAAC,CAAC;QAEH,uBAAuB;QACvB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9D,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,0EAA0E;QAC1E,wEAAwE;QACxE,gEAAgE;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAC;QAEnD,yEAAyE;QACzE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YAEzD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;gBAC5C,QAAQ,CAAC,aAAa,IAAI,KAAK,CAAC;gBAChC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE;oBACxB,GAAG,MAAM;oBACT,aAAa,EAAE,KAAK;oBACpB,SAAS,EAAE,CAAC,SAAS,CAAC;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,0EAA0E;QAC1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YAExD,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;gBAC5C,QAAQ,CAAC,aAAa,IAAI,KAAK,CAAC;gBAChC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE;oBACxB,GAAG,MAAM;oBACT,aAAa,EAAE,KAAK;oBACpB,SAAS,EAAE,CAAC,QAAQ,CAAC;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,8CAA8C;QAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;aAClD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC;aACjD,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAExB,OAAO,aAAa,CAAC;IACvB,CAAC,CAAC;AACJ,CAAC"}
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Per-user memory store backed by SQLite with FTS5 for keyword search and
3
+ * binary Float32Array blobs for embeddings. Vector search runs in-memory
4
+ * via cosine similarity.
5
+ */
6
+ export interface ScoredResult {
7
+ id: string;
8
+ content: string;
9
+ source: string;
10
+ timestamp: number;
11
+ score: number;
12
+ }
13
+ export interface MemoryStore {
14
+ /** Add a fact to the store. Returns the fact ID. */
15
+ addFact(content: string, source: string, embedding: number[]): string;
16
+ /** Remove a fact by ID. */
17
+ removeFact(id: string): void;
18
+ /** Search using keyword (FTS5). */
19
+ searchKeyword(query: string, limit: number): ScoredResult[];
20
+ /** Search using vector similarity (in-memory cosine similarity). */
21
+ searchVector(embedding: number[], limit: number): ScoredResult[];
22
+ /** Close the database connection. */
23
+ close(): void;
24
+ }
25
+ /**
26
+ * Create a per-user memory store.
27
+ *
28
+ * @param userId User ID for isolation
29
+ * @param dataDir Base data directory
30
+ */
31
+ export declare function createMemoryStore(userId: string, dataDir: string): MemoryStore;
32
+ //# sourceMappingURL=store.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/memory/store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,oDAAoD;IACpD,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAEtE,2BAA2B;IAC3B,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAE7B,mCAAmC;IACnC,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IAE5D,oEAAoE;IACpE,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,CAAC;IAEjE,qCAAqC;IACrC,KAAK,IAAI,IAAI,CAAC;CACf;AAuBD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,WAAW,CA6M9E"}
@@ -0,0 +1,205 @@
1
+ /**
2
+ * Per-user memory store backed by SQLite with FTS5 for keyword search and
3
+ * binary Float32Array blobs for embeddings. Vector search runs in-memory
4
+ * via cosine similarity.
5
+ */
6
+ import Database from "better-sqlite3";
7
+ import { cosineSimilarity } from "../util/math.js";
8
+ import fs from "node:fs";
9
+ import path from "node:path";
10
+ import crypto from "node:crypto";
11
+ /**
12
+ * Serialize an embedding vector to a raw binary buffer.
13
+ * Stored as Float32Array bytes (4 bytes per dimension) rather than JSON,
14
+ * which avoids the significant parse/stringify overhead for ~300-dim vectors
15
+ * and halves the storage footprint compared to text representation.
16
+ */
17
+ function serializeEmbedding(embedding) {
18
+ return Buffer.from(new Float32Array(embedding).buffer);
19
+ }
20
+ /**
21
+ * Deserialize a raw binary buffer back to a number array.
22
+ * Inverse of serializeEmbedding: reads raw Float32 bytes from the blob column.
23
+ */
24
+ function deserializeEmbedding(buffer) {
25
+ const array = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
26
+ return Array.from(array);
27
+ }
28
+ /**
29
+ * Create a per-user memory store.
30
+ *
31
+ * @param userId User ID for isolation
32
+ * @param dataDir Base data directory
33
+ */
34
+ export function createMemoryStore(userId, dataDir) {
35
+ // Schema overview:
36
+ // facts — main table; one row per stored fact (id, content, source, timestamp, hash)
37
+ // facts_fts — FTS5 virtual table shadowing facts.content for BM25 keyword search
38
+ // fact_embeddings — binary blob table keyed by the same id as facts
39
+ //
40
+ // Three triggers (facts_ai, facts_ad, facts_au) keep facts_fts in sync with
41
+ // facts automatically on insert, delete, and update.
42
+ const userDir = path.join(dataDir, "users", userId);
43
+ fs.mkdirSync(userDir, { recursive: true });
44
+ const dbPath = path.join(userDir, "memory.db");
45
+ const db = new Database(dbPath);
46
+ // Enable WAL mode for concurrent reads
47
+ db.pragma("journal_mode = WAL");
48
+ // Create tables
49
+ db.exec(`
50
+ -- Main facts table
51
+ CREATE TABLE IF NOT EXISTS facts (
52
+ id TEXT PRIMARY KEY,
53
+ content TEXT NOT NULL,
54
+ source TEXT NOT NULL,
55
+ timestamp INTEGER NOT NULL,
56
+ hash TEXT NOT NULL
57
+ );
58
+
59
+ -- FTS5 virtual table for keyword search
60
+ CREATE VIRTUAL TABLE IF NOT EXISTS facts_fts USING fts5(
61
+ content,
62
+ content='facts',
63
+ content_rowid='rowid'
64
+ );
65
+
66
+ -- Triggers to keep FTS in sync
67
+ CREATE TRIGGER IF NOT EXISTS facts_ai AFTER INSERT ON facts BEGIN
68
+ INSERT INTO facts_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
69
+ END;
70
+
71
+ CREATE TRIGGER IF NOT EXISTS facts_ad AFTER DELETE ON facts BEGIN
72
+ INSERT INTO facts_fts(facts_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
73
+ END;
74
+
75
+ CREATE TRIGGER IF NOT EXISTS facts_au AFTER UPDATE ON facts BEGIN
76
+ INSERT INTO facts_fts(facts_fts, rowid, content) VALUES('delete', OLD.rowid, OLD.content);
77
+ INSERT INTO facts_fts(rowid, content) VALUES (NEW.rowid, NEW.content);
78
+ END;
79
+
80
+ -- Table to store embeddings (binary Float32Array)
81
+ CREATE TABLE IF NOT EXISTS fact_embeddings (
82
+ id TEXT PRIMARY KEY,
83
+ embedding BLOB NOT NULL,
84
+ FOREIGN KEY (id) REFERENCES facts(id) ON DELETE CASCADE
85
+ );
86
+ `);
87
+ // Prepared statements for efficiency
88
+ const insertFact = db.prepare(`
89
+ INSERT INTO facts (id, content, source, timestamp, hash)
90
+ VALUES (?, ?, ?, ?, ?)
91
+ `);
92
+ const insertEmbedding = db.prepare(`
93
+ INSERT INTO fact_embeddings (id, embedding) VALUES (?, ?)
94
+ `);
95
+ const deleteFact = db.prepare(`
96
+ DELETE FROM facts WHERE id = ?
97
+ `);
98
+ const deleteEmbedding = db.prepare(`
99
+ DELETE FROM fact_embeddings WHERE id = ?
100
+ `);
101
+ const selectEmbeddings = db.prepare(`
102
+ SELECT f.id, f.content, f.source, f.timestamp, fe.embedding
103
+ FROM facts f
104
+ JOIN fact_embeddings fe ON f.id = fe.id
105
+ `);
106
+ return {
107
+ /**
108
+ * Add a fact to the store and return its generated UUID.
109
+ *
110
+ * @param content The fact text to store and index.
111
+ * @param source Provenance label (e.g. "reflection", "user").
112
+ * @param embedding Pre-computed embedding vector for this content.
113
+ *
114
+ * The `hash` field is a truncated SHA-256 of the content. It is used to
115
+ * deduplicate facts during bulk imports, not as an integrity check — the
116
+ * store does not verify it on read.
117
+ */
118
+ addFact(content, source, embedding) {
119
+ const id = crypto.randomUUID();
120
+ const timestamp = Date.now();
121
+ const hash = crypto.createHash("sha256").update(content).digest("hex").slice(0, 16);
122
+ // Insert into facts table
123
+ insertFact.run(id, content, source, timestamp, hash);
124
+ // Insert embedding as binary blob
125
+ insertEmbedding.run(id, serializeEmbedding(embedding));
126
+ return id;
127
+ },
128
+ removeFact(id) {
129
+ deleteEmbedding.run(id);
130
+ deleteFact.run(id);
131
+ },
132
+ /**
133
+ * Search facts by keyword using SQLite FTS5.
134
+ *
135
+ * Scoring uses FTS5's built-in BM25 implementation (via the `bm25()`
136
+ * function), which ranks results by term frequency and inverse document
137
+ * frequency. Returns at most `limit` results ordered by relevance.
138
+ */
139
+ searchKeyword(query, limit) {
140
+ if (!query.trim()) {
141
+ return [];
142
+ }
143
+ // Use FTS5 match with BM25 ranking.
144
+ // Wrapping the query in double quotes makes it a phrase query, which
145
+ // neutralises FTS5 operators (OR, AND, NOT, NEAR, *, etc.).
146
+ // Double quotes inside the phrase would break out, so we escape them
147
+ // by doubling ("" is the FTS5 escape for a literal double quote).
148
+ // Single quotes are stripped to avoid tokenizer surprises.
149
+ const escapedQuery = query.replace(/'/g, "").replace(/"/g, '""');
150
+ const stmt = db.prepare(`
151
+ SELECT f.id, f.content, f.source, f.timestamp,
152
+ bm25(facts_fts) as score
153
+ FROM facts_fts
154
+ JOIN facts f ON facts_fts.rowid = f.rowid
155
+ WHERE facts_fts MATCH ?
156
+ ORDER BY bm25(facts_fts)
157
+ LIMIT ?
158
+ `);
159
+ const results = stmt.all(`"${escapedQuery}"`, limit);
160
+ return results.map((row) => ({
161
+ id: row.id,
162
+ content: row.content,
163
+ source: row.source,
164
+ timestamp: row.timestamp,
165
+ score: row.score,
166
+ }));
167
+ },
168
+ /**
169
+ * Search facts by vector similarity.
170
+ *
171
+ * This is a full table scan: all embeddings are loaded from SQLite into
172
+ * memory, cosine similarity is computed for each one, and the top `limit`
173
+ * results are returned. This is acceptable up to roughly 100K facts.
174
+ * Beyond that, an approximate nearest-neighbour index would be needed
175
+ * (hnswlib-node or the sqlite-vec extension are natural fits here).
176
+ *
177
+ * TODO: add ANN index when fact count regularly exceeds ~100K per user.
178
+ */
179
+ searchVector(embedding, limit) {
180
+ const rows = selectEmbeddings.all();
181
+ if (rows.length === 0) {
182
+ return [];
183
+ }
184
+ // Compute cosine similarity for each fact
185
+ const scored = rows
186
+ .map((row) => ({
187
+ ...row,
188
+ similarity: cosineSimilarity(embedding, deserializeEmbedding(row.embedding)),
189
+ }))
190
+ .sort((a, b) => b.similarity - a.similarity)
191
+ .slice(0, limit);
192
+ return scored.map((row) => ({
193
+ id: row.id,
194
+ content: row.content,
195
+ source: row.source,
196
+ timestamp: row.timestamp,
197
+ score: row.similarity,
198
+ }));
199
+ },
200
+ close() {
201
+ db.close();
202
+ },
203
+ };
204
+ }
205
+ //# sourceMappingURL=store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/memory/store.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AA6BjC;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,SAAmB;IAC7C,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,MAAc;IAC1C,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACxF,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc,EAAE,OAAe;IAC/D,mBAAmB;IACnB,gGAAgG;IAChG,wFAAwF;IACxF,sEAAsE;IACtE,EAAE;IACF,4EAA4E;IAC5E,qDAAqD;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACpD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAEhC,uCAAuC;IACvC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAEhC,gBAAgB;IAChB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCP,CAAC,CAAC;IAEH,qCAAqC;IACrC,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAG7B,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO,CAAC;;GAElC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC;;GAE7B,CAAC,CAAC;IAEH,MAAM,eAAe,GAAG,EAAE,CAAC,OAAO,CAAC;;GAElC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAInC,CAAC,CAAC;IAEH,OAAO;QACL;;;;;;;;;;WAUG;QACH,OAAO,CAAC,OAAe,EAAE,MAAc,EAAE,SAAmB;YAC1D,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;YAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAEpF,0BAA0B;YAC1B,UAAU,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YAErD,kCAAkC;YAClC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC;YAEvD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,UAAU,CAAC,EAAU;YACnB,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxB,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC;QAED;;;;;;WAMG;QACH,aAAa,CAAC,KAAa,EAAE,KAAa;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBAClB,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,oCAAoC;YACpC,qEAAqE;YACrE,4DAA4D;YAC5D,qEAAqE;YACrE,kEAAkE;YAClE,2DAA2D;YAC3D,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAEjE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;OAQvB,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,YAAY,GAAG,EAAE,KAAK,CAMjD,CAAC;YAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC3B,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC,CAAC;QACN,CAAC;QAED;;;;;;;;;;WAUG;QACH,YAAY,CAAC,SAAmB,EAAE,KAAa;YAC7C,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,EAM/B,CAAC;YAEH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,0CAA0C;YAC1C,MAAM,MAAM,GAAG,IAAI;iBAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACb,GAAG,GAAG;gBACN,UAAU,EAAE,gBAAgB,CAAC,SAAS,EAAE,oBAAoB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;aAC7E,CAAC,CAAC;iBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;iBAC3C,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAEnB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC1B,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,KAAK,EAAE,GAAG,CAAC,UAAU;aACtB,CAAC,CAAC,CAAC;QACN,CAAC;QAED,KAAK;YACH,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Per-channel FIFO message queue with merge and backpressure.
3
+ *
4
+ * Two core guarantees:
5
+ *
6
+ * 1. Per-channel serialization: only one message is processed at a time per
7
+ * channel. Subsequent messages queue up behind it rather than racing into
8
+ * the agent loop in parallel.
9
+ *
10
+ * 2. Burst merging: rapid-fire messages that arrive within MERGE_WINDOW_MS of
11
+ * each other are joined into a single prompt before being dispatched. This
12
+ * handles the common "user sends three quick messages" pattern without the
13
+ * agent seeing three separate incomplete thoughts.
14
+ *
15
+ * New messages queue up to MAX_DEPTH; beyond that the caller gets a rejection
16
+ * callback (backpressure signal, not silent drop).
17
+ */
18
+ import type { IncomingMessage } from "./channels/types.js";
19
+ type ProcessFn = (msg: IncomingMessage) => Promise<void>;
20
+ type RejectFn = (msg: IncomingMessage) => Promise<void>;
21
+ /**
22
+ * Serialises processing per channel and merges rapid follow-up messages.
23
+ */
24
+ export declare class MessageQueue {
25
+ private readonly queues;
26
+ private readonly processFn;
27
+ private readonly rejectFn;
28
+ private readonly maxDepth;
29
+ private readonly mergeWindowMs;
30
+ private readonly rateLimiter;
31
+ constructor(processFn: ProcessFn, rejectFn: RejectFn, maxDepth?: number, mergeWindowMs?: number);
32
+ /**
33
+ * Enqueue a message. If nothing is processing for this channel, starts
34
+ * draining immediately. Rate-limited per user (10 messages/minute).
35
+ */
36
+ enqueue(msg: IncomingMessage): void;
37
+ /**
38
+ * Drain the queue for a channel sequentially, merging close messages.
39
+ */
40
+ private drain;
41
+ /**
42
+ * Take a batch of entries that should be merged together.
43
+ *
44
+ * The first entry is always included. Additional entries are included if they
45
+ * arrived within mergeWindowMs of the FIRST entry in the batch. This is a
46
+ * fixed window anchored to the first message, not a sliding window — an
47
+ * entry that arrives 1ms after the previous one is still excluded if it
48
+ * falls outside the window from the first entry.
49
+ */
50
+ private takeMergeBatch;
51
+ /**
52
+ * Merge multiple queue entries into a single IncomingMessage by joining
53
+ * their text with newlines. Metadata (userId, channelId, platform, etc.) is
54
+ * taken from the first entry.
55
+ *
56
+ * Note: images (and other non-text attachments) from subsequent burst
57
+ * entries are currently dropped — only their text is merged.
58
+ * TODO: carry images from all burst entries into the merged message.
59
+ */
60
+ private mergeEntries;
61
+ /**
62
+ * Number of queued (not-yet-processing) messages for a channel.
63
+ * Exposed for monitoring dashboards and unit tests.
64
+ */
65
+ queueDepth(channelId: string): number;
66
+ /**
67
+ * Whether the channel is currently mid-process (drain loop running).
68
+ * Exposed for monitoring dashboards and unit tests.
69
+ */
70
+ isProcessing(channelId: string): boolean;
71
+ }
72
+ export {};
73
+ //# sourceMappingURL=message-queue.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"message-queue.d.ts","sourceRoot":"","sources":["../src/message-queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAmB3D,KAAK,SAAS,GAAG,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AACzD,KAAK,QAAQ,GAAG,CAAC,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AA+CxD;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAC1D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAGxC,SAAS,EAAE,SAAS,EACpB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,SAAY,EACpB,aAAa,SAAkB;IASjC;;;OAGG;IACH,OAAO,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI;IAkCnC;;OAEG;YACW,KAAK;IAmBnB;;;;;;;;OAQG;IACH,OAAO,CAAC,cAAc;IAiBtB;;;;;;;;OAQG;IACH,OAAO,CAAC,YAAY;IAYpB;;;OAGG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM;IAIrC;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;CAGzC"}