@melandlabs/memory-store 0.1.6 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-D6MzksfT.d.ts → config-BYfTzG3h.d.ts} +17 -0
- package/dist/contracts.d.ts +7 -0
- package/dist/contracts.js +0 -2
- package/dist/http.d.ts +1 -1
- package/dist/http.js +355 -67
- package/dist/index.d.ts +2 -2
- package/dist/index.js +287 -52
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +313 -60
- package/dist/policies/memory-graph-correction-policy.js +0 -2
- package/dist/policies/memory-graph-write-policy.js +0 -2
- package/dist/search/unified-search.d.ts +39 -3
- package/dist/search/unified-search.js +311 -56
- package/dist/server/cli-http.js +360 -66
- package/dist/server/cli-mcp.js +318 -59
- package/dist/server/cli-shared.d.ts +1 -1
- package/dist/server/cli-shared.js +0 -2
- package/dist/storage/chroma-memory-index.js +0 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +1 -1
- package/dist/storage/postgres-raw-message-factory.js +0 -2
- package/dist/storage/raw-message-store.d.ts +9 -1
- package/dist/storage/raw-message-store.js +0 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +15 -2
- package/dist/storage/sqlite-raw-message-store.js +8 -3
- package/dist/storage/sqlite-vector-index.d.ts +1 -1
- package/dist/storage/sqlite-vector-index.js +0 -2
- package/package.json +81 -93
- package/dist/contracts.js.map +0 -1
- package/dist/http.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/mcp.js.map +0 -1
- package/dist/policies/memory-graph-correction-policy.js.map +0 -1
- package/dist/policies/memory-graph-write-policy.js.map +0 -1
- package/dist/search/unified-search.js.map +0 -1
- package/dist/server/cli-http.js.map +0 -1
- package/dist/server/cli-mcp.js.map +0 -1
- package/dist/server/cli-shared.js.map +0 -1
- package/dist/storage/chroma-memory-index.js.map +0 -1
- package/dist/storage/postgres-raw-message-factory.js.map +0 -1
- package/dist/storage/raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-vector-index.js.map +0 -1
package/dist/server/cli-http.js
CHANGED
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { serve } from '@hono/node-server';
|
|
3
|
-
import { Hono } from 'hono';
|
|
4
2
|
import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
|
|
5
3
|
import { ChromaVectorStore } from '@melandlabs/rag';
|
|
6
4
|
import { mkdirSync } from 'fs';
|
|
7
5
|
import { homedir } from 'os';
|
|
8
|
-
import {
|
|
6
|
+
import { dirname, join } from 'path';
|
|
9
7
|
import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
|
|
8
|
+
import { serve } from '@hono/node-server';
|
|
9
|
+
import { Hono } from 'hono';
|
|
10
|
+
|
|
11
|
+
var __defProp = Object.defineProperty;
|
|
12
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
13
|
+
var __esm = (fn, res) => function __init() {
|
|
14
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
15
|
+
};
|
|
16
|
+
var __export = (target, all) => {
|
|
17
|
+
for (var name in all)
|
|
18
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
19
|
+
};
|
|
10
20
|
|
|
11
21
|
// src/contracts.ts
|
|
12
22
|
function normalizeTimestampToMs(value) {
|
|
@@ -30,14 +40,10 @@ function rawMessageToMemoryRecord(message) {
|
|
|
30
40
|
embeddingContentHash: message.embeddingContentHash
|
|
31
41
|
};
|
|
32
42
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
"RAW_MESSAGE_VECTOR_STORE_BACKEND",
|
|
38
|
-
"MEMORY_VECTOR_STORE_BACKEND",
|
|
39
|
-
"VECTOR_STORE_BACKEND"
|
|
40
|
-
];
|
|
43
|
+
var init_contracts = __esm({
|
|
44
|
+
"src/contracts.ts"() {
|
|
45
|
+
}
|
|
46
|
+
});
|
|
41
47
|
function getBackend(keys) {
|
|
42
48
|
for (const key of keys) {
|
|
43
49
|
const value = process.env[key]?.trim().toLowerCase();
|
|
@@ -140,21 +146,51 @@ async function searchRawMessagesWithChroma(input) {
|
|
|
140
146
|
metadata: result.metadata ?? {}
|
|
141
147
|
})).slice(0, input.limit);
|
|
142
148
|
}
|
|
149
|
+
var DEFAULT_RAW_MESSAGES_COLLECTION, RAW_VECTOR_BACKEND_ENV_KEYS;
|
|
150
|
+
var init_chroma_memory_index = __esm({
|
|
151
|
+
"src/storage/chroma-memory-index.ts"() {
|
|
152
|
+
init_contracts();
|
|
153
|
+
DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
|
|
154
|
+
RAW_VECTOR_BACKEND_ENV_KEYS = [
|
|
155
|
+
"RAW_MESSAGE_VECTOR_STORE_BACKEND",
|
|
156
|
+
"MEMORY_VECTOR_STORE_BACKEND",
|
|
157
|
+
"VECTOR_STORE_BACKEND"
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
});
|
|
143
161
|
|
|
144
162
|
// src/storage/postgres-raw-message-factory.ts
|
|
145
|
-
var factory = null;
|
|
146
163
|
async function resolvePostgresFactory(env) {
|
|
147
|
-
return null;
|
|
164
|
+
if (!factory) return null;
|
|
165
|
+
return await factory(env);
|
|
148
166
|
}
|
|
149
167
|
function hasPostgresFactory() {
|
|
150
168
|
return factory !== null;
|
|
151
169
|
}
|
|
152
|
-
var
|
|
153
|
-
var
|
|
170
|
+
var factory;
|
|
171
|
+
var init_postgres_raw_message_factory = __esm({
|
|
172
|
+
"src/storage/postgres-raw-message-factory.ts"() {
|
|
173
|
+
factory = null;
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
// src/storage/sqlite-raw-message-store.ts
|
|
178
|
+
var sqlite_raw_message_store_exports = {};
|
|
179
|
+
__export(sqlite_raw_message_store_exports, {
|
|
180
|
+
__resetSQLiteRawMessageManagerForTests: () => __resetSQLiteRawMessageManagerForTests,
|
|
181
|
+
closeSQLiteRawMessageManager: () => closeSQLiteRawMessageManager,
|
|
182
|
+
getSQLiteRawMessageManager: () => getSQLiteRawMessageManager,
|
|
183
|
+
isSQLiteRawMessageStorageAvailable: () => isSQLiteRawMessageStorageAvailable,
|
|
184
|
+
lexicalSearchRawMessages: () => lexicalSearchRawMessages,
|
|
185
|
+
resolveSQLiteRawMessageDbPath: () => resolveSQLiteRawMessageDbPath
|
|
186
|
+
});
|
|
154
187
|
function resolveSQLiteRawMessageDbPath() {
|
|
155
188
|
const fromEnv = process.env.MEMORY_STORE_DB_PATH?.trim();
|
|
156
189
|
return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_SQLITE_PATH;
|
|
157
190
|
}
|
|
191
|
+
function isSQLiteRawMessageStorageAvailable(_env) {
|
|
192
|
+
return true;
|
|
193
|
+
}
|
|
158
194
|
async function getSQLiteRawMessageManager(_env) {
|
|
159
195
|
if (!manager) {
|
|
160
196
|
const dbPath = resolveSQLiteRawMessageDbPath();
|
|
@@ -174,8 +210,35 @@ async function closeSQLiteRawMessageManager() {
|
|
|
174
210
|
await manager.close();
|
|
175
211
|
manager = null;
|
|
176
212
|
}
|
|
213
|
+
function __resetSQLiteRawMessageManagerForTests() {
|
|
214
|
+
manager = null;
|
|
215
|
+
}
|
|
216
|
+
async function lexicalSearchRawMessages(input) {
|
|
217
|
+
const mgr = await getSQLiteRawMessageManager();
|
|
218
|
+
if (typeof mgr.lexicalSearchMessages !== "function") {
|
|
219
|
+
return [];
|
|
220
|
+
}
|
|
221
|
+
return mgr.lexicalSearchMessages(input);
|
|
222
|
+
}
|
|
223
|
+
var manager, DEFAULT_SQLITE_PATH;
|
|
224
|
+
var init_sqlite_raw_message_store = __esm({
|
|
225
|
+
"src/storage/sqlite-raw-message-store.ts"() {
|
|
226
|
+
init_chroma_memory_index();
|
|
227
|
+
manager = null;
|
|
228
|
+
DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
|
|
229
|
+
}
|
|
230
|
+
});
|
|
177
231
|
|
|
178
232
|
// src/storage/raw-message-store.ts
|
|
233
|
+
var raw_message_store_exports = {};
|
|
234
|
+
__export(raw_message_store_exports, {
|
|
235
|
+
closeRawMessageStore: () => closeRawMessageStore,
|
|
236
|
+
configureRawMessageStore: () => configureRawMessageStore,
|
|
237
|
+
createRawMessageStore: () => createRawMessageStore,
|
|
238
|
+
getRawMessageManager: () => getRawMessageManager,
|
|
239
|
+
getRawMessageStorageBackend: () => getRawMessageStorageBackend,
|
|
240
|
+
isRawMessageStorageAvailable: () => isRawMessageStorageAvailable
|
|
241
|
+
});
|
|
179
242
|
function resolveBackend(env) {
|
|
180
243
|
const explicit = process.env.OPENCONTEXT_MEMORY_STORE_BACKEND?.trim().toLowerCase();
|
|
181
244
|
if (explicit === "postgres") return "postgres";
|
|
@@ -185,7 +248,7 @@ function resolveEnv(_env) {
|
|
|
185
248
|
return {};
|
|
186
249
|
}
|
|
187
250
|
function createRawMessageStore(options = {}) {
|
|
188
|
-
resolveEnv(options.env);
|
|
251
|
+
const env = resolveEnv(options.env);
|
|
189
252
|
return {
|
|
190
253
|
getBackend() {
|
|
191
254
|
return resolveBackend();
|
|
@@ -195,7 +258,7 @@ function createRawMessageStore(options = {}) {
|
|
|
195
258
|
},
|
|
196
259
|
async getManager() {
|
|
197
260
|
if (resolveBackend() === "postgres") {
|
|
198
|
-
const pg = await resolvePostgresFactory();
|
|
261
|
+
const pg = await resolvePostgresFactory(env);
|
|
199
262
|
if (!pg) {
|
|
200
263
|
throw new Error(
|
|
201
264
|
"OPENCONTEXT_MEMORY_STORE_BACKEND=postgres but no Postgres factory is registered. Call registerPostgresFactory() at startup, or unset the env var to use the default SQLite backend."
|
|
@@ -210,14 +273,47 @@ function createRawMessageStore(options = {}) {
|
|
|
210
273
|
}
|
|
211
274
|
};
|
|
212
275
|
}
|
|
213
|
-
|
|
214
|
-
|
|
276
|
+
function configureRawMessageStore(config) {
|
|
277
|
+
moduleConfig = config;
|
|
278
|
+
moduleStore = createRawMessageStore({ env: config.env });
|
|
279
|
+
return moduleStore;
|
|
280
|
+
}
|
|
281
|
+
function getRawMessageStorageBackend() {
|
|
282
|
+
if (!moduleStore) {
|
|
283
|
+
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
284
|
+
}
|
|
285
|
+
return moduleStore.getBackend();
|
|
286
|
+
}
|
|
215
287
|
function isRawMessageStorageAvailable() {
|
|
216
288
|
if (!moduleStore) {
|
|
217
289
|
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
218
290
|
}
|
|
219
291
|
return moduleStore.isAvailable();
|
|
220
292
|
}
|
|
293
|
+
async function getRawMessageManager() {
|
|
294
|
+
if (!moduleStore) {
|
|
295
|
+
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
296
|
+
}
|
|
297
|
+
return moduleStore.getManager();
|
|
298
|
+
}
|
|
299
|
+
async function closeRawMessageStore() {
|
|
300
|
+
if (!moduleStore) return;
|
|
301
|
+
await moduleStore.close();
|
|
302
|
+
moduleStore = null;
|
|
303
|
+
}
|
|
304
|
+
var moduleStore, moduleConfig;
|
|
305
|
+
var init_raw_message_store = __esm({
|
|
306
|
+
"src/storage/raw-message-store.ts"() {
|
|
307
|
+
init_postgres_raw_message_factory();
|
|
308
|
+
init_sqlite_raw_message_store();
|
|
309
|
+
moduleStore = null;
|
|
310
|
+
moduleConfig = null;
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
// src/search/unified-search.ts
|
|
315
|
+
init_chroma_memory_index();
|
|
316
|
+
init_raw_message_store();
|
|
221
317
|
|
|
222
318
|
// src/search/utilities.ts
|
|
223
319
|
var DEFAULT_LIMIT = 10;
|
|
@@ -247,7 +343,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
247
343
|
}
|
|
248
344
|
return Math.min(1, Math.max(-1, parsed));
|
|
249
345
|
}
|
|
250
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
346
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
347
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
348
|
+
if (strategy === "rrf") {
|
|
349
|
+
const lists = options.rankedLists ?? [
|
|
350
|
+
{ name: "memory-semantic", hits: results }
|
|
351
|
+
];
|
|
352
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
353
|
+
}
|
|
251
354
|
return [...results].sort((a, b) => {
|
|
252
355
|
const scoreDelta = b.similarity - a.similarity;
|
|
253
356
|
if (scoreDelta !== 0) {
|
|
@@ -256,6 +359,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
256
359
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
257
360
|
}).slice(0, limit);
|
|
258
361
|
}
|
|
362
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
363
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
364
|
+
}
|
|
365
|
+
var DEFAULT_RRF_K = 60;
|
|
366
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
367
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
368
|
+
return [];
|
|
369
|
+
}
|
|
370
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
371
|
+
const scores = /* @__PURE__ */ new Map();
|
|
372
|
+
const order = [];
|
|
373
|
+
for (const list of lists) {
|
|
374
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
375
|
+
const hit = list.hits[index];
|
|
376
|
+
const key = `${hit.type}::${hit.id}`;
|
|
377
|
+
const contribution = 1 / (safeK + index + 1);
|
|
378
|
+
const existing = scores.get(key);
|
|
379
|
+
if (existing) {
|
|
380
|
+
existing.rrf += contribution;
|
|
381
|
+
} else {
|
|
382
|
+
scores.set(key, { hit, rrf: contribution });
|
|
383
|
+
order.push(key);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
388
|
+
if (b.rrf !== a.rrf) {
|
|
389
|
+
return b.rrf - a.rrf;
|
|
390
|
+
}
|
|
391
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
392
|
+
}).slice(0, limit);
|
|
393
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
394
|
+
...hit,
|
|
395
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
396
|
+
}));
|
|
397
|
+
}
|
|
259
398
|
function toKnowledgeResult(result) {
|
|
260
399
|
return {
|
|
261
400
|
type: "knowledge",
|
|
@@ -287,6 +426,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
287
426
|
}
|
|
288
427
|
|
|
289
428
|
// src/search/unified-search.ts
|
|
429
|
+
function deriveLexicalKeywords(query) {
|
|
430
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
431
|
+
}
|
|
290
432
|
function createUnifiedSearch(deps = {}) {
|
|
291
433
|
const logger = deps.logger ?? console;
|
|
292
434
|
async function searchUnifiedMemory(input) {
|
|
@@ -294,8 +436,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
294
436
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
295
437
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
296
438
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
439
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
297
440
|
const warnings = [];
|
|
298
|
-
const results = [];
|
|
299
441
|
if (!query) {
|
|
300
442
|
return {
|
|
301
443
|
query,
|
|
@@ -305,17 +447,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
305
447
|
warnings
|
|
306
448
|
};
|
|
307
449
|
}
|
|
450
|
+
let memorySubs;
|
|
308
451
|
if (sources.includes("memory")) {
|
|
309
452
|
if (isRawMessageStorageAvailable()) {
|
|
310
453
|
try {
|
|
311
|
-
|
|
312
|
-
results.push(...memoryHits);
|
|
454
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
313
455
|
} catch (error) {
|
|
314
456
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
315
457
|
warnings.push({
|
|
316
458
|
source: "memory",
|
|
317
459
|
code: "memory_search_failed",
|
|
318
|
-
message: error.message ?? "
|
|
460
|
+
message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
|
|
319
461
|
});
|
|
320
462
|
}
|
|
321
463
|
} else {
|
|
@@ -326,9 +468,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
326
468
|
});
|
|
327
469
|
}
|
|
328
470
|
}
|
|
471
|
+
const insightHits = [];
|
|
329
472
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
330
473
|
try {
|
|
331
|
-
const
|
|
474
|
+
const hits = await deps.searchInsights({
|
|
332
475
|
userId: input.userId,
|
|
333
476
|
query: input.query,
|
|
334
477
|
limit,
|
|
@@ -337,8 +480,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
337
480
|
includeArchived: input.includeArchivedInsights,
|
|
338
481
|
authToken: input.authToken
|
|
339
482
|
});
|
|
340
|
-
for (const hit of
|
|
341
|
-
|
|
483
|
+
for (const hit of hits) {
|
|
484
|
+
insightHits.push({
|
|
342
485
|
type: "insight",
|
|
343
486
|
id: hit.id,
|
|
344
487
|
content: hit.content,
|
|
@@ -354,16 +497,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
354
497
|
message: error.message ?? "insights_search_failed"
|
|
355
498
|
});
|
|
356
499
|
}
|
|
357
|
-
} else if (sources.includes("insights"))
|
|
358
|
-
|
|
359
|
-
source: "insights",
|
|
360
|
-
code: "insights_search_not_configured",
|
|
361
|
-
message: "Insights search is not configured for this host."
|
|
362
|
-
});
|
|
363
|
-
}
|
|
500
|
+
} else if (sources.includes("insights")) ;
|
|
501
|
+
const knowledgeHits = [];
|
|
364
502
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
365
503
|
try {
|
|
366
|
-
const
|
|
504
|
+
const hits = await deps.searchKnowledge({
|
|
367
505
|
userId: input.userId,
|
|
368
506
|
query: input.query,
|
|
369
507
|
options: {
|
|
@@ -373,8 +511,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
373
511
|
},
|
|
374
512
|
authToken: input.authToken
|
|
375
513
|
});
|
|
376
|
-
for (const hit of
|
|
377
|
-
|
|
514
|
+
for (const hit of hits) {
|
|
515
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
378
516
|
}
|
|
379
517
|
} catch (error) {
|
|
380
518
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -384,14 +522,14 @@ function createUnifiedSearch(deps = {}) {
|
|
|
384
522
|
message: error.message ?? "knowledge_search_failed"
|
|
385
523
|
});
|
|
386
524
|
}
|
|
387
|
-
} else if (sources.includes("knowledge"))
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
525
|
+
} else if (sources.includes("knowledge")) ;
|
|
526
|
+
const merged = mergeAcrossSources({
|
|
527
|
+
memorySubs,
|
|
528
|
+
insightHits,
|
|
529
|
+
knowledgeHits,
|
|
530
|
+
limit,
|
|
531
|
+
strategy: mergeStrategy
|
|
532
|
+
});
|
|
395
533
|
return {
|
|
396
534
|
query: input.query,
|
|
397
535
|
sources,
|
|
@@ -400,9 +538,35 @@ function createUnifiedSearch(deps = {}) {
|
|
|
400
538
|
warnings
|
|
401
539
|
};
|
|
402
540
|
}
|
|
403
|
-
async function runMemorySource(input, limit, threshold) {
|
|
541
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
404
542
|
if (typeof deps.embedQuery !== "function") {
|
|
405
|
-
|
|
543
|
+
warnings.push({
|
|
544
|
+
source: "memory",
|
|
545
|
+
code: "memory_lexical_search_fallback",
|
|
546
|
+
message: "Semantic search not configured, using keyword search as fallback"
|
|
547
|
+
});
|
|
548
|
+
const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
|
|
549
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
550
|
+
let lexical2 = [];
|
|
551
|
+
if (keywords.length > 0) {
|
|
552
|
+
try {
|
|
553
|
+
const filters2 = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
554
|
+
const results = await Promise.all(
|
|
555
|
+
filters2.map(
|
|
556
|
+
(botId) => lexicalSearchRawMessages2({
|
|
557
|
+
userId: input.userId,
|
|
558
|
+
keywords,
|
|
559
|
+
limit: Math.ceil(limit / filters2.length),
|
|
560
|
+
botId
|
|
561
|
+
})
|
|
562
|
+
)
|
|
563
|
+
);
|
|
564
|
+
lexical2 = results.flat().filter(Boolean).map(toMemoryResult);
|
|
565
|
+
} catch (error) {
|
|
566
|
+
logger.warn?.("[memory-store] Default lexical search failed:", error);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
return { semantic: [], lexical: lexical2 };
|
|
406
570
|
}
|
|
407
571
|
const queryEmbedding = await deps.embedQuery({
|
|
408
572
|
userId: input.userId,
|
|
@@ -410,9 +574,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
410
574
|
authToken: input.authToken
|
|
411
575
|
});
|
|
412
576
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
577
|
+
let semantic = [];
|
|
413
578
|
if (isRawMessageChromaEnabled()) {
|
|
414
579
|
try {
|
|
415
|
-
|
|
580
|
+
semantic = (await Promise.all(
|
|
416
581
|
filters.map((filter) => {
|
|
417
582
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
418
583
|
return searchRawMessagesWithChroma({
|
|
@@ -427,18 +592,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
427
592
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
428
593
|
backend: "chroma",
|
|
429
594
|
dimensions: queryEmbedding.length,
|
|
430
|
-
count:
|
|
595
|
+
count: semantic.length
|
|
431
596
|
});
|
|
432
|
-
return chromaHits;
|
|
433
597
|
} catch (error) {
|
|
434
598
|
logger.warn?.(
|
|
435
599
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
436
600
|
error
|
|
437
601
|
);
|
|
438
602
|
}
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
const rows = (await Promise.all(
|
|
603
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
604
|
+
semantic = (await Promise.all(
|
|
442
605
|
filters.map(
|
|
443
606
|
(filter) => deps.searchRawMessagesAnn({
|
|
444
607
|
userId: input.userId,
|
|
@@ -449,19 +612,113 @@ function createUnifiedSearch(deps = {}) {
|
|
|
449
612
|
})
|
|
450
613
|
)
|
|
451
614
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
452
|
-
return rows;
|
|
453
615
|
}
|
|
454
|
-
|
|
616
|
+
if (semantic.length === 0) {
|
|
617
|
+
try {
|
|
618
|
+
const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
|
|
619
|
+
const manager2 = await getRawMessageManager2();
|
|
620
|
+
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
621
|
+
const results = await manager2.searchMessagesSemantically({
|
|
622
|
+
userId: input.userId,
|
|
623
|
+
queryEmbedding,
|
|
624
|
+
limit
|
|
625
|
+
});
|
|
626
|
+
semantic = results.map((r) => ({
|
|
627
|
+
type: "memory",
|
|
628
|
+
id: r.id,
|
|
629
|
+
content: r.content,
|
|
630
|
+
similarity: r.similarity,
|
|
631
|
+
metadata: r.metadata ?? {}
|
|
632
|
+
}));
|
|
633
|
+
}
|
|
634
|
+
} catch (error) {
|
|
635
|
+
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
let lexical = [];
|
|
639
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
640
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
641
|
+
if (keywords.length > 0) {
|
|
642
|
+
try {
|
|
643
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
644
|
+
lexical = (await Promise.all(
|
|
645
|
+
lexFilters.map(
|
|
646
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
647
|
+
userId: input.userId,
|
|
648
|
+
keywords,
|
|
649
|
+
limit,
|
|
650
|
+
botId
|
|
651
|
+
})
|
|
652
|
+
)
|
|
653
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
654
|
+
...toMemoryResult(hit),
|
|
655
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
656
|
+
}));
|
|
657
|
+
} catch (error) {
|
|
658
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
659
|
+
warnings.push({
|
|
660
|
+
source: "memory",
|
|
661
|
+
code: "memory_lexical_search_failed",
|
|
662
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
663
|
+
});
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
667
|
+
warnings.push({
|
|
668
|
+
source: "memory",
|
|
669
|
+
code: "memory_lexical_search_not_configured",
|
|
670
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
return { semantic, lexical };
|
|
455
674
|
}
|
|
456
675
|
async function searchRawMemorySemantically(input) {
|
|
457
676
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
458
677
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
459
|
-
|
|
678
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
679
|
+
return mergeAcrossSources({
|
|
680
|
+
memorySubs: sub,
|
|
681
|
+
insightHits: [],
|
|
682
|
+
knowledgeHits: [],
|
|
683
|
+
limit,
|
|
684
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
685
|
+
});
|
|
460
686
|
}
|
|
461
687
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
462
688
|
}
|
|
689
|
+
function mergeAcrossSources(input) {
|
|
690
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
691
|
+
const all = [
|
|
692
|
+
...memorySubs.semantic,
|
|
693
|
+
...memorySubs.lexical,
|
|
694
|
+
...input.insightHits,
|
|
695
|
+
...input.knowledgeHits
|
|
696
|
+
];
|
|
697
|
+
if (input.strategy !== "rrf") {
|
|
698
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
699
|
+
}
|
|
700
|
+
const lists = [];
|
|
701
|
+
if (memorySubs.semantic.length > 0) {
|
|
702
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
703
|
+
}
|
|
704
|
+
if (memorySubs.lexical.length > 0) {
|
|
705
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
706
|
+
}
|
|
707
|
+
if (input.insightHits.length > 0) {
|
|
708
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
709
|
+
}
|
|
710
|
+
if (input.knowledgeHits.length > 0) {
|
|
711
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
712
|
+
}
|
|
713
|
+
if (lists.length <= 1) {
|
|
714
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
715
|
+
}
|
|
716
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
717
|
+
}
|
|
463
718
|
|
|
464
719
|
// src/http.ts
|
|
720
|
+
init_chroma_memory_index();
|
|
721
|
+
init_raw_message_store();
|
|
465
722
|
async function embedMissingMessages(messages, deps) {
|
|
466
723
|
if (typeof deps.embedQuery !== "function") {
|
|
467
724
|
return messages;
|
|
@@ -496,7 +753,23 @@ async function startHttpServer(options = {}) {
|
|
|
496
753
|
const rawStore = createRawMessageStore({
|
|
497
754
|
env: options.env
|
|
498
755
|
});
|
|
499
|
-
const
|
|
756
|
+
const manager2 = await rawStore.getManager();
|
|
757
|
+
const search = createUnifiedSearch({
|
|
758
|
+
...options.unified,
|
|
759
|
+
searchRawMessagesLexical: async (input) => {
|
|
760
|
+
if (typeof manager2.lexicalSearchMessages === "function") {
|
|
761
|
+
const results = await manager2.lexicalSearchMessages(input);
|
|
762
|
+
return results.filter(Boolean).map((r) => ({
|
|
763
|
+
type: "memory",
|
|
764
|
+
id: r.id,
|
|
765
|
+
content: r.content,
|
|
766
|
+
similarity: r.similarity,
|
|
767
|
+
metadata: r.metadata ?? {}
|
|
768
|
+
}));
|
|
769
|
+
}
|
|
770
|
+
return [];
|
|
771
|
+
}
|
|
772
|
+
});
|
|
500
773
|
const app = new Hono();
|
|
501
774
|
app.get("/health", (c) => c.json({ ok: true, store: "memory", ts: Date.now() }));
|
|
502
775
|
app.post("/v1/search", async (c) => {
|
|
@@ -524,20 +797,20 @@ async function startHttpServer(options = {}) {
|
|
|
524
797
|
if (!Array.isArray(body.messages) || !body.userId) {
|
|
525
798
|
return c.json({ error: "userId and messages[] required" }, 400);
|
|
526
799
|
}
|
|
527
|
-
const
|
|
800
|
+
const manager3 = await rawStore.getManager();
|
|
528
801
|
const incoming = body.messages;
|
|
529
802
|
const messages = body.embedOnInsert === true ? await embedMissingMessages(
|
|
530
803
|
incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId })),
|
|
531
804
|
options.unified ?? {}
|
|
532
805
|
) : incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId }));
|
|
533
806
|
let result;
|
|
534
|
-
if (typeof
|
|
535
|
-
result = await
|
|
807
|
+
if (typeof manager3.upsertRawMessages === "function") {
|
|
808
|
+
result = await manager3.upsertRawMessages({
|
|
536
809
|
userId: body.userId,
|
|
537
810
|
messages
|
|
538
811
|
});
|
|
539
|
-
} else if (typeof
|
|
540
|
-
const ids = await
|
|
812
|
+
} else if (typeof manager3.storeMessages === "function") {
|
|
813
|
+
const ids = await manager3.storeMessages(messages);
|
|
541
814
|
result = { inserted: ids.length, ids };
|
|
542
815
|
} else {
|
|
543
816
|
return c.json(
|
|
@@ -552,14 +825,33 @@ async function startHttpServer(options = {}) {
|
|
|
552
825
|
} catch (error) {
|
|
553
826
|
console.warn("[memory-store/http] chroma upsert failed:", error);
|
|
554
827
|
}
|
|
828
|
+
try {
|
|
829
|
+
const messagesWithEmbeddings = messages.filter(
|
|
830
|
+
(m) => Array.isArray(m.embedding) && m.embedding.length > 0
|
|
831
|
+
);
|
|
832
|
+
if (messagesWithEmbeddings.length > 0) {
|
|
833
|
+
for (const message of messagesWithEmbeddings) {
|
|
834
|
+
if (typeof manager3.upsertVectorForMessage === "function") {
|
|
835
|
+
manager3.upsertVectorForMessage(message.messageId, message.embedding);
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
console.log(
|
|
839
|
+
"[memory-store/http] Updated sqlite-vec vector table for",
|
|
840
|
+
messagesWithEmbeddings.length,
|
|
841
|
+
"message(s)"
|
|
842
|
+
);
|
|
843
|
+
}
|
|
844
|
+
} catch (error) {
|
|
845
|
+
console.warn("[memory-store/http] sqlite-vec vector update failed:", error);
|
|
846
|
+
}
|
|
555
847
|
return c.json({ ok: true, count: messages.length, result });
|
|
556
848
|
});
|
|
557
849
|
app.get("/v1/raw-messages/:id", async (c) => {
|
|
558
850
|
const id = c.req.param("id");
|
|
559
851
|
const userId = c.req.query("userId");
|
|
560
852
|
if (!userId) return c.json({ error: "userId query param required" }, 400);
|
|
561
|
-
const
|
|
562
|
-
const row = await
|
|
853
|
+
const manager3 = await rawStore.getManager();
|
|
854
|
+
const row = await manager3.getMessageById?.(id);
|
|
563
855
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
564
856
|
return c.json({ message: row });
|
|
565
857
|
});
|
|
@@ -574,7 +866,11 @@ async function startHttpServer(options = {}) {
|
|
|
574
866
|
};
|
|
575
867
|
}
|
|
576
868
|
|
|
869
|
+
// src/server/cli-shared.ts
|
|
870
|
+
init_raw_message_store();
|
|
871
|
+
|
|
577
872
|
// src/storage/sqlite-vector-index.ts
|
|
873
|
+
init_sqlite_raw_message_store();
|
|
578
874
|
function getMemoryVectorStoreBackend() {
|
|
579
875
|
const backend = (process.env.INSIGHT_VECTOR_STORE_BACKEND || process.env.MEMORY_VECTOR_STORE_BACKEND || process.env.VECTOR_STORE_BACKEND || "sqlite-vec").toLowerCase();
|
|
580
876
|
return backend === "chroma" ? "chroma" : "sqlite-vec";
|
|
@@ -934,5 +1230,3 @@ main().catch((error) => {
|
|
|
934
1230
|
console.error("[memory-store/http] fatal:", error);
|
|
935
1231
|
process.exit(1);
|
|
936
1232
|
});
|
|
937
|
-
//# sourceMappingURL=cli-http.js.map
|
|
938
|
-
//# sourceMappingURL=cli-http.js.map
|