@melandlabs/memory-store 0.1.6 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-D6MzksfT.d.ts → config-BYfTzG3h.d.ts} +17 -0
- package/dist/contracts.d.ts +7 -0
- package/dist/contracts.js +0 -2
- package/dist/http.d.ts +1 -1
- package/dist/http.js +355 -67
- package/dist/index.d.ts +2 -2
- package/dist/index.js +287 -52
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +313 -60
- package/dist/policies/memory-graph-correction-policy.js +0 -2
- package/dist/policies/memory-graph-write-policy.js +0 -2
- package/dist/search/unified-search.d.ts +39 -3
- package/dist/search/unified-search.js +311 -56
- package/dist/server/cli-http.js +360 -66
- package/dist/server/cli-mcp.js +318 -59
- package/dist/server/cli-shared.d.ts +1 -1
- package/dist/server/cli-shared.js +0 -2
- package/dist/storage/chroma-memory-index.js +0 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +1 -1
- package/dist/storage/postgres-raw-message-factory.js +0 -2
- package/dist/storage/raw-message-store.d.ts +9 -1
- package/dist/storage/raw-message-store.js +0 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +15 -2
- package/dist/storage/sqlite-raw-message-store.js +8 -3
- package/dist/storage/sqlite-vector-index.d.ts +1 -1
- package/dist/storage/sqlite-vector-index.js +0 -2
- package/package.json +81 -93
- package/dist/contracts.js.map +0 -1
- package/dist/http.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/mcp.js.map +0 -1
- package/dist/policies/memory-graph-correction-policy.js.map +0 -1
- package/dist/policies/memory-graph-write-policy.js.map +0 -1
- package/dist/search/unified-search.js.map +0 -1
- package/dist/server/cli-http.js.map +0 -1
- package/dist/server/cli-mcp.js.map +0 -1
- package/dist/server/cli-shared.js.map +0 -1
- package/dist/storage/chroma-memory-index.js.map +0 -1
- package/dist/storage/postgres-raw-message-factory.js.map +0 -1
- package/dist/storage/raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-vector-index.js.map +0 -1
package/dist/http.js
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import { serve } from '@hono/node-server';
|
|
2
|
-
import { Hono } from 'hono';
|
|
3
1
|
import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
|
|
4
2
|
import { ChromaVectorStore } from '@melandlabs/rag';
|
|
5
3
|
import { mkdirSync } from 'fs';
|
|
6
4
|
import { homedir } from 'os';
|
|
7
|
-
import {
|
|
5
|
+
import { dirname, join } from 'path';
|
|
8
6
|
import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
|
|
7
|
+
import { serve } from '@hono/node-server';
|
|
8
|
+
import { Hono } from 'hono';
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
var __defProp = Object.defineProperty;
|
|
11
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
12
|
+
var __esm = (fn, res) => function __init() {
|
|
13
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
14
|
+
};
|
|
15
|
+
var __export = (target, all) => {
|
|
16
|
+
for (var name in all)
|
|
17
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
18
|
+
};
|
|
11
19
|
|
|
12
20
|
// src/contracts.ts
|
|
13
21
|
function normalizeTimestampToMs(value) {
|
|
@@ -31,14 +39,10 @@ function rawMessageToMemoryRecord(message) {
|
|
|
31
39
|
embeddingContentHash: message.embeddingContentHash
|
|
32
40
|
};
|
|
33
41
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
"RAW_MESSAGE_VECTOR_STORE_BACKEND",
|
|
39
|
-
"MEMORY_VECTOR_STORE_BACKEND",
|
|
40
|
-
"VECTOR_STORE_BACKEND"
|
|
41
|
-
];
|
|
42
|
+
var init_contracts = __esm({
|
|
43
|
+
"src/contracts.ts"() {
|
|
44
|
+
}
|
|
45
|
+
});
|
|
42
46
|
function getBackend(keys) {
|
|
43
47
|
for (const key of keys) {
|
|
44
48
|
const value = process.env[key]?.trim().toLowerCase();
|
|
@@ -141,21 +145,51 @@ async function searchRawMessagesWithChroma(input) {
|
|
|
141
145
|
metadata: result.metadata ?? {}
|
|
142
146
|
})).slice(0, input.limit);
|
|
143
147
|
}
|
|
148
|
+
var DEFAULT_RAW_MESSAGES_COLLECTION, RAW_VECTOR_BACKEND_ENV_KEYS;
|
|
149
|
+
var init_chroma_memory_index = __esm({
|
|
150
|
+
"src/storage/chroma-memory-index.ts"() {
|
|
151
|
+
init_contracts();
|
|
152
|
+
DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
|
|
153
|
+
RAW_VECTOR_BACKEND_ENV_KEYS = [
|
|
154
|
+
"RAW_MESSAGE_VECTOR_STORE_BACKEND",
|
|
155
|
+
"MEMORY_VECTOR_STORE_BACKEND",
|
|
156
|
+
"VECTOR_STORE_BACKEND"
|
|
157
|
+
];
|
|
158
|
+
}
|
|
159
|
+
});
|
|
144
160
|
|
|
145
161
|
// src/storage/postgres-raw-message-factory.ts
|
|
146
|
-
var factory = null;
|
|
147
162
|
async function resolvePostgresFactory(env) {
|
|
148
|
-
return null;
|
|
163
|
+
if (!factory) return null;
|
|
164
|
+
return await factory(env);
|
|
149
165
|
}
|
|
150
166
|
function hasPostgresFactory() {
|
|
151
167
|
return factory !== null;
|
|
152
168
|
}
|
|
153
|
-
var
|
|
154
|
-
var
|
|
169
|
+
var factory;
|
|
170
|
+
var init_postgres_raw_message_factory = __esm({
|
|
171
|
+
"src/storage/postgres-raw-message-factory.ts"() {
|
|
172
|
+
factory = null;
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// src/storage/sqlite-raw-message-store.ts
|
|
177
|
+
var sqlite_raw_message_store_exports = {};
|
|
178
|
+
__export(sqlite_raw_message_store_exports, {
|
|
179
|
+
__resetSQLiteRawMessageManagerForTests: () => __resetSQLiteRawMessageManagerForTests,
|
|
180
|
+
closeSQLiteRawMessageManager: () => closeSQLiteRawMessageManager,
|
|
181
|
+
getSQLiteRawMessageManager: () => getSQLiteRawMessageManager,
|
|
182
|
+
isSQLiteRawMessageStorageAvailable: () => isSQLiteRawMessageStorageAvailable,
|
|
183
|
+
lexicalSearchRawMessages: () => lexicalSearchRawMessages,
|
|
184
|
+
resolveSQLiteRawMessageDbPath: () => resolveSQLiteRawMessageDbPath
|
|
185
|
+
});
|
|
155
186
|
function resolveSQLiteRawMessageDbPath() {
|
|
156
187
|
const fromEnv = process.env.MEMORY_STORE_DB_PATH?.trim();
|
|
157
188
|
return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_SQLITE_PATH;
|
|
158
189
|
}
|
|
190
|
+
function isSQLiteRawMessageStorageAvailable(_env) {
|
|
191
|
+
return true;
|
|
192
|
+
}
|
|
159
193
|
async function getSQLiteRawMessageManager(_env) {
|
|
160
194
|
if (!manager) {
|
|
161
195
|
const dbPath = resolveSQLiteRawMessageDbPath();
|
|
@@ -175,8 +209,35 @@ async function closeSQLiteRawMessageManager() {
|
|
|
175
209
|
await manager.close();
|
|
176
210
|
manager = null;
|
|
177
211
|
}
|
|
212
|
+
function __resetSQLiteRawMessageManagerForTests() {
|
|
213
|
+
manager = null;
|
|
214
|
+
}
|
|
215
|
+
async function lexicalSearchRawMessages(input) {
|
|
216
|
+
const mgr = await getSQLiteRawMessageManager();
|
|
217
|
+
if (typeof mgr.lexicalSearchMessages !== "function") {
|
|
218
|
+
return [];
|
|
219
|
+
}
|
|
220
|
+
return mgr.lexicalSearchMessages(input);
|
|
221
|
+
}
|
|
222
|
+
var manager, DEFAULT_SQLITE_PATH;
|
|
223
|
+
var init_sqlite_raw_message_store = __esm({
|
|
224
|
+
"src/storage/sqlite-raw-message-store.ts"() {
|
|
225
|
+
init_chroma_memory_index();
|
|
226
|
+
manager = null;
|
|
227
|
+
DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
|
|
228
|
+
}
|
|
229
|
+
});
|
|
178
230
|
|
|
179
231
|
// src/storage/raw-message-store.ts
|
|
232
|
+
var raw_message_store_exports = {};
|
|
233
|
+
__export(raw_message_store_exports, {
|
|
234
|
+
closeRawMessageStore: () => closeRawMessageStore,
|
|
235
|
+
configureRawMessageStore: () => configureRawMessageStore,
|
|
236
|
+
createRawMessageStore: () => createRawMessageStore,
|
|
237
|
+
getRawMessageManager: () => getRawMessageManager,
|
|
238
|
+
getRawMessageStorageBackend: () => getRawMessageStorageBackend,
|
|
239
|
+
isRawMessageStorageAvailable: () => isRawMessageStorageAvailable
|
|
240
|
+
});
|
|
180
241
|
function resolveBackend(env) {
|
|
181
242
|
const explicit = process.env.OPENCONTEXT_MEMORY_STORE_BACKEND?.trim().toLowerCase();
|
|
182
243
|
if (explicit === "postgres") return "postgres";
|
|
@@ -186,7 +247,7 @@ function resolveEnv(_env) {
|
|
|
186
247
|
return {};
|
|
187
248
|
}
|
|
188
249
|
function createRawMessageStore(options = {}) {
|
|
189
|
-
resolveEnv(options.env);
|
|
250
|
+
const env = resolveEnv(options.env);
|
|
190
251
|
return {
|
|
191
252
|
getBackend() {
|
|
192
253
|
return resolveBackend();
|
|
@@ -196,7 +257,7 @@ function createRawMessageStore(options = {}) {
|
|
|
196
257
|
},
|
|
197
258
|
async getManager() {
|
|
198
259
|
if (resolveBackend() === "postgres") {
|
|
199
|
-
const pg = await resolvePostgresFactory();
|
|
260
|
+
const pg = await resolvePostgresFactory(env);
|
|
200
261
|
if (!pg) {
|
|
201
262
|
throw new Error(
|
|
202
263
|
"OPENCONTEXT_MEMORY_STORE_BACKEND=postgres but no Postgres factory is registered. Call registerPostgresFactory() at startup, or unset the env var to use the default SQLite backend."
|
|
@@ -211,14 +272,47 @@ function createRawMessageStore(options = {}) {
|
|
|
211
272
|
}
|
|
212
273
|
};
|
|
213
274
|
}
|
|
214
|
-
|
|
215
|
-
|
|
275
|
+
function configureRawMessageStore(config) {
|
|
276
|
+
moduleConfig = config;
|
|
277
|
+
moduleStore = createRawMessageStore({ env: config.env });
|
|
278
|
+
return moduleStore;
|
|
279
|
+
}
|
|
280
|
+
function getRawMessageStorageBackend() {
|
|
281
|
+
if (!moduleStore) {
|
|
282
|
+
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
283
|
+
}
|
|
284
|
+
return moduleStore.getBackend();
|
|
285
|
+
}
|
|
216
286
|
function isRawMessageStorageAvailable() {
|
|
217
287
|
if (!moduleStore) {
|
|
218
288
|
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
219
289
|
}
|
|
220
290
|
return moduleStore.isAvailable();
|
|
221
291
|
}
|
|
292
|
+
async function getRawMessageManager() {
|
|
293
|
+
if (!moduleStore) {
|
|
294
|
+
moduleStore = createRawMessageStore({ env: moduleConfig?.env });
|
|
295
|
+
}
|
|
296
|
+
return moduleStore.getManager();
|
|
297
|
+
}
|
|
298
|
+
async function closeRawMessageStore() {
|
|
299
|
+
if (!moduleStore) return;
|
|
300
|
+
await moduleStore.close();
|
|
301
|
+
moduleStore = null;
|
|
302
|
+
}
|
|
303
|
+
var moduleStore, moduleConfig;
|
|
304
|
+
var init_raw_message_store = __esm({
|
|
305
|
+
"src/storage/raw-message-store.ts"() {
|
|
306
|
+
init_postgres_raw_message_factory();
|
|
307
|
+
init_sqlite_raw_message_store();
|
|
308
|
+
moduleStore = null;
|
|
309
|
+
moduleConfig = null;
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
// src/search/unified-search.ts
|
|
314
|
+
init_chroma_memory_index();
|
|
315
|
+
init_raw_message_store();
|
|
222
316
|
|
|
223
317
|
// src/search/utilities.ts
|
|
224
318
|
var DEFAULT_LIMIT = 10;
|
|
@@ -248,7 +342,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
248
342
|
}
|
|
249
343
|
return Math.min(1, Math.max(-1, parsed));
|
|
250
344
|
}
|
|
251
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
345
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
346
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
347
|
+
if (strategy === "rrf") {
|
|
348
|
+
const lists = options.rankedLists ?? [
|
|
349
|
+
{ name: "memory-semantic", hits: results }
|
|
350
|
+
];
|
|
351
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
352
|
+
}
|
|
252
353
|
return [...results].sort((a, b) => {
|
|
253
354
|
const scoreDelta = b.similarity - a.similarity;
|
|
254
355
|
if (scoreDelta !== 0) {
|
|
@@ -257,6 +358,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
257
358
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
258
359
|
}).slice(0, limit);
|
|
259
360
|
}
|
|
361
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
362
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
363
|
+
}
|
|
364
|
+
var DEFAULT_RRF_K = 60;
|
|
365
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
366
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
367
|
+
return [];
|
|
368
|
+
}
|
|
369
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
370
|
+
const scores = /* @__PURE__ */ new Map();
|
|
371
|
+
const order = [];
|
|
372
|
+
for (const list of lists) {
|
|
373
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
374
|
+
const hit = list.hits[index];
|
|
375
|
+
const key = `${hit.type}::${hit.id}`;
|
|
376
|
+
const contribution = 1 / (safeK + index + 1);
|
|
377
|
+
const existing = scores.get(key);
|
|
378
|
+
if (existing) {
|
|
379
|
+
existing.rrf += contribution;
|
|
380
|
+
} else {
|
|
381
|
+
scores.set(key, { hit, rrf: contribution });
|
|
382
|
+
order.push(key);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
387
|
+
if (b.rrf !== a.rrf) {
|
|
388
|
+
return b.rrf - a.rrf;
|
|
389
|
+
}
|
|
390
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
391
|
+
}).slice(0, limit);
|
|
392
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
393
|
+
...hit,
|
|
394
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
395
|
+
}));
|
|
396
|
+
}
|
|
260
397
|
function toKnowledgeResult(result) {
|
|
261
398
|
return {
|
|
262
399
|
type: "knowledge",
|
|
@@ -288,6 +425,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
288
425
|
}
|
|
289
426
|
|
|
290
427
|
// src/search/unified-search.ts
|
|
428
|
+
function deriveLexicalKeywords(query) {
|
|
429
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
430
|
+
}
|
|
291
431
|
function createUnifiedSearch(deps = {}) {
|
|
292
432
|
const logger = deps.logger ?? console;
|
|
293
433
|
async function searchUnifiedMemory(input) {
|
|
@@ -295,8 +435,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
295
435
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
296
436
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
297
437
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
438
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
298
439
|
const warnings = [];
|
|
299
|
-
const results = [];
|
|
300
440
|
if (!query) {
|
|
301
441
|
return {
|
|
302
442
|
query,
|
|
@@ -306,17 +446,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
306
446
|
warnings
|
|
307
447
|
};
|
|
308
448
|
}
|
|
449
|
+
let memorySubs;
|
|
309
450
|
if (sources.includes("memory")) {
|
|
310
451
|
if (isRawMessageStorageAvailable()) {
|
|
311
452
|
try {
|
|
312
|
-
|
|
313
|
-
results.push(...memoryHits);
|
|
453
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
314
454
|
} catch (error) {
|
|
315
455
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
316
456
|
warnings.push({
|
|
317
457
|
source: "memory",
|
|
318
458
|
code: "memory_search_failed",
|
|
319
|
-
message: error.message ?? "
|
|
459
|
+
message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
|
|
320
460
|
});
|
|
321
461
|
}
|
|
322
462
|
} else {
|
|
@@ -327,9 +467,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
327
467
|
});
|
|
328
468
|
}
|
|
329
469
|
}
|
|
470
|
+
const insightHits = [];
|
|
330
471
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
331
472
|
try {
|
|
332
|
-
const
|
|
473
|
+
const hits = await deps.searchInsights({
|
|
333
474
|
userId: input.userId,
|
|
334
475
|
query: input.query,
|
|
335
476
|
limit,
|
|
@@ -338,8 +479,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
338
479
|
includeArchived: input.includeArchivedInsights,
|
|
339
480
|
authToken: input.authToken
|
|
340
481
|
});
|
|
341
|
-
for (const hit of
|
|
342
|
-
|
|
482
|
+
for (const hit of hits) {
|
|
483
|
+
insightHits.push({
|
|
343
484
|
type: "insight",
|
|
344
485
|
id: hit.id,
|
|
345
486
|
content: hit.content,
|
|
@@ -355,16 +496,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
355
496
|
message: error.message ?? "insights_search_failed"
|
|
356
497
|
});
|
|
357
498
|
}
|
|
358
|
-
} else if (sources.includes("insights"))
|
|
359
|
-
|
|
360
|
-
source: "insights",
|
|
361
|
-
code: "insights_search_not_configured",
|
|
362
|
-
message: "Insights search is not configured for this host."
|
|
363
|
-
});
|
|
364
|
-
}
|
|
499
|
+
} else if (sources.includes("insights")) ;
|
|
500
|
+
const knowledgeHits = [];
|
|
365
501
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
366
502
|
try {
|
|
367
|
-
const
|
|
503
|
+
const hits = await deps.searchKnowledge({
|
|
368
504
|
userId: input.userId,
|
|
369
505
|
query: input.query,
|
|
370
506
|
options: {
|
|
@@ -374,8 +510,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
374
510
|
},
|
|
375
511
|
authToken: input.authToken
|
|
376
512
|
});
|
|
377
|
-
for (const hit of
|
|
378
|
-
|
|
513
|
+
for (const hit of hits) {
|
|
514
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
379
515
|
}
|
|
380
516
|
} catch (error) {
|
|
381
517
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -385,14 +521,14 @@ function createUnifiedSearch(deps = {}) {
|
|
|
385
521
|
message: error.message ?? "knowledge_search_failed"
|
|
386
522
|
});
|
|
387
523
|
}
|
|
388
|
-
} else if (sources.includes("knowledge"))
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
524
|
+
} else if (sources.includes("knowledge")) ;
|
|
525
|
+
const merged = mergeAcrossSources({
|
|
526
|
+
memorySubs,
|
|
527
|
+
insightHits,
|
|
528
|
+
knowledgeHits,
|
|
529
|
+
limit,
|
|
530
|
+
strategy: mergeStrategy
|
|
531
|
+
});
|
|
396
532
|
return {
|
|
397
533
|
query: input.query,
|
|
398
534
|
sources,
|
|
@@ -401,9 +537,35 @@ function createUnifiedSearch(deps = {}) {
|
|
|
401
537
|
warnings
|
|
402
538
|
};
|
|
403
539
|
}
|
|
404
|
-
async function runMemorySource(input, limit, threshold) {
|
|
540
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
405
541
|
if (typeof deps.embedQuery !== "function") {
|
|
406
|
-
|
|
542
|
+
warnings.push({
|
|
543
|
+
source: "memory",
|
|
544
|
+
code: "memory_lexical_search_fallback",
|
|
545
|
+
message: "Semantic search not configured, using keyword search as fallback"
|
|
546
|
+
});
|
|
547
|
+
const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
|
|
548
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
549
|
+
let lexical2 = [];
|
|
550
|
+
if (keywords.length > 0) {
|
|
551
|
+
try {
|
|
552
|
+
const filters2 = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
553
|
+
const results = await Promise.all(
|
|
554
|
+
filters2.map(
|
|
555
|
+
(botId) => lexicalSearchRawMessages2({
|
|
556
|
+
userId: input.userId,
|
|
557
|
+
keywords,
|
|
558
|
+
limit: Math.ceil(limit / filters2.length),
|
|
559
|
+
botId
|
|
560
|
+
})
|
|
561
|
+
)
|
|
562
|
+
);
|
|
563
|
+
lexical2 = results.flat().filter(Boolean).map(toMemoryResult);
|
|
564
|
+
} catch (error) {
|
|
565
|
+
logger.warn?.("[memory-store] Default lexical search failed:", error);
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
return { semantic: [], lexical: lexical2 };
|
|
407
569
|
}
|
|
408
570
|
const queryEmbedding = await deps.embedQuery({
|
|
409
571
|
userId: input.userId,
|
|
@@ -411,9 +573,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
411
573
|
authToken: input.authToken
|
|
412
574
|
});
|
|
413
575
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
576
|
+
let semantic = [];
|
|
414
577
|
if (isRawMessageChromaEnabled()) {
|
|
415
578
|
try {
|
|
416
|
-
|
|
579
|
+
semantic = (await Promise.all(
|
|
417
580
|
filters.map((filter) => {
|
|
418
581
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
419
582
|
return searchRawMessagesWithChroma({
|
|
@@ -428,18 +591,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
428
591
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
429
592
|
backend: "chroma",
|
|
430
593
|
dimensions: queryEmbedding.length,
|
|
431
|
-
count:
|
|
594
|
+
count: semantic.length
|
|
432
595
|
});
|
|
433
|
-
return chromaHits;
|
|
434
596
|
} catch (error) {
|
|
435
597
|
logger.warn?.(
|
|
436
598
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
437
599
|
error
|
|
438
600
|
);
|
|
439
601
|
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
const rows = (await Promise.all(
|
|
602
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
603
|
+
semantic = (await Promise.all(
|
|
443
604
|
filters.map(
|
|
444
605
|
(filter) => deps.searchRawMessagesAnn({
|
|
445
606
|
userId: input.userId,
|
|
@@ -450,19 +611,113 @@ function createUnifiedSearch(deps = {}) {
|
|
|
450
611
|
})
|
|
451
612
|
)
|
|
452
613
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
453
|
-
return rows;
|
|
454
614
|
}
|
|
455
|
-
|
|
615
|
+
if (semantic.length === 0) {
|
|
616
|
+
try {
|
|
617
|
+
const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
|
|
618
|
+
const manager2 = await getRawMessageManager2();
|
|
619
|
+
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
620
|
+
const results = await manager2.searchMessagesSemantically({
|
|
621
|
+
userId: input.userId,
|
|
622
|
+
queryEmbedding,
|
|
623
|
+
limit
|
|
624
|
+
});
|
|
625
|
+
semantic = results.map((r) => ({
|
|
626
|
+
type: "memory",
|
|
627
|
+
id: r.id,
|
|
628
|
+
content: r.content,
|
|
629
|
+
similarity: r.similarity,
|
|
630
|
+
metadata: r.metadata ?? {}
|
|
631
|
+
}));
|
|
632
|
+
}
|
|
633
|
+
} catch (error) {
|
|
634
|
+
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
let lexical = [];
|
|
638
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
639
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
640
|
+
if (keywords.length > 0) {
|
|
641
|
+
try {
|
|
642
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
643
|
+
lexical = (await Promise.all(
|
|
644
|
+
lexFilters.map(
|
|
645
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
646
|
+
userId: input.userId,
|
|
647
|
+
keywords,
|
|
648
|
+
limit,
|
|
649
|
+
botId
|
|
650
|
+
})
|
|
651
|
+
)
|
|
652
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
653
|
+
...toMemoryResult(hit),
|
|
654
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
655
|
+
}));
|
|
656
|
+
} catch (error) {
|
|
657
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
658
|
+
warnings.push({
|
|
659
|
+
source: "memory",
|
|
660
|
+
code: "memory_lexical_search_failed",
|
|
661
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
666
|
+
warnings.push({
|
|
667
|
+
source: "memory",
|
|
668
|
+
code: "memory_lexical_search_not_configured",
|
|
669
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
return { semantic, lexical };
|
|
456
673
|
}
|
|
457
674
|
async function searchRawMemorySemantically(input) {
|
|
458
675
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
459
676
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
460
|
-
|
|
677
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
678
|
+
return mergeAcrossSources({
|
|
679
|
+
memorySubs: sub,
|
|
680
|
+
insightHits: [],
|
|
681
|
+
knowledgeHits: [],
|
|
682
|
+
limit,
|
|
683
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
684
|
+
});
|
|
461
685
|
}
|
|
462
686
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
463
687
|
}
|
|
688
|
+
function mergeAcrossSources(input) {
|
|
689
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
690
|
+
const all = [
|
|
691
|
+
...memorySubs.semantic,
|
|
692
|
+
...memorySubs.lexical,
|
|
693
|
+
...input.insightHits,
|
|
694
|
+
...input.knowledgeHits
|
|
695
|
+
];
|
|
696
|
+
if (input.strategy !== "rrf") {
|
|
697
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
698
|
+
}
|
|
699
|
+
const lists = [];
|
|
700
|
+
if (memorySubs.semantic.length > 0) {
|
|
701
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
702
|
+
}
|
|
703
|
+
if (memorySubs.lexical.length > 0) {
|
|
704
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
705
|
+
}
|
|
706
|
+
if (input.insightHits.length > 0) {
|
|
707
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
708
|
+
}
|
|
709
|
+
if (input.knowledgeHits.length > 0) {
|
|
710
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
711
|
+
}
|
|
712
|
+
if (lists.length <= 1) {
|
|
713
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
714
|
+
}
|
|
715
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
716
|
+
}
|
|
464
717
|
|
|
465
718
|
// src/http.ts
|
|
719
|
+
init_chroma_memory_index();
|
|
720
|
+
init_raw_message_store();
|
|
466
721
|
async function embedMissingMessages(messages, deps) {
|
|
467
722
|
if (typeof deps.embedQuery !== "function") {
|
|
468
723
|
return messages;
|
|
@@ -497,7 +752,23 @@ async function startHttpServer(options = {}) {
|
|
|
497
752
|
const rawStore = createRawMessageStore({
|
|
498
753
|
env: options.env
|
|
499
754
|
});
|
|
500
|
-
const
|
|
755
|
+
const manager2 = await rawStore.getManager();
|
|
756
|
+
const search = createUnifiedSearch({
|
|
757
|
+
...options.unified,
|
|
758
|
+
searchRawMessagesLexical: async (input) => {
|
|
759
|
+
if (typeof manager2.lexicalSearchMessages === "function") {
|
|
760
|
+
const results = await manager2.lexicalSearchMessages(input);
|
|
761
|
+
return results.filter(Boolean).map((r) => ({
|
|
762
|
+
type: "memory",
|
|
763
|
+
id: r.id,
|
|
764
|
+
content: r.content,
|
|
765
|
+
similarity: r.similarity,
|
|
766
|
+
metadata: r.metadata ?? {}
|
|
767
|
+
}));
|
|
768
|
+
}
|
|
769
|
+
return [];
|
|
770
|
+
}
|
|
771
|
+
});
|
|
501
772
|
const app = new Hono();
|
|
502
773
|
app.get("/health", (c) => c.json({ ok: true, store: "memory", ts: Date.now() }));
|
|
503
774
|
app.post("/v1/search", async (c) => {
|
|
@@ -525,20 +796,20 @@ async function startHttpServer(options = {}) {
|
|
|
525
796
|
if (!Array.isArray(body.messages) || !body.userId) {
|
|
526
797
|
return c.json({ error: "userId and messages[] required" }, 400);
|
|
527
798
|
}
|
|
528
|
-
const
|
|
799
|
+
const manager3 = await rawStore.getManager();
|
|
529
800
|
const incoming = body.messages;
|
|
530
801
|
const messages = body.embedOnInsert === true ? await embedMissingMessages(
|
|
531
802
|
incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId })),
|
|
532
803
|
options.unified ?? {}
|
|
533
804
|
) : incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId }));
|
|
534
805
|
let result;
|
|
535
|
-
if (typeof
|
|
536
|
-
result = await
|
|
806
|
+
if (typeof manager3.upsertRawMessages === "function") {
|
|
807
|
+
result = await manager3.upsertRawMessages({
|
|
537
808
|
userId: body.userId,
|
|
538
809
|
messages
|
|
539
810
|
});
|
|
540
|
-
} else if (typeof
|
|
541
|
-
const ids = await
|
|
811
|
+
} else if (typeof manager3.storeMessages === "function") {
|
|
812
|
+
const ids = await manager3.storeMessages(messages);
|
|
542
813
|
result = { inserted: ids.length, ids };
|
|
543
814
|
} else {
|
|
544
815
|
return c.json(
|
|
@@ -553,14 +824,33 @@ async function startHttpServer(options = {}) {
|
|
|
553
824
|
} catch (error) {
|
|
554
825
|
console.warn("[memory-store/http] chroma upsert failed:", error);
|
|
555
826
|
}
|
|
827
|
+
try {
|
|
828
|
+
const messagesWithEmbeddings = messages.filter(
|
|
829
|
+
(m) => Array.isArray(m.embedding) && m.embedding.length > 0
|
|
830
|
+
);
|
|
831
|
+
if (messagesWithEmbeddings.length > 0) {
|
|
832
|
+
for (const message of messagesWithEmbeddings) {
|
|
833
|
+
if (typeof manager3.upsertVectorForMessage === "function") {
|
|
834
|
+
manager3.upsertVectorForMessage(message.messageId, message.embedding);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
console.log(
|
|
838
|
+
"[memory-store/http] Updated sqlite-vec vector table for",
|
|
839
|
+
messagesWithEmbeddings.length,
|
|
840
|
+
"message(s)"
|
|
841
|
+
);
|
|
842
|
+
}
|
|
843
|
+
} catch (error) {
|
|
844
|
+
console.warn("[memory-store/http] sqlite-vec vector update failed:", error);
|
|
845
|
+
}
|
|
556
846
|
return c.json({ ok: true, count: messages.length, result });
|
|
557
847
|
});
|
|
558
848
|
app.get("/v1/raw-messages/:id", async (c) => {
|
|
559
849
|
const id = c.req.param("id");
|
|
560
850
|
const userId = c.req.query("userId");
|
|
561
851
|
if (!userId) return c.json({ error: "userId query param required" }, 400);
|
|
562
|
-
const
|
|
563
|
-
const row = await
|
|
852
|
+
const manager3 = await rawStore.getManager();
|
|
853
|
+
const row = await manager3.getMessageById?.(id);
|
|
564
854
|
if (!row) return c.json({ error: "not found" }, 404);
|
|
565
855
|
return c.json({ message: row });
|
|
566
856
|
});
|
|
@@ -576,5 +866,3 @@ async function startHttpServer(options = {}) {
|
|
|
576
866
|
}
|
|
577
867
|
|
|
578
868
|
export { startHttpServer };
|
|
579
|
-
//# sourceMappingURL=http.js.map
|
|
580
|
-
//# sourceMappingURL=http.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryStoreConfig } from './config-
|
|
2
|
-
export { E as EmbedQueryFn, a as MemoryStoreDb, b as MemoryStoreEnv, U as UnifiedSearchInsightsResult, c as UnifiedSearchKnowledgeResult, V as VectorBackend } from './config-
|
|
1
|
+
import { M as MemoryStoreConfig } from './config-BYfTzG3h.js';
|
|
2
|
+
export { E as EmbedQueryFn, a as MemoryStoreDb, b as MemoryStoreEnv, U as UnifiedSearchInsightsResult, c as UnifiedSearchKnowledgeResult, V as VectorBackend } from './config-BYfTzG3h.js';
|
|
3
3
|
import { UnifiedSearch } from './search/unified-search.js';
|
|
4
4
|
export { UnifiedMemorySearchInput, UnifiedMemorySearchOutput, UnifiedMemorySearchResult, UnifiedMemorySearchSource, UnifiedMemorySearchWarning, clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult } from './search/unified-search.js';
|
|
5
5
|
import { RawMessageStore, getRawMessageManager } from './storage/raw-message-store.js';
|