@prometheus-ai/memory 0.5.4 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +4 -4
- package/dist/types/config.d.ts +13 -2
- package/dist/types/core/beam/store.d.ts +20 -0
- package/dist/types/core/embeddings.d.ts +2 -1
- package/dist/types/core/extraction/client.d.ts +11 -7
- package/dist/types/core/extraction.d.ts +2 -1
- package/dist/types/core/fastembed-runtime.d.ts +4 -0
- package/dist/types/core/index.d.ts +1 -0
- package/dist/types/core/llm-backends.d.ts +2 -0
- package/dist/types/core/local-llm.d.ts +8 -3
- package/dist/types/core/memory.d.ts +12 -3
- package/dist/types/core/query-cache.d.ts +1 -2
- package/dist/types/core/runtime-options.d.ts +10 -5
- package/dist/types/core/shmr.d.ts +11 -5
- package/dist/types/core/vector-index.d.ts +16 -0
- package/dist/types/index.d.ts +2 -1
- package/package.json +30 -7
- package/src/cli.ts +19 -19
- package/src/config.ts +98 -68
- package/src/core/banks.ts +2 -2
- package/src/core/beam/consolidate.ts +34 -5
- package/src/core/beam/helpers.ts +21 -28
- package/src/core/beam/index.ts +2 -2
- package/src/core/beam/recall.ts +98 -25
- package/src/core/beam/store.ts +96 -4
- package/src/core/binary-vectors.ts +1 -1
- package/src/core/content-sanitizer.ts +3 -3
- package/src/core/cost-log.ts +1 -1
- package/src/core/embeddings.ts +75 -50
- package/src/core/extraction/client.ts +44 -20
- package/src/core/extraction.ts +10 -9
- package/src/core/fastembed-runtime.ts +89 -0
- package/src/core/index.ts +1 -0
- package/src/core/llm-backends.ts +3 -0
- package/src/core/local-llm.ts +81 -43
- package/src/core/memory.ts +25 -5
- package/src/core/plugins.ts +1 -1
- package/src/core/polyphonic-recall.ts +4 -4
- package/src/core/query-cache.ts +2 -3
- package/src/core/runtime-options.ts +13 -5
- package/src/core/shmr.ts +141 -39
- package/src/core/streaming.ts +1 -1
- package/src/core/triples.ts +3 -3
- package/src/core/vector-index.ts +84 -0
- package/src/diagnose.ts +2 -2
- package/src/dr/recovery.ts +5 -5
- package/src/index.ts +1 -1
- package/src/mcp-server.ts +2 -2
- package/src/mcp-tools.ts +61 -61
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
-
import type { Api, Model } from "@prometheus-ai/ai";
|
|
2
|
+
import type { Api, ApiKey, Model } from "@prometheus-ai/ai";
|
|
3
3
|
|
|
4
4
|
export interface MnemopiLlmCompleteOptions {
|
|
5
5
|
maxTokens?: number;
|
|
@@ -31,14 +31,14 @@ export interface MnemopiEmbeddingRuntimeOptions {
|
|
|
31
31
|
disabled?: boolean;
|
|
32
32
|
model?: string;
|
|
33
33
|
apiUrl?: string;
|
|
34
|
-
apiKey?:
|
|
34
|
+
apiKey?: ApiKey;
|
|
35
35
|
provider?: MnemopiEmbeddingProvider | ((texts: readonly string[]) => EmbeddingOutput | Promise<EmbeddingOutput>);
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export interface MnemopiLlmRuntimeOptions {
|
|
39
39
|
enabled?: boolean;
|
|
40
40
|
baseUrl?: string;
|
|
41
|
-
apiKey?:
|
|
41
|
+
apiKey?: ApiKey;
|
|
42
42
|
model?: string | Model<Api>;
|
|
43
43
|
maxTokens?: number;
|
|
44
44
|
complete?: MnemopiLlmCompletion;
|
|
@@ -51,20 +51,22 @@ export interface MnemopiLlmRuntimeOptions {
|
|
|
51
51
|
export interface MnemopiRuntimeOptions {
|
|
52
52
|
embeddings?: false | MnemopiEmbeddingRuntimeOptions;
|
|
53
53
|
llm?: false | MnemopiLlmRuntimeOptions | Model<Api> | MnemopiLlmCompletion;
|
|
54
|
+
/** Verbose diagnostics: escalates best-effort failure logs from debug to warn. */
|
|
55
|
+
debug?: boolean;
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
export interface ResolvedMnemopiEmbeddingRuntimeOptions {
|
|
57
59
|
disabled?: boolean;
|
|
58
60
|
model?: string;
|
|
59
61
|
apiUrl?: string;
|
|
60
|
-
apiKey?:
|
|
62
|
+
apiKey?: ApiKey;
|
|
61
63
|
provider?: MnemopiEmbeddingProvider;
|
|
62
64
|
}
|
|
63
65
|
|
|
64
66
|
export interface ResolvedMnemopiLlmRuntimeOptions {
|
|
65
67
|
enabled?: boolean;
|
|
66
68
|
baseUrl?: string;
|
|
67
|
-
apiKey?:
|
|
69
|
+
apiKey?: ApiKey;
|
|
68
70
|
model?: string | Model<Api>;
|
|
69
71
|
maxTokens?: number;
|
|
70
72
|
complete?: MnemopiLlmCompletion;
|
|
@@ -75,6 +77,7 @@ export interface ResolvedMnemopiLlmRuntimeOptions {
|
|
|
75
77
|
export interface ResolvedMnemopiRuntimeOptions {
|
|
76
78
|
embeddings?: ResolvedMnemopiEmbeddingRuntimeOptions;
|
|
77
79
|
llm?: ResolvedMnemopiLlmRuntimeOptions;
|
|
80
|
+
debug?: boolean;
|
|
78
81
|
}
|
|
79
82
|
|
|
80
83
|
const runtimeOptionsStorage = new AsyncLocalStorage<ResolvedMnemopiRuntimeOptions>();
|
|
@@ -90,6 +93,11 @@ export function getMnemopiRuntimeOptions(): ResolvedMnemopiRuntimeOptions | unde
|
|
|
90
93
|
return runtimeOptionsStorage.getStore();
|
|
91
94
|
}
|
|
92
95
|
|
|
96
|
+
/** Whether the active runtime scope requested verbose diagnostics (`mnemopi.debug`). */
|
|
97
|
+
export function mnemopiDebugEnabled(): boolean {
|
|
98
|
+
return runtimeOptionsStorage.getStore()?.debug === true;
|
|
99
|
+
}
|
|
100
|
+
|
|
93
101
|
export function resolveEmbeddingProvider(
|
|
94
102
|
provider:
|
|
95
103
|
| MnemopiEmbeddingProvider
|
package/src/core/shmr.ts
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
|
+
import { logger } from "@prometheus-ai/utils";
|
|
4
|
+
import * as embeddings from "./embeddings";
|
|
3
5
|
import { cosineSimilarity } from "./vector-math";
|
|
4
6
|
|
|
5
7
|
export { cosineSimilarity };
|
|
6
8
|
|
|
7
|
-
export const SHMR_BATCH_SIZE = Number.parseInt(process.env.
|
|
8
|
-
export const SHMR_MAX_ITERATIONS = Number.parseInt(process.env.
|
|
9
|
+
export const SHMR_BATCH_SIZE = Number.parseInt(process.env.MNEMOPROMETHEUS_SHMR_BATCH_SIZE ?? "50", 10);
|
|
10
|
+
export const SHMR_MAX_ITERATIONS = Number.parseInt(process.env.MNEMOPROMETHEUS_SHMR_MAX_ITERATIONS ?? "3", 10);
|
|
9
11
|
export const SHMR_SIMILARITY_THRESHOLD = Number.parseFloat(
|
|
10
|
-
process.env.
|
|
12
|
+
process.env.MNEMOPROMETHEUS_SHMR_SIMILARITY_THRESHOLD ?? "0.70",
|
|
11
13
|
);
|
|
12
|
-
export const SHMR_HARMONY_THRESHOLD = Number.parseFloat(process.env.
|
|
13
|
-
export const SHMR_MIN_CLUSTER_SIZE = Number.parseInt(process.env.
|
|
14
|
+
export const SHMR_HARMONY_THRESHOLD = Number.parseFloat(process.env.MNEMOPROMETHEUS_SHMR_HARMONY_THRESHOLD ?? "0.60");
|
|
15
|
+
export const SHMR_MIN_CLUSTER_SIZE = Number.parseInt(process.env.MNEMOPROMETHEUS_SHMR_MIN_CLUSTER_SIZE ?? "2", 10);
|
|
14
16
|
export const EMBEDDING_DIM = 384;
|
|
15
17
|
|
|
16
18
|
export type Vector = Float32Array;
|
|
@@ -105,7 +107,7 @@ CREATE INDEX IF NOT EXISTS idx_beliefs_confidence ON harmonic_beliefs(confidence
|
|
|
105
107
|
export function initSchema(db: Database): void {
|
|
106
108
|
db.exec(FACTS_SCHEMA_SQL);
|
|
107
109
|
}
|
|
108
|
-
function
|
|
110
|
+
function hashEmbedding(text: string): Vector {
|
|
109
111
|
const out = new Float32Array(EMBEDDING_DIM);
|
|
110
112
|
const words = text.toLowerCase().match(/[a-z0-9]+/g) ?? [];
|
|
111
113
|
for (const word of words) {
|
|
@@ -116,21 +118,62 @@ function textForEmbedding(text: string): Vector {
|
|
|
116
118
|
return out;
|
|
117
119
|
}
|
|
118
120
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
+
/**
|
|
122
|
+
* Embed a batch of texts with the configured embedding provider. Falls back to the
|
|
123
|
+
* deterministic SHA1 bag-of-words hash for the entire batch when no provider is
|
|
124
|
+
* available or the provider fails, so every returned vector shares one space.
|
|
125
|
+
*/
|
|
126
|
+
export async function embedBatch(texts: readonly string[]): Promise<Vector[]> {
|
|
127
|
+
if (texts.length === 0) return [];
|
|
128
|
+
let matrix: embeddings.EmbeddingMatrix | null = null;
|
|
129
|
+
try {
|
|
130
|
+
matrix = await embeddings.embed(texts);
|
|
131
|
+
} catch (error) {
|
|
132
|
+
logger.debug("mnemopi shmr embedding failed; using hash fallback", { error: String(error) });
|
|
133
|
+
}
|
|
134
|
+
if (matrix !== null && matrix.length === texts.length) return matrix;
|
|
135
|
+
return texts.map(hashEmbedding);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export async function embed(text: string): Promise<Vector> {
|
|
139
|
+
const [vector] = await embedBatch([text]);
|
|
140
|
+
return vector ?? hashEmbedding(text);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Resolve one vector per item: caller-provided embeddings are kept, the rest are
|
|
145
|
+
* batch-embedded in a single provider call (hash fallback when unavailable).
|
|
146
|
+
*/
|
|
147
|
+
async function resolveItemVectors(items: readonly ShmrItem[]): Promise<Vector[]> {
|
|
148
|
+
const vectors: (Vector | undefined)[] = items.map(item => item.embedding);
|
|
149
|
+
const missingIndices: number[] = [];
|
|
150
|
+
const missingTexts: string[] = [];
|
|
151
|
+
for (let i = 0; i < items.length; i++) {
|
|
152
|
+
if (vectors[i] !== undefined) continue;
|
|
153
|
+
missingIndices.push(i);
|
|
154
|
+
missingTexts.push(items[i]?.object ?? items[i]?.content ?? "");
|
|
155
|
+
}
|
|
156
|
+
if (missingIndices.length > 0) {
|
|
157
|
+
const fresh = await embedBatch(missingTexts);
|
|
158
|
+
for (let k = 0; k < missingIndices.length; k++) {
|
|
159
|
+
const index = missingIndices[k];
|
|
160
|
+
const vector = fresh[k];
|
|
161
|
+
if (index !== undefined && vector !== undefined) vectors[index] = vector;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return vectors.map((vector, i) => vector ?? hashEmbedding(items[i]?.object ?? items[i]?.content ?? ""));
|
|
121
165
|
}
|
|
122
166
|
|
|
123
|
-
export function clusterBySimilarity(items: readonly ShmrItem[], threshold: number): ShmrItem[][] {
|
|
167
|
+
export async function clusterBySimilarity(items: readonly ShmrItem[], threshold: number): Promise<ShmrItem[][]> {
|
|
124
168
|
if (items.length === 0) return [];
|
|
169
|
+
const vectors = await resolveItemVectors(items);
|
|
125
170
|
const adjacency: number[][] = Array.from({ length: items.length }, () => []);
|
|
126
171
|
for (let i = 0; i < items.length; i++) {
|
|
127
|
-
const
|
|
128
|
-
if (
|
|
129
|
-
const leftEmbedding = left.embedding ?? embed(left.object ?? left.content ?? "");
|
|
172
|
+
const leftEmbedding = vectors[i];
|
|
173
|
+
if (leftEmbedding === undefined) continue;
|
|
130
174
|
for (let j = i + 1; j < items.length; j++) {
|
|
131
|
-
const
|
|
132
|
-
if (
|
|
133
|
-
const rightEmbedding = right.embedding ?? embed(right.object ?? right.content ?? "");
|
|
175
|
+
const rightEmbedding = vectors[j];
|
|
176
|
+
if (rightEmbedding === undefined) continue;
|
|
134
177
|
if (cosineSimilarity(leftEmbedding, rightEmbedding) >= threshold) {
|
|
135
178
|
adjacency[i]?.push(j);
|
|
136
179
|
adjacency[j]?.push(i);
|
|
@@ -250,16 +293,22 @@ function deterministicBeliefs(cluster: readonly ShmrItem[]): Belief[] {
|
|
|
250
293
|
];
|
|
251
294
|
}
|
|
252
295
|
|
|
253
|
-
export function computeHarmonyScore(beliefs: readonly Belief[], cluster: readonly ShmrItem[]): number {
|
|
296
|
+
export async function computeHarmonyScore(beliefs: readonly Belief[], cluster: readonly ShmrItem[]): Promise<number> {
|
|
254
297
|
if (beliefs.length === 0 || cluster.length === 0) return 0;
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
298
|
+
const itemVectors = await resolveItemVectors(cluster);
|
|
299
|
+
const beliefVectors = await embedBatch(beliefs.map(belief => `${belief.predicate} ${belief.object}`));
|
|
300
|
+
let dim = 0;
|
|
301
|
+
for (const vector of itemVectors) if (vector.length > dim) dim = vector.length;
|
|
302
|
+
const centroid = new Float32Array(dim);
|
|
303
|
+
for (const vector of itemVectors)
|
|
304
|
+
for (let i = 0; i < vector.length; i++) centroid[i] = (centroid[i] ?? 0) + (vector[i] ?? 0) / cluster.length;
|
|
260
305
|
let total = 0;
|
|
261
|
-
for (
|
|
262
|
-
|
|
306
|
+
for (let k = 0; k < beliefs.length; k++) {
|
|
307
|
+
const belief = beliefs[k];
|
|
308
|
+
const vector = beliefVectors[k];
|
|
309
|
+
if (belief === undefined || vector === undefined) continue;
|
|
310
|
+
total += cosineSimilarity(vector, centroid) * belief.confidence;
|
|
311
|
+
}
|
|
263
312
|
return total / beliefs.length;
|
|
264
313
|
}
|
|
265
314
|
|
|
@@ -305,24 +354,62 @@ function tableExists(db: Database, table: string): boolean {
|
|
|
305
354
|
return db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) !== null;
|
|
306
355
|
}
|
|
307
356
|
|
|
308
|
-
|
|
357
|
+
function parseEmbeddingJson(raw: unknown): Vector | null {
|
|
358
|
+
if (typeof raw !== "string") return null;
|
|
359
|
+
try {
|
|
360
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
361
|
+
if (!Array.isArray(parsed) || parsed.length === 0) return null;
|
|
362
|
+
const out = new Float32Array(parsed.length);
|
|
363
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
364
|
+
const value = Number(parsed[i]);
|
|
365
|
+
if (!Number.isFinite(value)) return null;
|
|
366
|
+
out[i] = value;
|
|
367
|
+
}
|
|
368
|
+
return out;
|
|
369
|
+
} catch {
|
|
370
|
+
return null;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
/** Precomputed vectors from `memory_embeddings` (written by `scheduleEmbedding()`). */
|
|
375
|
+
function precomputedVectors(db: Database, memoryIds: readonly (string | undefined)[]): Map<string, Vector> {
|
|
376
|
+
const out = new Map<string, Vector>();
|
|
377
|
+
const ids = memoryIds.filter((id): id is string => id !== undefined);
|
|
378
|
+
if (ids.length === 0 || !tableExists(db, "memory_embeddings")) return out;
|
|
379
|
+
for (let offset = 0; offset < ids.length; offset += 500) {
|
|
380
|
+
const chunk = ids.slice(offset, offset + 500);
|
|
381
|
+
const rows = db
|
|
382
|
+
.query(
|
|
383
|
+
`SELECT memory_id, embedding_json FROM memory_embeddings WHERE memory_id IN (${chunk.map(() => "?").join(", ")})`,
|
|
384
|
+
)
|
|
385
|
+
.all(...chunk) as Array<{ memory_id: string; embedding_json: string | null }>;
|
|
386
|
+
for (const row of rows) {
|
|
387
|
+
const vector = parseEmbeddingJson(row.embedding_json);
|
|
388
|
+
if (vector !== null) out.set(row.memory_id, vector);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return out;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
export async function harmonize(
|
|
309
395
|
beam: BeamLike,
|
|
310
396
|
batchSize = SHMR_BATCH_SIZE,
|
|
311
397
|
maxIterations = SHMR_MAX_ITERATIONS,
|
|
312
398
|
similarityThreshold = SHMR_SIMILARITY_THRESHOLD,
|
|
313
|
-
): HarmonizeStats {
|
|
399
|
+
): Promise<HarmonizeStats> {
|
|
314
400
|
const started = performance.now();
|
|
315
401
|
const db = dbOf(beam);
|
|
316
402
|
initSchema(db);
|
|
317
|
-
const
|
|
403
|
+
const bare: ShmrItem[] = [];
|
|
404
|
+
const memoryIds: (string | undefined)[] = [];
|
|
318
405
|
if (tableExists(db, "facts")) {
|
|
319
406
|
const rows = db
|
|
320
407
|
.query(
|
|
321
408
|
"SELECT fact_id, subject, predicate, object, confidence, timestamp FROM facts ORDER BY created_at DESC LIMIT ?",
|
|
322
409
|
)
|
|
323
410
|
.all(batchSize) as FactRow[];
|
|
324
|
-
for (const row of rows)
|
|
325
|
-
|
|
411
|
+
for (const row of rows) {
|
|
412
|
+
bare.push({
|
|
326
413
|
fact_id: row.fact_id,
|
|
327
414
|
subject: row.subject,
|
|
328
415
|
predicate: row.predicate,
|
|
@@ -330,16 +417,17 @@ export function harmonize(
|
|
|
330
417
|
confidence: row.confidence ?? 0.5,
|
|
331
418
|
timestamp: row.timestamp ?? undefined,
|
|
332
419
|
source: "fact",
|
|
333
|
-
embedding: embed(row.object),
|
|
334
420
|
});
|
|
421
|
+
memoryIds.push(undefined);
|
|
422
|
+
}
|
|
335
423
|
}
|
|
336
424
|
if (tableExists(db, "episodic_memory")) {
|
|
337
425
|
const rows = db
|
|
338
426
|
.query("SELECT id, content, importance, created_at FROM episodic_memory ORDER BY created_at DESC LIMIT ?")
|
|
339
427
|
.all(Math.max(1, Math.floor(batchSize / 2))) as EpisodeRow[];
|
|
340
428
|
for (const row of rows)
|
|
341
|
-
if (row.content.length > 10)
|
|
342
|
-
|
|
429
|
+
if (row.content.length > 10) {
|
|
430
|
+
bare.push({
|
|
343
431
|
fact_id: `ep_${row.id}`,
|
|
344
432
|
subject: "memory",
|
|
345
433
|
predicate: "contains",
|
|
@@ -347,10 +435,11 @@ export function harmonize(
|
|
|
347
435
|
confidence: row.importance ?? 0.5,
|
|
348
436
|
timestamp: row.created_at ?? undefined,
|
|
349
437
|
source: "episodic",
|
|
350
|
-
embedding: embed(row.content.slice(0, 300)),
|
|
351
438
|
});
|
|
439
|
+
memoryIds.push(row.id);
|
|
440
|
+
}
|
|
352
441
|
}
|
|
353
|
-
if (
|
|
442
|
+
if (bare.length < SHMR_MIN_CLUSTER_SIZE)
|
|
354
443
|
return {
|
|
355
444
|
clusters_found: 0,
|
|
356
445
|
beliefs_generated: 0,
|
|
@@ -359,7 +448,15 @@ export function harmonize(
|
|
|
359
448
|
duration_ms: Math.floor(performance.now() - started),
|
|
360
449
|
status: "insufficient_candidates",
|
|
361
450
|
};
|
|
362
|
-
const
|
|
451
|
+
const precomputed = precomputedVectors(db, memoryIds);
|
|
452
|
+
const seeded: ShmrItem[] = bare.map((item, i) => {
|
|
453
|
+
const memoryId = memoryIds[i];
|
|
454
|
+
const vector = memoryId !== undefined ? precomputed.get(memoryId) : undefined;
|
|
455
|
+
return vector === undefined ? item : { ...item, embedding: vector };
|
|
456
|
+
});
|
|
457
|
+
const itemVectors = await resolveItemVectors(seeded);
|
|
458
|
+
const candidates: ShmrItem[] = seeded.map((item, i) => ({ ...item, embedding: itemVectors[i] }));
|
|
459
|
+
const clusters = (await clusterBySimilarity(candidates, similarityThreshold)).filter(
|
|
363
460
|
cluster => cluster.length >= SHMR_MIN_CLUSTER_SIZE,
|
|
364
461
|
);
|
|
365
462
|
let totalBeliefs = 0;
|
|
@@ -371,7 +468,10 @@ export function harmonize(
|
|
|
371
468
|
const clusterId = `shmr_${Date.now()}_${clusterIndex}`;
|
|
372
469
|
for (let iteration = 0; iteration < maxIterations; iteration++) {
|
|
373
470
|
const beliefs = deterministicBeliefs(cluster);
|
|
374
|
-
const score = Math.max(
|
|
471
|
+
const score = Math.max(
|
|
472
|
+
await computeHarmonyScore(beliefs, cluster),
|
|
473
|
+
beliefs.length > 0 ? SHMR_HARMONY_THRESHOLD : 0,
|
|
474
|
+
);
|
|
375
475
|
scores.push(score);
|
|
376
476
|
if (score >= SHMR_HARMONY_THRESHOLD) {
|
|
377
477
|
applyBeliefs(db, beliefs, cluster, clusterId);
|
|
@@ -406,19 +506,21 @@ export function harmonize(
|
|
|
406
506
|
};
|
|
407
507
|
}
|
|
408
508
|
|
|
409
|
-
export function recallBeliefs(beam: BeamLike, query: string, topK = 10): Array<Record<string, unknown
|
|
509
|
+
export async function recallBeliefs(beam: BeamLike, query: string, topK = 10): Promise<Array<Record<string, unknown>>> {
|
|
410
510
|
const db = dbOf(beam);
|
|
411
511
|
initSchema(db);
|
|
412
|
-
const queryEmbedding = embed(query);
|
|
413
512
|
const rows = db
|
|
414
513
|
.query(
|
|
415
514
|
"SELECT belief_id, subject, predicate, object, confidence, provenance, created_at FROM harmonic_beliefs ORDER BY confidence DESC LIMIT ?",
|
|
416
515
|
)
|
|
417
516
|
.all(topK * 2) as BeliefRow[];
|
|
517
|
+
const vectors = await embedBatch([query, ...rows.map(row => row.object)]);
|
|
518
|
+
const queryEmbedding = vectors[0] ?? hashEmbedding(query);
|
|
418
519
|
return rows
|
|
419
|
-
.map(row => ({
|
|
520
|
+
.map((row, index) => ({
|
|
420
521
|
row,
|
|
421
|
-
score:
|
|
522
|
+
score:
|
|
523
|
+
cosineSimilarity(queryEmbedding, vectors[index + 1] ?? hashEmbedding(row.object)) * (row.confidence ?? 0.5),
|
|
422
524
|
}))
|
|
423
525
|
.sort((a, b) => b.score - a.score)
|
|
424
526
|
.slice(0, topK)
|
package/src/core/streaming.ts
CHANGED
|
@@ -290,7 +290,7 @@ function assertDeltaTable(table: unknown): asserts table is DeltaTable {
|
|
|
290
290
|
function checkpointRoot(host: MemoryHost): string {
|
|
291
291
|
const path = host.dbPath ?? host.db_path;
|
|
292
292
|
return path === undefined || path === ":memory:"
|
|
293
|
-
? join(process.cwd(), ".
|
|
293
|
+
? join(process.cwd(), ".mnemopi-sync")
|
|
294
294
|
: join(path, "..", "sync_checkpoints");
|
|
295
295
|
}
|
|
296
296
|
|
package/src/core/triples.ts
CHANGED
|
@@ -74,12 +74,12 @@ function homeDir(env: ProcessEnv = process.env): string {
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
export function legacyDataDir(env: ProcessEnv = process.env): string {
|
|
77
|
-
return join(homeDir(env), ".
|
|
77
|
+
return join(homeDir(env), ".hermes", "mnemopi", "data");
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
export function defaultDataDir(env: ProcessEnv = process.env): string {
|
|
81
|
-
return env.
|
|
82
|
-
? env.
|
|
81
|
+
return env.MNEMOPROMETHEUS_DATA_DIR && env.MNEMOPROMETHEUS_DATA_DIR.length > 0
|
|
82
|
+
? env.MNEMOPROMETHEUS_DATA_DIR
|
|
83
83
|
: legacyDataDir(env);
|
|
84
84
|
}
|
|
85
85
|
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
export interface ExactVectorSearchHit<TId> {
|
|
2
|
+
id: TId;
|
|
3
|
+
score: number;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export interface ExactVectorIndex<TId> {
|
|
7
|
+
readonly ids: readonly TId[];
|
|
8
|
+
readonly matrix: Float32Array;
|
|
9
|
+
readonly dimensions: number;
|
|
10
|
+
readonly count: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface VectorIndexRow<TId> {
|
|
14
|
+
id: TId;
|
|
15
|
+
vector: readonly number[] | null | undefined;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function buildExactVectorIndex<TId>(rows: readonly VectorIndexRow<TId>[]): ExactVectorIndex<TId> {
|
|
19
|
+
const valid: Array<{ id: TId; vector: readonly number[]; norm: number }> = [];
|
|
20
|
+
let dimensions = 0;
|
|
21
|
+
for (const row of rows) {
|
|
22
|
+
const vector = row.vector;
|
|
23
|
+
if (!vector || vector.length === 0) continue;
|
|
24
|
+
let normSq = 0;
|
|
25
|
+
for (let i = 0; i < vector.length; i += 1) {
|
|
26
|
+
const value = vector[i] ?? 0;
|
|
27
|
+
if (!Number.isFinite(value)) {
|
|
28
|
+
normSq = 0;
|
|
29
|
+
break;
|
|
30
|
+
}
|
|
31
|
+
normSq += value * value;
|
|
32
|
+
}
|
|
33
|
+
if (normSq <= 0) continue;
|
|
34
|
+
valid.push({ id: row.id, vector, norm: Math.sqrt(normSq) });
|
|
35
|
+
if (vector.length > dimensions) dimensions = vector.length;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Float32Array keeps a compact contiguous matrix that matches the shape we'd
|
|
39
|
+
// feed into future ANN/quantized backends. Exact cosine ranking remains sound
|
|
40
|
+
// here because we store normalized vectors and only compare normalized dots.
|
|
41
|
+
const matrix = new Float32Array(valid.length * dimensions);
|
|
42
|
+
const ids: TId[] = [];
|
|
43
|
+
for (let row = 0; row < valid.length; row += 1) {
|
|
44
|
+
const item = valid[row];
|
|
45
|
+
ids.push(item.id);
|
|
46
|
+
const offset = row * dimensions;
|
|
47
|
+
for (let col = 0; col < item.vector.length; col += 1) {
|
|
48
|
+
matrix[offset + col] = (item.vector[col] ?? 0) / item.norm;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
return { ids, matrix, dimensions, count: ids.length };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function searchExactVectorIndex<TId>(
|
|
56
|
+
index: ExactVectorIndex<TId>,
|
|
57
|
+
query: readonly number[],
|
|
58
|
+
limit: number,
|
|
59
|
+
): ExactVectorSearchHit<TId>[] {
|
|
60
|
+
const k = Math.max(0, Math.trunc(limit));
|
|
61
|
+
if (k === 0 || index.count === 0 || index.dimensions === 0 || query.length === 0) return [];
|
|
62
|
+
|
|
63
|
+
let queryNormSq = 0;
|
|
64
|
+
for (const value of query) {
|
|
65
|
+
if (!Number.isFinite(value)) return [];
|
|
66
|
+
queryNormSq += value * value;
|
|
67
|
+
}
|
|
68
|
+
if (queryNormSq <= 0) return [];
|
|
69
|
+
const queryNorm = Math.sqrt(queryNormSq);
|
|
70
|
+
const queryDimensions = Math.min(query.length, index.dimensions);
|
|
71
|
+
const hits: ExactVectorSearchHit<TId>[] = [];
|
|
72
|
+
|
|
73
|
+
for (let row = 0; row < index.count; row += 1) {
|
|
74
|
+
const offset = row * index.dimensions;
|
|
75
|
+
let score = 0;
|
|
76
|
+
for (let col = 0; col < queryDimensions; col += 1) {
|
|
77
|
+
score += index.matrix[offset + col] * ((query[col] ?? 0) / queryNorm);
|
|
78
|
+
}
|
|
79
|
+
hits.push({ id: index.ids[row] as TId, score });
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
hits.sort((a, b) => b.score - a.score);
|
|
83
|
+
return hits.slice(0, Math.min(k, hits.length));
|
|
84
|
+
}
|
package/src/diagnose.ts
CHANGED
|
@@ -101,8 +101,8 @@ export function inspectDatabase(options: DiagnosticOptions = {}): DiagnosticSumm
|
|
|
101
101
|
|
|
102
102
|
log("env", "bun_version", Bun.version);
|
|
103
103
|
log("env", "platform", `${process.platform}-${process.arch}`);
|
|
104
|
-
log("env", "
|
|
105
|
-
log("env", "
|
|
104
|
+
log("env", "MNEMOPROMETHEUS_DATA_DIR", safeEnv("MNEMOPROMETHEUS_DATA_DIR"));
|
|
105
|
+
log("env", "MNEMOPROMETHEUS_VEC_TYPE", safeEnv("MNEMOPROMETHEUS_VEC_TYPE"));
|
|
106
106
|
log("db", "db_path", "OK", path);
|
|
107
107
|
log("db", "data_dir", "OK", options.dataDir ?? configuredDataDir());
|
|
108
108
|
log("db", "data_dir_parent", existsSync(dirname(path)) ? "OK" : "MISSING", dirname(path));
|
package/src/dr/recovery.ts
CHANGED
|
@@ -111,7 +111,7 @@ function hasErrorCode(error: unknown, code: string): boolean {
|
|
|
111
111
|
function writeBackupFile(destinationDir: string, timestamp: string, bytes: Uint8Array): string {
|
|
112
112
|
for (let attempt = 0; attempt < 64; attempt += 1) {
|
|
113
113
|
const suffix = attempt === 0 ? "" : `_${nextUniqueToken()}`;
|
|
114
|
-
const backupPath = join(destinationDir, `
|
|
114
|
+
const backupPath = join(destinationDir, `mnemopi_backup_${timestamp}${suffix}.db.gz`);
|
|
115
115
|
try {
|
|
116
116
|
writeFileSync(backupPath, bytes, { flag: "wx" });
|
|
117
117
|
return backupPath;
|
|
@@ -128,7 +128,7 @@ function restoreTempPath(targetPath: string): string {
|
|
|
128
128
|
}
|
|
129
129
|
|
|
130
130
|
function defaultBackupDir(env: Env = process.env): string {
|
|
131
|
-
const explicit = env.
|
|
131
|
+
const explicit = env.MNEMOPROMETHEUS_BACKUP_DIR;
|
|
132
132
|
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
133
133
|
const dir = configuredDataDir(env);
|
|
134
134
|
return join(dirname(dir), "backups");
|
|
@@ -291,7 +291,7 @@ export function emergencyRestore(backupDir?: string | null, dbPath?: string | nu
|
|
|
291
291
|
const targetPath = dbPath ?? paths.dbPath;
|
|
292
292
|
const backups = existsSync(dir)
|
|
293
293
|
? readdirSync(dir)
|
|
294
|
-
.filter(name => /^
|
|
294
|
+
.filter(name => /^mnemopi_backup_.*\.db\.gz$/.test(name))
|
|
295
295
|
.sort()
|
|
296
296
|
.reverse()
|
|
297
297
|
.map(name => join(dir, name))
|
|
@@ -331,7 +331,7 @@ export function listBackups(backupDir?: string | null): BackupInfo[] {
|
|
|
331
331
|
if (!existsSync(dir)) return [];
|
|
332
332
|
|
|
333
333
|
return readdirSync(dir)
|
|
334
|
-
.filter(name => /^
|
|
334
|
+
.filter(name => /^mnemopi_backup_.*\.db\.gz$/.test(name))
|
|
335
335
|
.sort()
|
|
336
336
|
.reverse()
|
|
337
337
|
.map(name => {
|
|
@@ -352,7 +352,7 @@ export function rotateBackups(backupDir?: string | null, keep = 10): RotateBacku
|
|
|
352
352
|
const dir = backupDir ?? getDefaultPaths().backupDir;
|
|
353
353
|
const backups = existsSync(dir)
|
|
354
354
|
? readdirSync(dir)
|
|
355
|
-
.filter(name => /^
|
|
355
|
+
.filter(name => /^mnemopi_backup_.*\.db\.gz$/.test(name))
|
|
356
356
|
.sort()
|
|
357
357
|
.map(name => join(dir, name))
|
|
358
358
|
: [];
|
package/src/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { configureRecallFeatures, type RecallFeatureFlags } from "./config";
|
|
1
2
|
export * from "./core/beam/index";
|
|
2
3
|
export * from "./core/embeddings";
|
|
3
4
|
export * from "./core/llm-backends";
|
|
@@ -12,7 +13,6 @@ export {
|
|
|
12
13
|
getDefaultInstance,
|
|
13
14
|
getStats,
|
|
14
15
|
Mnemopi,
|
|
15
|
-
Mnemopi as PrometheusMemory,
|
|
16
16
|
query,
|
|
17
17
|
recall,
|
|
18
18
|
recallEnhanced,
|
package/src/mcp-server.ts
CHANGED
|
@@ -72,7 +72,7 @@ export async function handleJsonRpc(request: JsonRpcRequest): Promise<JsonRpcRes
|
|
|
72
72
|
if (method === "initialize") {
|
|
73
73
|
return ok(id, {
|
|
74
74
|
protocolVersion: "2024-11-05",
|
|
75
|
-
serverInfo: { name: "
|
|
75
|
+
serverInfo: { name: "mnemopi", version: "3.1.2" },
|
|
76
76
|
capabilities: { tools: {} },
|
|
77
77
|
});
|
|
78
78
|
}
|
|
@@ -130,7 +130,7 @@ export function runMcpServer(
|
|
|
130
130
|
transport = "stdio",
|
|
131
131
|
options: { port?: number; bank?: string; host?: string } = {},
|
|
132
132
|
): Promise<void> {
|
|
133
|
-
if (options.bank !== undefined && options.bank.length > 0) process.env.
|
|
133
|
+
if (options.bank !== undefined && options.bank.length > 0) process.env.MNEMOPROMETHEUS_MCP_BANK = options.bank;
|
|
134
134
|
if (transport !== "stdio") throw new Error("Only stdio transport is implemented in the TypeScript port");
|
|
135
135
|
return runStdio();
|
|
136
136
|
}
|