@exaudeus/memory-mcp 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedder.d.ts +66 -0
- package/dist/embedder.js +204 -0
- package/dist/store.d.ts +22 -0
- package/dist/store.js +138 -2
- package/dist/text-analyzer.d.ts +7 -0
- package/dist/text-analyzer.js +18 -3
- package/dist/types.d.ts +12 -0
- package/dist/types.js +5 -0
- package/package.json +1 -1
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { EmbeddingVector } from './types.js';
|
|
2
|
+
/** Why embedding failed — discriminated union for observability.
|
|
3
|
+
* The store doesn't branch on the reason (always falls back to keywords),
|
|
4
|
+
* but structured errors enable logging and diagnostics. */
|
|
5
|
+
export type EmbedFailure = {
|
|
6
|
+
readonly kind: 'provider-unavailable';
|
|
7
|
+
readonly reason: string;
|
|
8
|
+
} | {
|
|
9
|
+
readonly kind: 'timeout';
|
|
10
|
+
readonly ms: number;
|
|
11
|
+
} | {
|
|
12
|
+
readonly kind: 'invalid-input';
|
|
13
|
+
readonly reason: string;
|
|
14
|
+
};
|
|
15
|
+
/** Embedding outcome — success carries the vector, failure carries structured diagnostics.
|
|
16
|
+
* Matches the StoreResult/CorrectResult pattern from types.ts. */
|
|
17
|
+
export type EmbedResult = {
|
|
18
|
+
readonly ok: true;
|
|
19
|
+
readonly vector: EmbeddingVector;
|
|
20
|
+
} | {
|
|
21
|
+
readonly ok: false;
|
|
22
|
+
readonly failure: EmbedFailure;
|
|
23
|
+
};
|
|
24
|
+
/** Embedding provider boundary — injected into the store.
|
|
25
|
+
* Implementations: OllamaEmbedder (production), FakeEmbedder (tests).
|
|
26
|
+
*
|
|
27
|
+
* Minimal interface: one method + one property.
|
|
28
|
+
* No isAvailable() — redundant with embed() returning a failure.
|
|
29
|
+
* No embedBatch() — interface segregation; batch is a standalone utility. */
|
|
30
|
+
export interface Embedder {
|
|
31
|
+
embed(text: string, signal?: AbortSignal): Promise<EmbedResult>;
|
|
32
|
+
readonly dimensions: number;
|
|
33
|
+
}
|
|
34
|
+
/** Production embedder using local Ollama instance.
|
|
35
|
+
* Uses node:http directly — no external dependencies. */
|
|
36
|
+
export declare class OllamaEmbedder implements Embedder {
|
|
37
|
+
readonly dimensions: number;
|
|
38
|
+
private readonly model;
|
|
39
|
+
private readonly baseUrl;
|
|
40
|
+
private readonly timeoutMs;
|
|
41
|
+
constructor(opts?: {
|
|
42
|
+
readonly model?: string;
|
|
43
|
+
readonly baseUrl?: string;
|
|
44
|
+
readonly timeoutMs?: number;
|
|
45
|
+
readonly dimensions?: number;
|
|
46
|
+
});
|
|
47
|
+
embed(text: string, signal?: AbortSignal): Promise<EmbedResult>;
|
|
48
|
+
}
|
|
49
|
+
/** Deterministic embedder for testing — uses character trigram frequency to produce
|
|
50
|
+
* a fixed-dimension vector from input text. Same text → same vector. Similar texts
|
|
51
|
+
* → high cosine similarity. Dissimilar texts → low cosine similarity.
|
|
52
|
+
*
|
|
53
|
+
* Limitation: models *string* similarity, not *semantic* similarity.
|
|
54
|
+
* "async workflows" and "asynchronous work patterns" score low despite being
|
|
55
|
+
* semantically identical. FakeEmbedder tests prove pipeline mechanics (merge,
|
|
56
|
+
* degradation, round-trip). Real semantic ordering is tested separately with
|
|
57
|
+
* fixture vectors from nomic-embed-text. */
|
|
58
|
+
export declare class FakeEmbedder implements Embedder {
|
|
59
|
+
readonly dimensions: number;
|
|
60
|
+
constructor(dimensions?: number);
|
|
61
|
+
embed(text: string, _signal?: AbortSignal): Promise<EmbedResult>;
|
|
62
|
+
}
|
|
63
|
+
/** Batch embed texts sequentially. Pure composition over Embedder.embed().
|
|
64
|
+
* Sequential because local Ollama benefits from serialized requests (single GPU/CPU).
|
|
65
|
+
* Not on the interface — interface segregation. */
|
|
66
|
+
export declare function batchEmbed(embedder: Embedder, texts: readonly string[]): Promise<ReadonlyArray<EmbedResult>>;
|
package/dist/embedder.js
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// Embedding provider boundary — the only seam that touches embedding infrastructure.
|
|
2
|
+
// Injected into the store for testability; FakeEmbedder for tests, OllamaEmbedder for production.
|
|
3
|
+
//
|
|
4
|
+
// Design:
|
|
5
|
+
// - Errors are data: EmbedResult discriminated union, never throws
|
|
6
|
+
// - Small interface: one method + one property. Batch is a standalone utility.
|
|
7
|
+
// - Cancellation first-class: AbortSignal propagated through embed()
|
|
8
|
+
// - Graceful degradation: embed failure → keyword-only search, never a crash
|
|
9
|
+
import http from 'node:http';
|
|
10
|
+
import { asEmbeddingVector } from './types.js';
|
|
11
|
+
// ─── OllamaEmbedder ───────────────────────────────────────────────────────
|
|
12
|
+
/** Default timeout for embed calls — generous for cold model loads */
|
|
13
|
+
const DEFAULT_TIMEOUT_MS = 5000;
|
|
14
|
+
/** Production embedder using local Ollama instance.
|
|
15
|
+
* Uses node:http directly — no external dependencies. */
|
|
16
|
+
export class OllamaEmbedder {
|
|
17
|
+
constructor(opts) {
|
|
18
|
+
this.model = opts?.model ?? 'nomic-embed-text';
|
|
19
|
+
this.baseUrl = opts?.baseUrl ?? 'http://localhost:11434';
|
|
20
|
+
this.timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
21
|
+
this.dimensions = opts?.dimensions ?? 384;
|
|
22
|
+
}
|
|
23
|
+
async embed(text, signal) {
|
|
24
|
+
// Validate at boundary — no HTTP call wasted on empty input
|
|
25
|
+
const trimmed = text.trim();
|
|
26
|
+
if (!trimmed) {
|
|
27
|
+
return { ok: false, failure: { kind: 'invalid-input', reason: 'empty text' } };
|
|
28
|
+
}
|
|
29
|
+
// Compose caller's signal with internal timeout
|
|
30
|
+
const timeoutSignal = AbortSignal.timeout(this.timeoutMs);
|
|
31
|
+
const combinedSignal = signal
|
|
32
|
+
? AbortSignal.any([signal, timeoutSignal])
|
|
33
|
+
: timeoutSignal;
|
|
34
|
+
const body = JSON.stringify({ model: this.model, prompt: trimmed });
|
|
35
|
+
const url = new URL('/api/embeddings', this.baseUrl);
|
|
36
|
+
const httpResult = await httpPost(url, body, combinedSignal);
|
|
37
|
+
// HTTP layer failed — map to embed failure
|
|
38
|
+
if (!httpResult.ok) {
|
|
39
|
+
if (httpResult.failure === 'aborted') {
|
|
40
|
+
return { ok: false, failure: { kind: 'timeout', ms: this.timeoutMs } };
|
|
41
|
+
}
|
|
42
|
+
return { ok: false, failure: { kind: 'provider-unavailable', reason: httpResult.failure } };
|
|
43
|
+
}
|
|
44
|
+
const response = httpResult.body;
|
|
45
|
+
// Validate response shape — Ollama returns { embedding: number[] }
|
|
46
|
+
if (!response.embedding || !Array.isArray(response.embedding)) {
|
|
47
|
+
return {
|
|
48
|
+
ok: false,
|
|
49
|
+
failure: { kind: 'provider-unavailable', reason: 'unexpected response format — missing embedding array' },
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
// Validate array contents — coercion to Float32Array silently produces NaN for non-numbers
|
|
53
|
+
if (response.embedding.length > 0 && typeof response.embedding[0] !== 'number') {
|
|
54
|
+
return {
|
|
55
|
+
ok: false,
|
|
56
|
+
failure: { kind: 'provider-unavailable', reason: 'embedding array contains non-numeric values' },
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
const vector = asEmbeddingVector(new Float32Array(response.embedding));
|
|
60
|
+
// Dimension sanity check — catches model mismatch at boundary
|
|
61
|
+
if (vector.length !== this.dimensions) {
|
|
62
|
+
return {
|
|
63
|
+
ok: false,
|
|
64
|
+
failure: {
|
|
65
|
+
kind: 'provider-unavailable',
|
|
66
|
+
reason: `dimension mismatch: expected ${this.dimensions}, got ${vector.length}`,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
return { ok: true, vector };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// ─── FakeEmbedder ─────────────────────────────────────────────────────────
|
|
74
|
+
/** Deterministic embedder for testing — uses character trigram frequency to produce
|
|
75
|
+
* a fixed-dimension vector from input text. Same text → same vector. Similar texts
|
|
76
|
+
* → high cosine similarity. Dissimilar texts → low cosine similarity.
|
|
77
|
+
*
|
|
78
|
+
* Limitation: models *string* similarity, not *semantic* similarity.
|
|
79
|
+
* "async workflows" and "asynchronous work patterns" score low despite being
|
|
80
|
+
* semantically identical. FakeEmbedder tests prove pipeline mechanics (merge,
|
|
81
|
+
* degradation, round-trip). Real semantic ordering is tested separately with
|
|
82
|
+
* fixture vectors from nomic-embed-text. */
|
|
83
|
+
export class FakeEmbedder {
|
|
84
|
+
constructor(dimensions = 384) {
|
|
85
|
+
this.dimensions = dimensions;
|
|
86
|
+
}
|
|
87
|
+
async embed(text, _signal) {
|
|
88
|
+
const trimmed = text.trim();
|
|
89
|
+
if (!trimmed) {
|
|
90
|
+
return { ok: false, failure: { kind: 'invalid-input', reason: 'empty text' } };
|
|
91
|
+
}
|
|
92
|
+
const vector = trigramVector(trimmed, this.dimensions);
|
|
93
|
+
return { ok: true, vector };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/** Produce a deterministic vector from text using character trigram frequency.
|
|
97
|
+
* Each trigram hashes to a bucket (dimension index). The resulting vector is
|
|
98
|
+
* L2-normalized so cosine similarity between two vectors is meaningful.
|
|
99
|
+
*
|
|
100
|
+
* Pure function — no side effects, deterministic. */
|
|
101
|
+
function trigramVector(text, dimensions) {
|
|
102
|
+
const normalized = text.toLowerCase();
|
|
103
|
+
const raw = new Float32Array(dimensions);
|
|
104
|
+
// Accumulate trigram counts into dimension buckets
|
|
105
|
+
for (let i = 0; i <= normalized.length - 3; i++) {
|
|
106
|
+
const trigram = normalized.substring(i, i + 3);
|
|
107
|
+
const bucket = trigramHash(trigram, dimensions);
|
|
108
|
+
raw[bucket] += 1;
|
|
109
|
+
}
|
|
110
|
+
// L2 normalize — cosine similarity requires unit-length or at least
|
|
111
|
+
// consistent normalization to be meaningful
|
|
112
|
+
let norm = 0;
|
|
113
|
+
for (let i = 0; i < dimensions; i++) {
|
|
114
|
+
norm += raw[i] * raw[i];
|
|
115
|
+
}
|
|
116
|
+
norm = Math.sqrt(norm);
|
|
117
|
+
if (norm > 0) {
|
|
118
|
+
for (let i = 0; i < dimensions; i++) {
|
|
119
|
+
raw[i] /= norm;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return asEmbeddingVector(raw);
|
|
123
|
+
}
|
|
124
|
+
/** Simple hash of a trigram string to a bucket index.
|
|
125
|
+
* Deterministic: same trigram always maps to the same bucket. */
|
|
126
|
+
function trigramHash(trigram, buckets) {
|
|
127
|
+
let hash = 0;
|
|
128
|
+
for (let i = 0; i < trigram.length; i++) {
|
|
129
|
+
hash = ((hash << 5) - hash + trigram.charCodeAt(i)) | 0;
|
|
130
|
+
}
|
|
131
|
+
return ((hash % buckets) + buckets) % buckets;
|
|
132
|
+
}
|
|
133
|
+
// ─── Batch utility ────────────────────────────────────────────────────────
|
|
134
|
+
/** Batch embed texts sequentially. Pure composition over Embedder.embed().
|
|
135
|
+
* Sequential because local Ollama benefits from serialized requests (single GPU/CPU).
|
|
136
|
+
* Not on the interface — interface segregation. */
|
|
137
|
+
export async function batchEmbed(embedder, texts) {
|
|
138
|
+
const results = [];
|
|
139
|
+
for (const text of texts) {
|
|
140
|
+
results.push(await embedder.embed(text));
|
|
141
|
+
}
|
|
142
|
+
return results;
|
|
143
|
+
}
|
|
144
|
+
/** Minimal HTTP POST using node:http — returns a result type, never throws.
|
|
145
|
+
* No external dependencies — the Ollama API is simple enough for raw HTTP.
|
|
146
|
+
*
|
|
147
|
+
* Uses a settled guard to prevent double-resolution when abort races with
|
|
148
|
+
* a network error (req.destroy triggers error handler alongside abort handler). */
|
|
149
|
+
function httpPost(url, body, signal) {
|
|
150
|
+
return new Promise((resolve) => {
|
|
151
|
+
let settled = false;
|
|
152
|
+
const settle = (result) => {
|
|
153
|
+
if (settled)
|
|
154
|
+
return;
|
|
155
|
+
settled = true;
|
|
156
|
+
signal.removeEventListener('abort', onAbort);
|
|
157
|
+
resolve(result);
|
|
158
|
+
};
|
|
159
|
+
// Already aborted before we start — fail fast
|
|
160
|
+
if (signal.aborted) {
|
|
161
|
+
resolve({ ok: false, failure: 'aborted' });
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
const onAbort = () => {
|
|
165
|
+
req.destroy();
|
|
166
|
+
settle({ ok: false, failure: 'aborted' });
|
|
167
|
+
};
|
|
168
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
169
|
+
const req = http.request({
|
|
170
|
+
hostname: url.hostname,
|
|
171
|
+
port: url.port || 80,
|
|
172
|
+
path: url.pathname,
|
|
173
|
+
method: 'POST',
|
|
174
|
+
headers: {
|
|
175
|
+
'Content-Type': 'application/json',
|
|
176
|
+
'Content-Length': Buffer.byteLength(body),
|
|
177
|
+
},
|
|
178
|
+
}, (res) => {
|
|
179
|
+
const chunks = [];
|
|
180
|
+
res.on('data', (chunk) => chunks.push(chunk));
|
|
181
|
+
res.on('end', () => {
|
|
182
|
+
const raw = Buffer.concat(chunks).toString('utf-8');
|
|
183
|
+
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
|
184
|
+
settle({ ok: false, failure: `HTTP ${res.statusCode}: ${raw.slice(0, 200)}` });
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
settle({ ok: true, body: JSON.parse(raw) });
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
settle({ ok: false, failure: `invalid JSON response: ${raw.slice(0, 200)}` });
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
res.on('error', (err) => {
|
|
195
|
+
settle({ ok: false, failure: err.message });
|
|
196
|
+
});
|
|
197
|
+
});
|
|
198
|
+
req.on('error', (err) => {
|
|
199
|
+
settle({ ok: false, failure: err.message });
|
|
200
|
+
});
|
|
201
|
+
req.write(body);
|
|
202
|
+
req.end();
|
|
203
|
+
});
|
|
204
|
+
}
|
package/dist/store.d.ts
CHANGED
|
@@ -4,7 +4,9 @@ export declare class MarkdownMemoryStore {
|
|
|
4
4
|
private readonly memoryPath;
|
|
5
5
|
private readonly clock;
|
|
6
6
|
private readonly git;
|
|
7
|
+
private readonly embedder;
|
|
7
8
|
private entries;
|
|
9
|
+
private vectors;
|
|
8
10
|
private corruptFileCount;
|
|
9
11
|
constructor(config: MemoryConfig);
|
|
10
12
|
/** Resolved behavior thresholds — user config merged over defaults.
|
|
@@ -46,6 +48,26 @@ export declare class MarkdownMemoryStore {
|
|
|
46
48
|
private persistEntry;
|
|
47
49
|
/** Delete the file for an entry */
|
|
48
50
|
private deleteEntryFile;
|
|
51
|
+
/** .vec file format version — future-proofs for metadata additions.
|
|
52
|
+
* v1: [u8 version=0x01][u32LE dimensions][f32LE × dimensions] */
|
|
53
|
+
private static readonly VEC_FORMAT_VERSION;
|
|
54
|
+
/** Write a vector sidecar file for an entry.
|
|
55
|
+
* Format: [u8 version][u32LE dims][f32LE × dims]. Total: 5 + dims*4 bytes. */
|
|
56
|
+
private persistVector;
|
|
57
|
+
/** Load and validate a vector sidecar file.
|
|
58
|
+
* Returns null on: missing file, unknown version, wrong dimensions, corrupt size.
|
|
59
|
+
* Null is not an error — it means the entry works via keyword fallback. */
|
|
60
|
+
private loadVector;
|
|
61
|
+
/** Delete a vector sidecar file. Fire-and-forget — failure is harmless. */
|
|
62
|
+
private deleteVector;
|
|
63
|
+
/** Derive the .vec path from a .md relative path */
|
|
64
|
+
private vecPathFromMdPath;
|
|
65
|
+
/** Load vectors for all known entries. Cleans up orphan .vec files.
|
|
66
|
+
* Skipped entirely when embedder is null (no wasted I/O).
|
|
67
|
+
* Returns a fresh map — no mutation of store state. */
|
|
68
|
+
private loadVectorSnapshot;
|
|
69
|
+
/** Recursively find all .vec files in a directory */
|
|
70
|
+
private findVecFiles;
|
|
49
71
|
/** Load all entries from disk and return as an immutable snapshot.
|
|
50
72
|
* Pure read — no mutation. Callers decide whether to cache.
|
|
51
73
|
* Tracks corrupt files for observability without failing the load. */
|
package/dist/store.js
CHANGED
|
@@ -6,7 +6,7 @@ import path from 'path';
|
|
|
6
6
|
import crypto from 'crypto';
|
|
7
7
|
import { execFile } from 'child_process';
|
|
8
8
|
import { promisify } from 'util';
|
|
9
|
-
import { DEFAULT_CONFIDENCE, realClock, parseTopicScope, parseTrustLevel, parseTags } from './types.js';
|
|
9
|
+
import { DEFAULT_CONFIDENCE, realClock, parseTopicScope, parseTrustLevel, parseTags, asEmbeddingVector } from './types.js';
|
|
10
10
|
import { DEDUP_SIMILARITY_THRESHOLD, CONFLICT_SIMILARITY_THRESHOLD_SAME_TOPIC, CONFLICT_SIMILARITY_THRESHOLD_CROSS_TOPIC, CONFLICT_MIN_CONTENT_CHARS, OPPOSITION_PAIRS, PREFERENCE_SURFACE_THRESHOLD, REFERENCE_BOOST_MULTIPLIER, TOPIC_BOOST, MODULE_TOPIC_BOOST, USER_ALWAYS_INCLUDE_SCORE_FRACTION, DEFAULT_STALE_DAYS_STANDARD, DEFAULT_STALE_DAYS_PREFERENCES, DEFAULT_MAX_STALE_IN_BRIEFING, DEFAULT_MAX_DEDUP_SUGGESTIONS, DEFAULT_MAX_CONFLICT_PAIRS, DEFAULT_MAX_PREFERENCE_SUGGESTIONS, TAG_MATCH_BOOST, } from './thresholds.js';
|
|
11
11
|
import { realGitService } from './git-service.js';
|
|
12
12
|
import { extractKeywords, stem, similarity, matchesFilter, computeRelevanceScore, } from './text-analyzer.js';
|
|
@@ -17,11 +17,13 @@ const execFileAsync = promisify(execFile);
|
|
|
17
17
|
export class MarkdownMemoryStore {
|
|
18
18
|
constructor(config) {
|
|
19
19
|
this.entries = new Map();
|
|
20
|
+
this.vectors = new Map();
|
|
20
21
|
this.corruptFileCount = 0;
|
|
21
22
|
this.config = config;
|
|
22
23
|
this.memoryPath = config.memoryPath;
|
|
23
24
|
this.clock = config.clock ?? realClock;
|
|
24
25
|
this.git = config.git ?? realGitService;
|
|
26
|
+
this.embedder = config.embedder ?? null;
|
|
25
27
|
}
|
|
26
28
|
/** Resolved behavior thresholds — user config merged over defaults.
|
|
27
29
|
* Centralizes threshold resolution so every caller gets the same value. */
|
|
@@ -544,6 +546,8 @@ export class MarkdownMemoryStore {
|
|
|
544
546
|
await fs.unlink(fullPath);
|
|
545
547
|
}
|
|
546
548
|
catch { /* already gone */ }
|
|
549
|
+
// Delete companion .vec file if it exists
|
|
550
|
+
await this.deleteVector(relativePath);
|
|
547
551
|
// Clean up empty parent directories
|
|
548
552
|
try {
|
|
549
553
|
const dir = path.dirname(fullPath);
|
|
@@ -554,6 +558,128 @@ export class MarkdownMemoryStore {
|
|
|
554
558
|
}
|
|
555
559
|
catch { /* ignore */ }
|
|
556
560
|
}
|
|
561
|
+
/** Write a vector sidecar file for an entry.
|
|
562
|
+
* Format: [u8 version][u32LE dims][f32LE × dims]. Total: 5 + dims*4 bytes. */
|
|
563
|
+
async persistVector(relativePath, vector) {
|
|
564
|
+
const vecPath = this.vecPathFromMdPath(relativePath);
|
|
565
|
+
const buf = Buffer.alloc(5 + vector.length * 4);
|
|
566
|
+
buf.writeUInt8(MarkdownMemoryStore.VEC_FORMAT_VERSION, 0);
|
|
567
|
+
buf.writeUInt32LE(vector.length, 1);
|
|
568
|
+
for (let i = 0; i < vector.length; i++) {
|
|
569
|
+
buf.writeFloatLE(vector[i], 5 + i * 4);
|
|
570
|
+
}
|
|
571
|
+
await fs.mkdir(path.dirname(vecPath), { recursive: true });
|
|
572
|
+
await fs.writeFile(vecPath, buf);
|
|
573
|
+
}
|
|
574
|
+
/** Load and validate a vector sidecar file.
|
|
575
|
+
* Returns null on: missing file, unknown version, wrong dimensions, corrupt size.
|
|
576
|
+
* Null is not an error — it means the entry works via keyword fallback. */
|
|
577
|
+
async loadVector(relativePath) {
|
|
578
|
+
if (!this.embedder)
|
|
579
|
+
return null;
|
|
580
|
+
const vecPath = this.vecPathFromMdPath(relativePath);
|
|
581
|
+
let buf;
|
|
582
|
+
try {
|
|
583
|
+
buf = await fs.readFile(vecPath);
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
return null; // missing — entry predates embeddings
|
|
587
|
+
}
|
|
588
|
+
// Minimum valid size: 1 (version) + 4 (dims) = 5 bytes
|
|
589
|
+
if (buf.length < 5) {
|
|
590
|
+
process.stderr.write(`[memory-mcp] Corrupt .vec file (too small): ${vecPath}\n`);
|
|
591
|
+
return null;
|
|
592
|
+
}
|
|
593
|
+
const version = buf.readUInt8(0);
|
|
594
|
+
if (version !== MarkdownMemoryStore.VEC_FORMAT_VERSION) {
|
|
595
|
+
process.stderr.write(`[memory-mcp] Unknown .vec version ${version}: ${vecPath}\n`);
|
|
596
|
+
return null;
|
|
597
|
+
}
|
|
598
|
+
const storedDims = buf.readUInt32LE(1);
|
|
599
|
+
if (storedDims !== this.embedder.dimensions) {
|
|
600
|
+
// Dimension mismatch — model changed. Entry needs re-embed, not an error.
|
|
601
|
+
process.stderr.write(`[memory-mcp] Dimension mismatch in ${vecPath}: stored ${storedDims}, expected ${this.embedder.dimensions}\n`);
|
|
602
|
+
return null;
|
|
603
|
+
}
|
|
604
|
+
const expectedSize = 5 + storedDims * 4;
|
|
605
|
+
if (buf.length !== expectedSize) {
|
|
606
|
+
process.stderr.write(`[memory-mcp] Corrupt .vec file (expected ${expectedSize} bytes, got ${buf.length}): ${vecPath}\n`);
|
|
607
|
+
return null;
|
|
608
|
+
}
|
|
609
|
+
const raw = new Float32Array(storedDims);
|
|
610
|
+
for (let i = 0; i < storedDims; i++) {
|
|
611
|
+
raw[i] = buf.readFloatLE(5 + i * 4);
|
|
612
|
+
}
|
|
613
|
+
return asEmbeddingVector(raw);
|
|
614
|
+
}
|
|
615
|
+
/** Delete a vector sidecar file. Fire-and-forget — failure is harmless. */
|
|
616
|
+
async deleteVector(mdRelativePath) {
|
|
617
|
+
const vecPath = this.vecPathFromMdPath(mdRelativePath);
|
|
618
|
+
try {
|
|
619
|
+
await fs.unlink(vecPath);
|
|
620
|
+
}
|
|
621
|
+
catch { /* already gone or never existed */ }
|
|
622
|
+
}
|
|
623
|
+
/** Derive the .vec path from a .md relative path */
|
|
624
|
+
vecPathFromMdPath(mdRelativePath) {
|
|
625
|
+
const vecRelative = mdRelativePath.replace(/\.md$/, '.vec');
|
|
626
|
+
return path.join(this.memoryPath, vecRelative);
|
|
627
|
+
}
|
|
628
|
+
/** Load vectors for all known entries. Cleans up orphan .vec files.
|
|
629
|
+
* Skipped entirely when embedder is null (no wasted I/O).
|
|
630
|
+
* Returns a fresh map — no mutation of store state. */
|
|
631
|
+
async loadVectorSnapshot(entries) {
|
|
632
|
+
if (!this.embedder)
|
|
633
|
+
return new Map();
|
|
634
|
+
const vectors = new Map();
|
|
635
|
+
const entryIds = new Set(entries.keys());
|
|
636
|
+
// Load vectors for known entries
|
|
637
|
+
for (const entry of entries.values()) {
|
|
638
|
+
const relativePath = this.entryToRelativePath(entry);
|
|
639
|
+
const vector = await this.loadVector(relativePath);
|
|
640
|
+
if (vector) {
|
|
641
|
+
vectors.set(entry.id, vector);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
// Clean up orphan .vec files — entries deleted while embedder was unavailable
|
|
645
|
+
try {
|
|
646
|
+
const vecFiles = await this.findVecFiles(this.memoryPath);
|
|
647
|
+
for (const vecFile of vecFiles) {
|
|
648
|
+
const relativePath = path.relative(this.memoryPath, vecFile);
|
|
649
|
+
// Derive the entry ID: the .vec filename without extension matches the .md filename
|
|
650
|
+
const mdRelative = relativePath.replace(/\.vec$/, '.md');
|
|
651
|
+
// Check if any entry maps to this path
|
|
652
|
+
const hasMatchingEntry = Array.from(entries.values()).some(e => this.entryToRelativePath(e) === mdRelative);
|
|
653
|
+
if (!hasMatchingEntry) {
|
|
654
|
+
try {
|
|
655
|
+
await fs.unlink(vecFile);
|
|
656
|
+
process.stderr.write(`[memory-mcp] Cleaned up orphan .vec: ${relativePath}\n`);
|
|
657
|
+
}
|
|
658
|
+
catch { /* ignore cleanup failures */ }
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
catch { /* ignore — directory may not exist yet */ }
|
|
663
|
+
return vectors;
|
|
664
|
+
}
|
|
665
|
+
/** Recursively find all .vec files in a directory */
|
|
666
|
+
async findVecFiles(dir) {
|
|
667
|
+
const results = [];
|
|
668
|
+
try {
|
|
669
|
+
const dirEntries = await fs.readdir(dir, { withFileTypes: true });
|
|
670
|
+
for (const dirEntry of dirEntries) {
|
|
671
|
+
const fullPath = path.join(dir, dirEntry.name);
|
|
672
|
+
if (dirEntry.isDirectory()) {
|
|
673
|
+
results.push(...await this.findVecFiles(fullPath));
|
|
674
|
+
}
|
|
675
|
+
else if (dirEntry.name.endsWith('.vec')) {
|
|
676
|
+
results.push(fullPath);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
catch { /* ignore */ }
|
|
681
|
+
return results;
|
|
682
|
+
}
|
|
557
683
|
/** Load all entries from disk and return as an immutable snapshot.
|
|
558
684
|
* Pure read — no mutation. Callers decide whether to cache.
|
|
559
685
|
* Tracks corrupt files for observability without failing the load. */
|
|
@@ -576,13 +702,16 @@ export class MarkdownMemoryStore {
|
|
|
576
702
|
catch {
|
|
577
703
|
// Empty memory — first run
|
|
578
704
|
}
|
|
579
|
-
|
|
705
|
+
// Load vectors after entries — needs the entry map for orphan detection
|
|
706
|
+
const vectors = await this.loadVectorSnapshot(entries);
|
|
707
|
+
return { entries, vectors, corruptFileCount };
|
|
580
708
|
}
|
|
581
709
|
/** Reload entries from disk into the store's working state.
|
|
582
710
|
* This is the single mutation point for disk reads. */
|
|
583
711
|
async reloadFromDisk() {
|
|
584
712
|
const snapshot = await this.loadSnapshot();
|
|
585
713
|
this.entries = new Map(snapshot.entries);
|
|
714
|
+
this.vectors = new Map(snapshot.vectors);
|
|
586
715
|
this.corruptFileCount = snapshot.corruptFileCount;
|
|
587
716
|
}
|
|
588
717
|
/** Recursively find all .md files in a directory */
|
|
@@ -889,3 +1018,10 @@ export class MarkdownMemoryStore {
|
|
|
889
1018
|
}));
|
|
890
1019
|
}
|
|
891
1020
|
}
|
|
1021
|
+
// ─── Vector sidecar storage ─────────────────────────────────────────────
|
|
1022
|
+
// Sidecar .vec files alongside .md entries. Embeddings are a cache — delete
|
|
1023
|
+
// all .vec files and the system degrades to keyword search. Non-atomic with
|
|
1024
|
+
// .md writes (both failure modes are benign since vectors are derived data).
|
|
1025
|
+
/** .vec file format version — future-proofs for metadata additions.
|
|
1026
|
+
* v1: [u8 version=0x01][u32LE dimensions][f32LE × dimensions] */
|
|
1027
|
+
MarkdownMemoryStore.VEC_FORMAT_VERSION = 0x01;
|
package/dist/text-analyzer.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { EmbeddingVector } from './types.js';
|
|
1
2
|
/** Parsed filter group: required, excluded, exact-match, and tag terms */
|
|
2
3
|
export interface FilterGroup {
|
|
3
4
|
readonly must: Set<string>;
|
|
@@ -36,6 +37,12 @@ export declare function jaccardSimilarity(a: Set<string>, b: Set<string>): numbe
|
|
|
36
37
|
/** Containment similarity: |intersection| / min(|a|, |b|)
|
|
37
38
|
* Catches when one entry is a subset of a larger one */
|
|
38
39
|
export declare function containmentSimilarity(a: Set<string>, b: Set<string>): number;
|
|
40
|
+
/** Cosine similarity between two embedding vectors.
|
|
41
|
+
* Returns 0.0–1.0 for normalized vectors (most embedding models normalize output).
|
|
42
|
+
* Returns 0 for zero vectors (not NaN) — no information means no similarity.
|
|
43
|
+
*
|
|
44
|
+
* Pure function. No allocations beyond locals. ~1ms for 200 entries at 384 dims. */
|
|
45
|
+
export declare function cosineSimilarity(a: EmbeddingVector, b: EmbeddingVector): number;
|
|
39
46
|
/** Combined similarity: max(jaccard, containment) with title boost.
|
|
40
47
|
* Title keywords get double weight by being included twice. */
|
|
41
48
|
export declare function similarity(titleA: string, contentA: string, titleB: string, contentB: string): number;
|
package/dist/text-analyzer.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// Pure text analysis: stemming, keyword extraction, similarity, filter parsing.
|
|
2
2
|
// Stateless — all functions are pure. No I/O, no side effects.
|
|
3
3
|
//
|
|
4
|
-
// Design: this module is the seam for
|
|
4
|
+
// Design: this module is the seam for search strategies.
|
|
5
5
|
// v1: keyword matching with naive stemming (this file)
|
|
6
|
-
// v2:
|
|
7
|
-
// v3:
|
|
6
|
+
// v2: embedding-based cosine similarity (this file + embedder.ts)
|
|
7
|
+
// v3: graph-enriched retrieval (if proven needed — see graph-library-design.md)
|
|
8
8
|
// Stopwords for keyword extraction — common English words with no semantic value
|
|
9
9
|
const STOPWORDS = new Set([
|
|
10
10
|
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
@@ -139,6 +139,21 @@ export function containmentSimilarity(a, b) {
|
|
|
139
139
|
}
|
|
140
140
|
return intersection / Math.min(a.size, b.size);
|
|
141
141
|
}
|
|
142
|
+
/** Cosine similarity between two embedding vectors.
|
|
143
|
+
* Returns 0.0–1.0 for normalized vectors (most embedding models normalize output).
|
|
144
|
+
* Returns 0 for zero vectors (not NaN) — no information means no similarity.
|
|
145
|
+
*
|
|
146
|
+
* Pure function. No allocations beyond locals. ~1ms for 200 entries at 384 dims. */
|
|
147
|
+
export function cosineSimilarity(a, b) {
|
|
148
|
+
let dot = 0, normA = 0, normB = 0;
|
|
149
|
+
for (let i = 0; i < a.length; i++) {
|
|
150
|
+
dot += a[i] * b[i];
|
|
151
|
+
normA += a[i] * a[i];
|
|
152
|
+
normB += b[i] * b[i];
|
|
153
|
+
}
|
|
154
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
155
|
+
return denom === 0 ? 0 : dot / denom;
|
|
156
|
+
}
|
|
142
157
|
/** Combined similarity: max(jaccard, containment) with title boost.
|
|
143
158
|
* Title keywords get double weight by being included twice. */
|
|
144
159
|
export function similarity(titleA, contentA, titleB, contentB) {
|
package/dist/types.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Embedder } from './embedder.js';
|
|
1
2
|
/** Trust levels for knowledge sources, ordered by reliability */
|
|
2
3
|
export type TrustLevel = 'user' | 'agent-confirmed' | 'agent-inferred';
|
|
3
4
|
/** Ephemeral detection severity — three distinct levels so consumers can branch exhaustively.
|
|
@@ -12,6 +13,16 @@ export type TopicScope = 'user' | 'preferences' | 'architecture' | 'conventions'
|
|
|
12
13
|
export type Tag = string & {
|
|
13
14
|
readonly __brand: 'Tag';
|
|
14
15
|
};
|
|
16
|
+
/** Embedding vector — branded Float32Array to prevent passing arbitrary float arrays.
|
|
17
|
+
* The brand carries domain intent: this is a valid embedding from a specific model,
|
|
18
|
+
* not an arbitrary numeric buffer. Decoupled from embedder.ts so text-analyzer.ts
|
|
19
|
+
* can import it without depending on the embedding provider. */
|
|
20
|
+
export type EmbeddingVector = Float32Array & {
|
|
21
|
+
readonly __brand: 'EmbeddingVector';
|
|
22
|
+
};
|
|
23
|
+
/** Construct an EmbeddingVector from a Float32Array. Boundary validation only —
|
|
24
|
+
* callers (OllamaEmbedder, FakeEmbedder, vector deserialization) validate dimensions. */
|
|
25
|
+
export declare function asEmbeddingVector(raw: Float32Array): EmbeddingVector;
|
|
15
26
|
/** Parse a raw string into a Tag, returning null for invalid input.
|
|
16
27
|
* Normalizes to lowercase. Rejects empty, too-long, or non-slug strings. */
|
|
17
28
|
export declare function parseTag(raw: string): Tag | null;
|
|
@@ -220,6 +231,7 @@ export interface MemoryConfig {
|
|
|
220
231
|
readonly behavior?: BehaviorConfig;
|
|
221
232
|
readonly clock?: Clock;
|
|
222
233
|
readonly git?: GitService;
|
|
234
|
+
readonly embedder?: Embedder;
|
|
223
235
|
}
|
|
224
236
|
/** Default confidence values by trust level */
|
|
225
237
|
export declare const DEFAULT_CONFIDENCE: Record<TrustLevel, number>;
|
package/dist/types.js
CHANGED
|
@@ -10,6 +10,11 @@ export function parseTrustLevel(raw) {
|
|
|
10
10
|
return TRUST_LEVELS.includes(raw) ? raw : null;
|
|
11
11
|
}
|
|
12
12
|
const FIXED_TOPICS = ['user', 'preferences', 'architecture', 'conventions', 'gotchas', 'recent-work'];
|
|
13
|
+
/** Construct an EmbeddingVector from a Float32Array. Boundary validation only —
|
|
14
|
+
* callers (OllamaEmbedder, FakeEmbedder, vector deserialization) validate dimensions. */
|
|
15
|
+
export function asEmbeddingVector(raw) {
|
|
16
|
+
return raw;
|
|
17
|
+
}
|
|
13
18
|
const TAG_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
|
|
14
19
|
const MAX_TAG_LENGTH = 50;
|
|
15
20
|
const MAX_TAGS_PER_ENTRY = 10;
|