@wrongstack/vector-memory 1.0.0 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -14,796 +14,357 @@ var VectorMemoryProviderUnavailableError = class extends VectorMemoryError {
14
14
  }
15
15
  };
16
16
 
17
- // src/transformers-provider.ts
18
- var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
19
- var DEFAULT_VECTOR_DIMENSIONS = 384;
20
- var DEFAULT_VECTOR_DTYPE = "q8";
21
- var TransformersEmbeddingProvider = class {
22
- id;
23
- dimensions;
24
- modelId;
25
- cacheDir;
26
- dtype;
27
- device;
28
- batchSize;
29
- maxChars;
30
- allowRemote;
31
- extractor;
32
- loadPromise;
33
- constructor(opts = {}) {
34
- this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
35
- this.cacheDir = opts.cacheDir;
36
- this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
37
- this.device = opts.device ?? "cpu";
38
- this.batchSize = opts.batchSize ?? 16;
39
- this.maxChars = opts.maxChars ?? 2048;
40
- this.allowRemote = opts.allowRemoteModels ?? true;
41
- this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
42
- this.id = `transformers-js:${this.modelId}:${this.dtype}`;
17
+ // src/sage-event-mirror.ts
18
+ import * as fs from "node:fs";
19
+ import * as path from "node:path";
20
+ import { getSageSurface } from "@wrongstack/sage";
21
+ function subscribeVectorMemoryToSage(opts) {
22
+ const { store, memoryStore } = opts;
23
+ const log = opts.logger;
24
+ if (opts.enabled === false) {
25
+ return { dispose: () => void 0 };
43
26
  }
44
- /**
45
- * Synchronous capability check. Returns false when the optional
46
- * `@huggingface/transformers` dependency is not installed.
47
- *
48
- * NOTE: this probes via dynamic import and caches the result, but does
49
- * NOT load the model itself — model loading is deferred to `embed()`.
50
- */
51
- async isAvailable() {
27
+ const surface = getSageSurface(memoryStore);
28
+ if (!surface) {
29
+ log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
30
+ return { dispose: () => void 0 };
31
+ }
32
+ const events = memoryStore.events;
33
+ if (!events) {
34
+ log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
35
+ return { dispose: () => void 0 };
36
+ }
37
+ const fetch = async (memoryId) => {
52
38
  try {
53
- await this.loadModule();
54
- return true;
55
- } catch {
56
- return false;
39
+ return await surface.getSage(memoryId);
40
+ } catch (err) {
41
+ log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg(err)}`);
42
+ return null;
57
43
  }
58
- }
59
- async embed(texts) {
60
- if (texts.length === 0) return [];
61
- const extractor = await this.getExtractor();
62
- const prepared = texts.map((t) => this.prepare(t));
63
- const batches = [];
64
- for (let i = 0; i < prepared.length; i += this.batchSize) {
65
- batches.push(prepared.slice(i, i + this.batchSize));
44
+ };
45
+ const mirror = async (memoryId) => {
46
+ const memory = await fetch(memoryId);
47
+ if (!memory) return;
48
+ if (memory.scope === "session") return;
49
+ try {
50
+ const existing = store.findBySageId(memoryId);
51
+ if (existing) await store.forget(existing.id);
52
+ await store.remember({
53
+ text: memory.text,
54
+ ...memory.summary ? { summary: memory.summary } : {},
55
+ tags: memory.tags ?? [],
56
+ scope: "project",
57
+ kind: "note",
58
+ metadata: {
59
+ source: "sage",
60
+ sageId: memory.id,
61
+ sageKind: memory.kind,
62
+ sageScope: memory.scope,
63
+ importance: memory.importance,
64
+ confidence: memory.confidence
65
+ }
66
+ });
67
+ } catch (err) {
68
+ log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg(err)}`);
66
69
  }
67
- const results = [];
68
- for (const batch of batches) {
69
- const out = await extractor(batch, { pooling: "mean", normalize: true });
70
- results.push(...this.tensorToVectors(out, batch.length));
70
+ };
71
+ const forgetMirror = async (memoryId) => {
72
+ try {
73
+ const existing = store.findBySageId(memoryId);
74
+ if (existing) await store.forget(existing.id);
75
+ } catch (err) {
76
+ log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg(err)}`);
71
77
  }
72
- return results;
73
- }
74
- /** Truncate + normalize text before embedding. */
75
- prepare(text) {
76
- if (!text) return "";
77
- const normalized = text.normalize("NFKC").trim();
78
- return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
79
- }
80
- tensorToVectors(out, batchSize) {
81
- if (typeof out.tolist === "function") {
82
- const nested = out.tolist();
83
- if (Array.isArray(nested) && Array.isArray(nested[0])) {
84
- return nested.map((row) => Float32Array.from(row));
85
- }
86
- return [Float32Array.from(nested)];
78
+ };
79
+ const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
80
+ const memoryId = payload?.memoryId;
81
+ if (typeof memoryId !== "string") return;
82
+ void mirror(memoryId);
83
+ });
84
+ const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
85
+ const memoryId = payload?.memoryId;
86
+ if (typeof memoryId !== "string") return;
87
+ void mirror(memoryId);
88
+ });
89
+ const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
90
+ const memoryId = payload?.memoryId;
91
+ if (typeof memoryId !== "string") return;
92
+ const status = payload?.status;
93
+ if (status === "deleted") return;
94
+ void mirror(memoryId);
95
+ });
96
+ const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
97
+ const memoryId = payload?.memoryId;
98
+ if (typeof memoryId !== "string") return;
99
+ void forgetMirror(memoryId);
100
+ });
101
+ return {
102
+ dispose: () => {
103
+ offAccepted();
104
+ offRecovered();
105
+ offUpdated();
106
+ offDeleted();
87
107
  }
88
- const flat = out.data;
89
- if (flat instanceof Float32Array) {
90
- if (batchSize === 1) return [flat];
91
- const dim = flat.length / batchSize;
92
- const vectors = [];
93
- for (let i = 0; i < batchSize; i++) {
94
- vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
108
+ };
109
+ }
110
+ async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
111
+ const surface = getSageSurface(memoryStore);
112
+ if (!surface) return { scanned: 0, removed: 0 };
113
+ let scanned = 0;
114
+ let removed = 0;
115
+ const PAGE = Math.max(1, options?.pageSize ?? 500);
116
+ let after;
117
+ for (; ; ) {
118
+ const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
119
+ if (page.length === 0) break;
120
+ const last = page[page.length - 1];
121
+ after = { updatedAt: last.updatedAt, id: last.id };
122
+ for (const entry of page) {
123
+ scanned++;
124
+ const sageId = entry.metadata?.sageId;
125
+ if (typeof sageId !== "string") continue;
126
+ try {
127
+ const memory = await surface.getSage(sageId);
128
+ if (memory !== null && memory.status !== "deleted") continue;
129
+ await store.forget(entry.id);
130
+ removed++;
131
+ } catch (err) {
132
+ logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg(err)}`);
95
133
  }
96
- return vectors;
97
- }
98
- if (Array.isArray(flat) && Array.isArray(flat[0])) {
99
- return flat.map((row) => Float32Array.from(row));
100
134
  }
101
- if (Array.isArray(flat)) {
102
- return [Float32Array.from(flat)];
135
+ if (page.length < PAGE) break;
136
+ }
137
+ return { scanned, removed };
138
+ }
139
+ function errMsg(err) {
140
+ return err instanceof Error ? err.message : String(err);
141
+ }
142
+ var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
143
+ var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
144
+ async function sweepStaleSageMirrors(opts) {
145
+ const markerPath2 = path.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
146
+ const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
147
+ if (!opts.force) {
148
+ try {
149
+ const raw = JSON.parse(fs.readFileSync(markerPath2, "utf8"));
150
+ const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
151
+ if (Number.isFinite(at) && Date.now() - at < interval) {
152
+ return { swept: false, reason: "throttled" };
153
+ }
154
+ } catch {
103
155
  }
104
- throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
105
156
  }
106
- async getExtractor() {
107
- if (this.extractor) return this.extractor;
108
- if (!this.loadPromise) this.loadPromise = this.loadExtractor();
109
- this.extractor = await this.loadPromise;
110
- return this.extractor;
157
+ try {
158
+ fs.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
159
+ } catch {
111
160
  }
112
- async loadExtractor() {
113
- const mod = await this.loadModule();
114
- if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
115
- mod.env.allowRemoteModels = this.allowRemote;
116
- if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
117
- const pipe = await mod.pipeline("feature-extraction", this.modelId, {
118
- dtype: this.dtype,
119
- device: this.device
120
- });
121
- return pipe;
161
+ try {
162
+ const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
163
+ opts.logger?.debug?.(
164
+ `vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
165
+ );
166
+ return { swept: true, ...result };
167
+ } catch (err) {
168
+ opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg(err)}`);
169
+ return { swept: false, reason: errMsg(err) };
122
170
  }
123
- async loadModule() {
171
+ }
172
+
173
+ // src/sage-fusion.ts
174
+ var DEFAULT_RRF_K = 60;
175
+ var DEFAULT_VECTOR_WEIGHT = 0.3;
176
+ function lexicalRankScore(index, total) {
177
+ if (total <= 1) return 1;
178
+ return 1 - index / Math.max(1, total - 1);
179
+ }
180
+ async function fuseWithVectorMemory(query, lexical, options = {}) {
181
+ const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
182
+ const k = options.rrfK ?? DEFAULT_RRF_K;
183
+ const limit = options.limit ?? 25;
184
+ let vectorHits = [];
185
+ if (options.vectorHits) {
186
+ vectorHits = [...options.vectorHits];
187
+ } else if (options.store) {
124
188
  try {
125
- return await import("@huggingface/transformers");
126
- } catch (err) {
127
- throw new VectorMemoryProviderUnavailableError(
128
- "@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
129
- err
130
- );
189
+ vectorHits = await options.store.search(query, {
190
+ ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
191
+ limit: Math.max(limit * 2, 50)
192
+ });
193
+ } catch {
194
+ vectorHits = [];
131
195
  }
132
196
  }
133
- };
134
-
135
- // src/schema.ts
136
- var VECTOR_SCHEMA_VERSION = 2;
137
- var VECTOR_PROVIDER_KEY = "active_provider_id";
138
- var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
139
- function initVectorSchema(db) {
140
- db.exec("PRAGMA journal_mode = WAL");
141
- db.exec("PRAGMA synchronous = NORMAL");
142
- db.exec("PRAGMA busy_timeout = 30000");
143
- db.exec("PRAGMA temp_store = MEMORY");
144
- db.exec("PRAGMA foreign_keys = ON");
145
- db.exec(`
146
- CREATE TABLE IF NOT EXISTS schema_meta (
147
- key TEXT PRIMARY KEY,
148
- value TEXT NOT NULL
149
- );
150
- `);
151
- db.exec(`
152
- CREATE TABLE IF NOT EXISTS entries (
153
- id TEXT PRIMARY KEY,
154
- text TEXT NOT NULL,
155
- summary TEXT,
156
- metadata TEXT NOT NULL DEFAULT '{}',
157
- tags TEXT NOT NULL DEFAULT '[]',
158
- scope TEXT NOT NULL DEFAULT 'project',
159
- kind TEXT NOT NULL DEFAULT 'note',
160
- content_hash TEXT NOT NULL,
161
- created_at TEXT NOT NULL,
162
- updated_at TEXT NOT NULL
163
- );
164
- `);
165
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
166
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
167
- db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
168
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
169
- db.exec(`
170
- CREATE TABLE IF NOT EXISTS vectors (
171
- entry_id TEXT NOT NULL,
172
- provider_id TEXT NOT NULL,
173
- dimensions INTEGER NOT NULL,
174
- vector BLOB NOT NULL,
175
- created_at TEXT NOT NULL,
176
- PRIMARY KEY (entry_id, provider_id),
177
- FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
178
- );
179
- `);
180
- db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
181
- db.exec(`
182
- CREATE TABLE IF NOT EXISTS embedding_cache (
183
- content_hash TEXT NOT NULL,
184
- provider_id TEXT NOT NULL,
185
- dimensions INTEGER NOT NULL,
186
- vector BLOB NOT NULL,
187
- text TEXT NOT NULL,
188
- created_at TEXT NOT NULL,
189
- last_used_at TEXT NOT NULL,
190
- use_count INTEGER NOT NULL DEFAULT 0,
191
- PRIMARY KEY (content_hash, provider_id, dimensions)
192
- );
193
- `);
194
- db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
195
- db.exec(`
196
- CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
197
- id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
198
- );
199
- `);
200
- db.exec(`
201
- CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
202
- INSERT INTO entries_fts(rowid, text, tags)
203
- VALUES (new.rowid, new.text, new.tags);
204
- END;
205
- `);
206
- db.exec(`
207
- CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
208
- INSERT INTO entries_fts(entries_fts, rowid, text, tags)
209
- VALUES('delete', old.rowid, old.text, old.tags);
210
- END;
211
- `);
212
- db.exec(`
213
- CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
214
- INSERT INTO entries_fts(entries_fts, rowid, text, tags)
215
- VALUES('delete', old.rowid, old.text, old.tags);
216
- INSERT INTO entries_fts(rowid, text, tags)
217
- VALUES (new.rowid, new.text, new.tags);
218
- END;
219
- `);
220
- }
221
- function upsertEmbeddingCache(db, row) {
222
- db.prepare(
223
- `INSERT INTO embedding_cache
224
- (content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
225
- VALUES (?, ?, ?, ?, ?, ?, ?, 1)
226
- ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
227
- last_used_at = excluded.last_used_at,
228
- use_count = embedding_cache.use_count + 1,
229
- vector = excluded.vector`
230
- ).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
231
- }
232
- function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
233
- const row = db.prepare(
234
- `SELECT vector FROM embedding_cache
235
- WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
236
- LIMIT 1`
237
- ).get(contentHash, providerId, dimensions);
238
- if (!row) return void 0;
239
- try {
240
- db.prepare(
241
- `UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
242
- WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
243
- ).run(now, contentHash, providerId, dimensions);
244
- } catch {
197
+ const sageById = /* @__PURE__ */ new Map();
198
+ for (const memory of lexical) sageById.set(memory.id, memory);
199
+ const lexicalRanked = lexical.map((memory, index) => ({
200
+ memory,
201
+ rankScore: lexicalRankScore(index, lexical.length),
202
+ lexicalScore: lexicalRankScore(index, lexical.length)
203
+ }));
204
+ const vectorRanked = [];
205
+ const seenVectorSageIds = /* @__PURE__ */ new Set();
206
+ for (let i = 0; i < vectorHits.length; i++) {
207
+ const hit = vectorHits[i];
208
+ const sageId = hit.entry.metadata?.["sageId"];
209
+ if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
210
+ const memory = sageById.get(sageId);
211
+ if (!memory) continue;
212
+ seenVectorSageIds.add(sageId);
213
+ vectorRanked.push({ memory, vectorScore: hit.score });
245
214
  }
246
- return decodeVector(row.vector);
247
- }
248
- function encodeVector(vec) {
249
- return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
250
- }
251
- function decodeVector(buf) {
252
- if (buf.byteLength % 4 !== 0) {
253
- throw new Error(
254
- `decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
255
- );
215
+ const fused = /* @__PURE__ */ new Map();
216
+ for (let i = 0; i < lexicalRanked.length; i++) {
217
+ const c = lexicalRanked[i];
218
+ const rrf = (1 - weight) * (1 / (k + i + 1));
219
+ fused.set(c.memory.id, {
220
+ memory: c.memory,
221
+ vectorScore: null,
222
+ lexicalScore: c.lexicalScore,
223
+ finalScore: rrf,
224
+ source: "lexical"
225
+ });
256
226
  }
257
- const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
258
- const copy = new Float32Array(buf.byteLength / 4);
259
- for (let i = 0; i < copy.length; i++) {
260
- copy[i] = view.getFloat32(i * 4, true);
227
+ for (let i = 0; i < vectorRanked.length; i++) {
228
+ const v = vectorRanked[i];
229
+ const rrf = weight * (1 / (k + i + 1));
230
+ const existing = fused.get(v.memory.id);
231
+ if (existing) {
232
+ existing.finalScore += rrf;
233
+ existing.vectorScore = v.vectorScore;
234
+ existing.source = "both";
235
+ }
261
236
  }
262
- return copy;
263
- }
264
-
265
- // src/store.ts
266
- import { createHash, randomUUID } from "node:crypto";
267
- import * as fs from "node:fs";
268
- import * as path from "node:path";
269
- import { withFileLock } from "@wrongstack/core/utils";
270
- import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
271
- import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
272
- var DEFAULT_SEARCH_LIMIT = 10;
273
- function normalizeLimit(limit) {
274
- if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
275
- const floored = Math.floor(limit);
276
- return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
237
+ const out = Array.from(fused.values());
238
+ out.sort((a, b) => b.finalScore - a.finalScore);
239
+ return out.slice(0, limit);
277
240
  }
278
- var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
279
- var DEFAULT_FILENAME = "vector-memory.db";
280
- var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
281
- var CACHE_EVICT_BATCH = 256;
282
- var VectorMemoryStore = class _VectorMemoryStore {
283
- db;
284
- dbPath;
285
- rootDir;
286
- provider;
287
- closed = false;
288
- constructor(opts) {
289
- if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
290
- if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
291
- this.provider = opts.provider;
292
- const dir = opts.directory ?? DEFAULT_DIRECTORY;
293
- const filename = opts.filename ?? DEFAULT_FILENAME;
294
- if (path.isAbsolute(dir)) {
295
- throw new Error("Vector memory directory must be project-relative.");
241
+ function asVectorRecallProvider(store) {
242
+ return {
243
+ async search(query, opts) {
244
+ const hits = await store.search(query, {
245
+ limit: opts.limit,
246
+ ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
247
+ });
248
+ return hits.map((h) => ({
249
+ id: h.entry.id,
250
+ score: h.score,
251
+ text: h.entry.text,
252
+ ...h.entry.summary ? { summary: h.entry.summary } : {},
253
+ tags: h.entry.tags,
254
+ ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
255
+ }));
296
256
  }
297
- const rootDir = path.resolve(opts.projectRoot, dir);
298
- const rel = path.relative(path.resolve(opts.projectRoot), rootDir);
299
- if (rel.startsWith("..") || path.isAbsolute(rel)) {
300
- throw new Error("Vector memory directory must stay inside the project root.");
257
+ };
258
+ }
259
+ function clamp01(value) {
260
+ if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
261
+ if (value < 0) return 0;
262
+ if (value > 1) return 1;
263
+ return value;
264
+ }
265
+
266
+ // src/sage-port-wrapper.ts
267
+ import {
268
+ augmentLexicalWithVectorRecall,
269
+ isSageVisibleForSearch,
270
+ SAGE_RETRIEVAL_CAPABILITY,
271
+ SAGE_SURFACE_CAPABILITY
272
+ } from "@wrongstack/sage";
273
+ function asVectorRecallProviderAdapter(store) {
274
+ return {
275
+ async search(query, opts) {
276
+ const hits = await store.search(query, {
277
+ limit: opts.limit,
278
+ ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
279
+ });
280
+ return hits.map((h) => ({
281
+ id: h.entry.id,
282
+ score: h.score,
283
+ text: h.entry.text,
284
+ ...h.entry.summary ? { summary: h.entry.summary } : {},
285
+ tags: h.entry.tags,
286
+ ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
287
+ }));
301
288
  }
302
- fs.mkdirSync(rootDir, { recursive: true });
303
- this.rootDir = rootDir;
304
- this.dbPath = path.join(rootDir, filename);
305
- const Database = loadRuntimeDatabaseSync();
306
- this.db = new Database(this.dbPath);
307
- initVectorSchema(this.db);
308
- this.recordActiveProvider();
309
- }
310
- /**
311
- * Absolute path of the store's data directory (the resolved
312
- * `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
313
- * to place sidecar state (e.g. the first-boot SAGE sync marker) next to
314
- * the db instead of re-deriving the path and drifting from it.
315
- */
316
- get directory() {
317
- return this.rootDir;
318
- }
319
- /**
320
- * Absolute path of the SQLite database file. Hosts use this to take a
321
- * file-level lock that covers all mutating operations (see
322
- * `withFileLock(this.dbPath + '.lock', …)`).
323
- */
324
- get databasePath() {
325
- return this.dbPath;
326
- }
327
- /** The lockfile path used to serialize mutating operations. */
328
- get lockPath() {
329
- return `${this.dbPath}.lock`;
330
- }
331
- get activeProviderId() {
332
- const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
333
- return row?.value ?? this.provider.id;
334
- }
335
- recordActiveProvider() {
336
- this.db.exec("BEGIN");
337
- try {
338
- this.db.prepare(
339
- `INSERT INTO schema_meta (key, value) VALUES (?, ?)
340
- ON CONFLICT(key) DO UPDATE SET value = excluded.value`
341
- ).run(VECTOR_PROVIDER_KEY, this.provider.id);
342
- this.db.prepare(
343
- `INSERT INTO schema_meta (key, value) VALUES (?, ?)
344
- ON CONFLICT(key) DO UPDATE SET value = excluded.value`
345
- ).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
346
- this.db.exec("COMMIT");
347
- } catch (e) {
348
- this.db.exec("ROLLBACK");
349
- throw e;
350
- }
351
- }
352
- static contentHash(text) {
353
- return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
354
- }
355
- /**
356
- * Look up a vector for `text` in the provider-level embedding cache.
357
- * Cache hit returns the cached vector (no ONNX pass). Miss returns
358
- * `undefined`.
359
- */
360
- cachedVector(text, now) {
361
- return lookupEmbeddingCache(
362
- this.db,
363
- _VectorMemoryStore.contentHash(text),
364
- this.provider.id,
365
- this.provider.dimensions,
366
- now
367
- );
368
- }
369
- /** Persist `vec` for `text` to the embedding cache. */
370
- cacheVector(text, vec, now) {
371
- upsertEmbeddingCache(this.db, {
372
- contentHash: _VectorMemoryStore.contentHash(text),
373
- providerId: this.provider.id,
374
- dimensions: vec.length,
375
- vector: encodeVector(vec),
376
- text,
377
- now
378
- });
379
- }
380
- /**
381
- * Embed `text`, hitting the provider-level cache first. Cache miss falls
382
- * through to the configured provider and writes the result back. Returns
383
- * `undefined` when the provider fails — the caller can persist the entry
384
- * without a vector (fail-open).
385
- */
386
- async embedWithCache(text) {
387
- const now = (/* @__PURE__ */ new Date()).toISOString();
388
- const cached = this.cachedVector(text, now);
389
- if (cached) return cached;
390
- try {
391
- const result = await this.provider.embed([text]);
392
- const vec = result[0];
393
- if (vec) this.cacheVector(text, vec, now);
394
- return vec;
395
- } catch {
396
- return void 0;
397
- }
398
- }
399
- /**
400
- * Look up an existing entry by `content_hash`. Returns `undefined` when
401
- * the entry is not present. Used by `remember()` to make writes idempotent
402
- * and by `syncFromSage()` to skip already-indexed SAGE memories.
403
- */
404
- findByContentHash(contentHash) {
405
- this.assertOpen();
406
- const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
407
- if (!row) return void 0;
408
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
409
- return this.rowToEntry(row, vectorRow);
410
- }
411
- /**
412
- * Look up the entry mirroring a given SAGE memory id (i.e. the entry
413
- * whose `metadata.sageId` equals `sageId`). Returns `undefined` when
414
- * no such entry exists. Used by the event-driven mirror to delete
415
- * vector entries on SAGE delete events (the emitter knows the SAGE
416
- * id, not the vector entry id).
417
- *
418
- * Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
419
- * column is the JSON blob `syncFromSage` writes, so this avoids a
420
- * full table scan.
421
- */
422
- findBySageId(sageId) {
423
- this.assertOpen();
424
- const row = this.db.prepare(
425
- `SELECT * FROM entries
426
- WHERE json_extract(metadata, '$.sageId') = ?
427
- ORDER BY updated_at DESC
428
- LIMIT 1`
429
- ).get(sageId);
430
- if (!row) return void 0;
431
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
432
- return this.rowToEntry(row, vectorRow);
433
- }
434
- /**
435
- * Persist a new entry. Idempotent: if an entry with the same
436
- * `content_hash` already exists, that entry is returned unchanged
437
- * instead of inserting a duplicate. Mutating ops are wrapped in
438
- * `withFileLock` so two processes cannot race the dedup check.
439
- */
440
- async remember(input) {
441
- this.assertOpen();
442
- if (!input.text || input.text.trim().length === 0) {
443
- throw new Error("VectorMemoryStore.remember: text must be non-empty");
444
- }
445
- return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
446
- timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
447
- });
448
- }
449
- async rememberUnlocked(input) {
450
- const now = (/* @__PURE__ */ new Date()).toISOString();
451
- const contentHash = _VectorMemoryStore.contentHash(input.text);
452
- const existing = this.findByContentHash(contentHash);
453
- if (existing) return existing;
454
- const metadata = input.metadata ?? {};
455
- const tags = input.tags ?? [];
456
- const scope = input.scope ?? "project";
457
- const kind = input.kind ?? "note";
458
- const id = randomUUID();
459
- const vector = await this.embedWithCache(input.text);
460
- const providerId = vector ? this.provider.id : void 0;
461
- this.db.exec("BEGIN");
462
- try {
463
- this.db.prepare(
464
- `INSERT INTO entries
465
- (id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
466
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
467
- ).run(
468
- id,
469
- input.text,
470
- input.summary ?? null,
471
- JSON.stringify(metadata),
472
- JSON.stringify(tags),
473
- scope,
474
- kind,
475
- contentHash,
476
- now,
477
- now
478
- );
479
- if (vector && providerId) {
480
- this.db.prepare(
481
- `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
482
- VALUES (?, ?, ?, ?, ?)
483
- ON CONFLICT(entry_id, provider_id) DO UPDATE SET
484
- vector = excluded.vector,
485
- dimensions = excluded.dimensions,
486
- created_at = excluded.created_at`
487
- ).run(id, providerId, vector.length, encodeVector(vector), now);
488
- }
489
- this.db.exec("COMMIT");
490
- } catch (e) {
491
- this.db.exec("ROLLBACK");
492
- throw e;
493
- }
494
- const result = {
495
- id,
496
- text: input.text,
497
- summary: input.summary ?? void 0,
498
- metadata,
499
- tags,
500
- scope,
501
- kind,
502
- contentHash,
503
- createdAt: now,
504
- updatedAt: now,
505
- providerId: providerId ?? "",
506
- dimensions: vector?.length ?? 0
507
- };
508
- if (vector) result.vector = vector;
509
- return result;
510
- }
511
- get(id) {
512
- this.assertOpen();
513
- const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
514
- if (!row) return void 0;
515
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
516
- return this.rowToEntry(row, vectorRow);
517
- }
518
- /** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
519
- async forget(id) {
520
- this.assertOpen();
521
- return withFileLock(
522
- this.lockPath,
523
- async () => {
524
- this.db.exec("BEGIN");
525
- try {
526
- const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
527
- this.db.exec("COMMIT");
528
- return info.changes > 0;
529
- } catch (e) {
530
- this.db.exec("ROLLBACK");
531
- throw e;
532
- }
533
- },
534
- { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
535
- );
536
- }
537
- async search(query, opts = {}) {
538
- this.assertOpen();
539
- const limit = normalizeLimit(opts.limit);
540
- const threshold = opts.threshold ?? 0;
541
- const includeVectors = opts.includeVectors === true;
542
- if (typeof query !== "string" || query.trim().length === 0) return [];
543
- const queryVec = await this.embedWithCache(query);
544
- if (!queryVec || queryVec.length === 0) return [];
545
- const providerId = this.provider.id;
546
- const dimensions = this.provider.dimensions;
547
- const filters = ["v.provider_id = ?", "v.dimensions = ?"];
548
- const params = [providerId, dimensions];
549
- if (opts.scope !== void 0) {
550
- filters.push("e.scope = ?");
551
- params.push(opts.scope);
552
- }
553
- if (opts.kind !== void 0) {
554
- filters.push("e.kind = ?");
555
- params.push(opts.kind);
556
- }
557
- const scanRows = this.db.prepare(
558
- `SELECT e.id AS id, v.vector AS vec_blob
559
- FROM entries e
560
- JOIN vectors v ON v.entry_id = e.id
561
- WHERE ${filters.join(" AND ")}`
562
- ).all(...params);
563
- const top = [];
564
- for (const row of scanRows) {
565
- const vec = decodeVector(row.vec_blob);
566
- const raw = cosineSimilarity(queryVec, vec);
567
- const score = Math.max(0, Math.min(1, raw));
568
- if (score < threshold) continue;
569
- if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
570
- let at = top.length;
571
- while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
572
- top.splice(at, 0, { id: row.id, score, vector: vec });
573
- if (top.length > limit) top.length = limit;
574
- }
575
- if (top.length === 0) return [];
576
- const placeholders = top.map(() => "?").join(",");
577
- const hydrated = this.db.prepare(
578
- `SELECT id, text, summary, metadata, tags, scope, kind,
579
- content_hash, created_at, updated_at
580
- FROM entries WHERE id IN (${placeholders})`
581
- ).all(...top.map((t) => t.id));
582
- const entryById = /* @__PURE__ */ new Map();
583
- for (const row of hydrated) {
584
- entryById.set(row.id, this.rowToEntry(row));
585
- }
586
- const scored = [];
587
- for (const candidate of top) {
588
- const entry = entryById.get(candidate.id);
589
- if (!entry) continue;
590
- const hit = { entry, score: candidate.score, providerId };
591
- if (includeVectors) hit.vector = candidate.vector;
592
- scored.push(hit);
593
- }
594
- return scored;
595
- }
596
- /**
597
- * Page through entries, newest first.
598
- *
599
- * Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
600
- * without the id tiebreak two entries written in the same millisecond can
601
- * swap places between calls and a paging caller silently skips one.
602
- *
603
- * Pagination is keyset (`after`), not offset, because the only caller that
604
- * pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
605
- * `OFFSET` every deletion shifts the remaining rows left and the next page
606
- * skips exactly as many entries as were removed. Keyset is immune: it
607
- * resumes from a position, and the rows a deletion removes are ones the
608
- * sweep has already passed.
609
- */
610
- list(opts = {}) {
611
- this.assertOpen();
612
- const where = [];
613
- const params = [];
614
- if (opts.scope !== void 0) {
615
- where.push("scope = ?");
616
- params.push(opts.scope);
617
- }
618
- if (opts.kind !== void 0) {
619
- where.push("kind = ?");
620
- params.push(opts.kind);
621
- }
622
- if (opts.after) {
623
- where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
624
- params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
625
- }
626
- const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
627
- ORDER BY updated_at DESC, id DESC LIMIT ?`;
628
- params.push(opts.limit ?? 100);
629
- const rows = this.db.prepare(sql).all(...params);
630
- return rows.map((r) => this.rowToEntry(r));
631
- }
632
- async reindexAll() {
633
- this.assertOpen();
634
- return withFileLock(
635
- this.lockPath,
636
- async () => {
637
- const rows = this.db.prepare("SELECT id, text FROM entries").all();
638
- let processed = 0;
639
- let errors = 0;
640
- for (const row of rows) {
641
- try {
642
- const result = await this.provider.embed([row.text]);
643
- const v = result[0];
644
- if (!v) {
645
- errors++;
646
- continue;
647
- }
648
- const now = (/* @__PURE__ */ new Date()).toISOString();
649
- this.db.prepare(
650
- `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
651
- VALUES (?, ?, ?, ?, ?)
652
- ON CONFLICT(entry_id, provider_id) DO UPDATE SET
653
- vector = excluded.vector,
654
- dimensions = excluded.dimensions,
655
- created_at = excluded.created_at`
656
- ).run(row.id, this.provider.id, v.length, encodeVector(v), now);
657
- this.cacheVector(row.text, v, now);
658
- processed++;
659
- } catch {
660
- errors++;
661
- }
662
- }
663
- return { processed, errors };
664
- },
665
- { timeoutMs: 6e4 }
666
- );
667
- }
668
- stats() {
669
- this.assertOpen();
670
- const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
671
- const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
672
- const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
673
- return {
674
- entries: entryCount,
675
- vectors: vectorCount,
676
- providers: providerRows.map((r) => r.provider_id),
677
- modelAvailable: true,
678
- modelId: this.provider.id,
679
- dimensions: this.provider.dimensions
680
- };
681
- }
682
- /**
683
- * Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
684
- * Useful for the WebUI's vector-memory panel and for diagnosing the
685
- * "why is search slow?" question.
686
- */
687
- cacheStats() {
688
- this.assertOpen();
689
- const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
690
- const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
691
- const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
692
- const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
693
- return {
694
- entries,
695
- providers,
696
- totalUseCount,
697
- oldestLastUsedAt: oldest?.t ?? null
698
- };
699
- }
700
- /**
701
- * LRU-evict the embedding cache down to `keepMostRecent` rows. The
702
- * `embedding_cache` table is independent of `entries`, so a sweep here
703
- * only removes cached vectors — never stored entries. Called by hosts
704
- * that want to bound cache growth on long-lived processes.
705
- */
706
- async evictCache(keepMostRecent) {
707
- this.assertOpen();
708
- if (keepMostRecent < 0) {
709
- throw new Error("evictCache: keepMostRecent must be >= 0");
710
- }
711
- return withFileLock(
712
- this.lockPath,
713
- async () => {
714
- const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
715
- if (total <= keepMostRecent) return { removed: 0 };
716
- const toRemove = total - keepMostRecent;
717
- const stmt = this.db.prepare(
718
- `DELETE FROM embedding_cache
719
- WHERE content_hash IN (
720
- SELECT content_hash FROM embedding_cache
721
- ORDER BY last_used_at ASC
722
- LIMIT ?
723
- )`
724
- );
725
- const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
726
- return { removed: Number(info.changes) };
727
- },
728
- { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
289
+ };
290
+ }
291
+ function wrapMemoryPortWithVectorRecall(port, options) {
292
+ const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
293
+ const materializeFor = (searchOpts) => async (sageId) => {
294
+ const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
295
+ if (!surface?.getSage) return void 0;
296
+ const memory = await surface.getSage(sageId);
297
+ if (!memory) return void 0;
298
+ return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
299
+ };
300
+ const fusionOptions = (searchOpts) => ({
301
+ vectorRecall: recall,
302
+ materializeVectorOnly: materializeFor(searchOpts),
303
+ ...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
304
+ ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
305
+ ...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
306
+ ...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
307
+ ...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
308
+ });
309
+ const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
310
+ const wrapSearchSage = (original) => async (query, searchOpts) => {
311
+ const opts = searchOpts;
312
+ const lexical = await original(query, searchOpts);
313
+ if (callerOwnsFusion(opts)) return lexical;
314
+ const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
315
+ return fused.map((hit) => hit.memory);
316
+ };
317
+ const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
318
+ const opts = searchOpts;
319
+ const lexicalHits = await original(query, searchOpts);
320
+ if (callerOwnsFusion(opts)) return lexicalHits;
321
+ return augmentLexicalWithVectorRecall(
322
+ query,
323
+ lexicalHits.map((hit) => hit.memory),
324
+ fusionOptions(opts)
729
325
  );
730
- }
731
- close() {
732
- if (this.closed) return;
733
- this.closed = true;
734
- this.db.close();
735
- }
736
- assertOpen() {
737
- if (this.closed) throw new Error("VectorMemoryStore is closed");
738
- }
739
- rowToEntry(row, vectorRow) {
740
- const summaryValue = row.summary;
741
- const entry = {
742
- id: row.id,
743
- text: row.text,
744
- summary: summaryValue ?? void 0,
745
- metadata: safeParseJson(row.metadata, {}),
746
- tags: safeParseJson(row.tags, []),
747
- scope: row.scope,
748
- kind: row.kind,
749
- contentHash: row.content_hash,
750
- createdAt: row.created_at,
751
- updatedAt: row.updated_at,
752
- providerId: vectorRow?.provider_id ?? "",
753
- dimensions: vectorRow?.dimensions ?? 0
754
- };
755
- if (vectorRow?.vector) {
756
- entry.vector = decodeVector(vectorRow.vector);
326
+ };
327
+ const wrapped = Object.create(
328
+ Object.getPrototypeOf(port),
329
+ Object.getOwnPropertyDescriptors(port)
330
+ );
331
+ wrapped.getCapability = (capability) => {
332
+ if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
333
+ const original = port.getCapability(capability);
334
+ if (!original) return void 0;
335
+ return {
336
+ ...original,
337
+ searchSage: wrapSearchSage(original.searchSage),
338
+ ...original.searchSageWithBreakdown ? {
339
+ searchSageWithBreakdown: wrapSearchWithBreakdown(
340
+ original.searchSageWithBreakdown
341
+ )
342
+ } : {}
343
+ };
757
344
  }
758
- return entry;
759
- }
760
- async syncFromSage(sage) {
761
- this.assertOpen();
762
- const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
763
- let indexed = 0;
764
- let skipped = 0;
765
- let failed = 0;
766
- const errors = [];
767
- for (const memory of memories) {
768
- try {
769
- const hash = _VectorMemoryStore.contentHash(memory.text);
770
- const existing = this.findByContentHash(hash);
771
- if (existing) {
772
- skipped++;
773
- continue;
774
- }
775
- await this.rememberUnlocked({
776
- text: memory.text,
777
- summary: memory.summary ?? void 0,
778
- metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
779
- tags: memory.tags ?? [],
780
- scope: "project",
781
- kind: "note"
782
- });
783
- indexed++;
784
- } catch (err) {
785
- failed++;
786
- errors.push({ memoryId: memory.id, message: errMsg(err) });
787
- }
345
+ if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
346
+ const original = port.getCapability(capability);
347
+ if (!original) return void 0;
348
+ return {
349
+ ...original,
350
+ searchSage: wrapSearchSage(original.searchSage),
351
+ ...original.searchSageWithBreakdown ? {
352
+ searchSageWithBreakdown: wrapSearchWithBreakdown(
353
+ original.searchSageWithBreakdown
354
+ )
355
+ } : {}
356
+ };
788
357
  }
789
- return { scanned: memories.length, indexed, skipped, failed, errors };
790
- }
791
- };
792
- function fallbackHashingProvider(dimensions) {
793
- return new HashingEmbeddingProvider({ dimensions });
794
- }
795
- function safeParseJson(value, fallback) {
796
- if (typeof value !== "string") return fallback;
797
- try {
798
- return JSON.parse(value);
799
- } catch {
800
- return fallback;
801
- }
802
- }
803
- function errMsg(err) {
804
- return err instanceof Error ? err.message : String(err);
358
+ return port.getCapability(capability);
359
+ };
360
+ return wrapped;
805
361
  }
806
362
 
363
+ // src/sage-sync.ts
364
+ import * as fs2 from "node:fs";
365
+ import * as path2 from "node:path";
366
+ import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
367
+
807
368
  // src/sage-sync-source.ts
808
369
  var DEFAULT_PAGE_SIZE = 500;
809
370
  var HARD_FLOOR_PAGE = 1;
@@ -840,7 +401,7 @@ function createSageSurfaceSyncSource(sage, opts = {}) {
840
401
  }
841
402
  });
842
403
  }
843
- if (!page.nextCursor || rows.length === 0) break;
404
+ if (!page.nextCursor) break;
844
405
  if (rows.length === 0) {
845
406
  noProgressPages++;
846
407
  if (noProgressPages > 3) break;
@@ -858,9 +419,6 @@ function clamp(value, min, max) {
858
419
  }
859
420
 
860
421
  // src/sage-sync.ts
861
- import { getSageSurface } from "@wrongstack/sage";
862
- import * as fs2 from "node:fs";
863
- import * as path2 from "node:path";
864
422
  var SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
865
423
  var RUNNING_STALE_MS = 10 * 60 * 1e3;
866
424
  async function startFirstBootSageSync(opts) {
@@ -901,7 +459,7 @@ async function startFirstBootSageSync(opts) {
901
459
  return { synced: false, reason: "provider-unavailable" };
902
460
  }
903
461
  }
904
- const surface = getSageSurface(memoryStore);
462
+ const surface = getSageSurface2(memoryStore);
905
463
  if (!surface) {
906
464
  log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
907
465
  return { synced: false, reason: "no-sage-surface" };
@@ -1033,608 +591,1048 @@ function counts(report) {
1033
591
  };
1034
592
  }
1035
593
 
1036
- // src/tools.ts
1037
- function createVectorMemoryTools(store) {
1038
- return [
1039
- vectorMemoryRememberTool(store),
1040
- vectorMemorySearchTool(store),
1041
- vectorMemoryStatsTool(store),
1042
- vectorMemoryForgetTool(store)
1043
- ];
594
+ // src/schema.ts
595
+ var VECTOR_SCHEMA_VERSION = 2;
596
+ var VECTOR_PROVIDER_KEY = "active_provider_id";
597
+ var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
598
+ function initVectorSchema(db) {
599
+ db.exec("PRAGMA journal_mode = WAL");
600
+ db.exec("PRAGMA synchronous = NORMAL");
601
+ db.exec("PRAGMA busy_timeout = 30000");
602
+ db.exec("PRAGMA temp_store = MEMORY");
603
+ db.exec("PRAGMA foreign_keys = ON");
604
+ db.exec(`
605
+ CREATE TABLE IF NOT EXISTS schema_meta (
606
+ key TEXT PRIMARY KEY,
607
+ value TEXT NOT NULL
608
+ );
609
+ `);
610
+ db.exec(`
611
+ CREATE TABLE IF NOT EXISTS entries (
612
+ id TEXT PRIMARY KEY,
613
+ text TEXT NOT NULL,
614
+ summary TEXT,
615
+ metadata TEXT NOT NULL DEFAULT '{}',
616
+ tags TEXT NOT NULL DEFAULT '[]',
617
+ scope TEXT NOT NULL DEFAULT 'project',
618
+ kind TEXT NOT NULL DEFAULT 'note',
619
+ content_hash TEXT NOT NULL,
620
+ created_at TEXT NOT NULL,
621
+ updated_at TEXT NOT NULL
622
+ );
623
+ `);
624
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
625
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
626
+ db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
627
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
628
+ db.exec(`
629
+ CREATE TABLE IF NOT EXISTS vectors (
630
+ entry_id TEXT NOT NULL,
631
+ provider_id TEXT NOT NULL,
632
+ dimensions INTEGER NOT NULL,
633
+ vector BLOB NOT NULL,
634
+ created_at TEXT NOT NULL,
635
+ PRIMARY KEY (entry_id, provider_id),
636
+ FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
637
+ );
638
+ `);
639
+ db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
640
+ db.exec(`
641
+ CREATE TABLE IF NOT EXISTS embedding_cache (
642
+ content_hash TEXT NOT NULL,
643
+ provider_id TEXT NOT NULL,
644
+ dimensions INTEGER NOT NULL,
645
+ vector BLOB NOT NULL,
646
+ text TEXT NOT NULL,
647
+ created_at TEXT NOT NULL,
648
+ last_used_at TEXT NOT NULL,
649
+ use_count INTEGER NOT NULL DEFAULT 0,
650
+ PRIMARY KEY (content_hash, provider_id, dimensions)
651
+ );
652
+ `);
653
+ db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
654
+ db.exec(`
655
+ CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
656
+ id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
657
+ );
658
+ `);
659
+ db.exec(`
660
+ CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
661
+ INSERT INTO entries_fts(rowid, text, tags)
662
+ VALUES (new.rowid, new.text, new.tags);
663
+ END;
664
+ `);
665
+ db.exec(`
666
+ CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
667
+ INSERT INTO entries_fts(entries_fts, rowid, text, tags)
668
+ VALUES('delete', old.rowid, old.text, old.tags);
669
+ END;
670
+ `);
671
+ db.exec(`
672
+ CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
673
+ INSERT INTO entries_fts(entries_fts, rowid, text, tags)
674
+ VALUES('delete', old.rowid, old.text, old.tags);
675
+ INSERT INTO entries_fts(rowid, text, tags)
676
+ VALUES (new.rowid, new.text, new.tags);
677
+ END;
678
+ `);
1044
679
  }
1045
- function vectorMemoryRememberTool(store) {
1046
- return {
1047
- name: "vector_memory_remember",
1048
- category: "Memory",
1049
- description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
1050
- usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
1051
- permission: "confirm",
1052
- mutating: true,
1053
- riskTier: "standard",
1054
- timeoutMs: 5e3,
1055
- capabilities: ["memory.write"],
1056
- icon: "settings",
1057
- inputSchema: {
1058
- type: "object",
1059
- properties: {
1060
- text: { type: "string", minLength: 1, description: "The text to embed and store." },
1061
- summary: { type: "string", description: "Optional short label." },
1062
- tags: {
1063
- type: "array",
1064
- items: { type: "string" },
1065
- description: "Optional tags for later filtering."
1066
- },
1067
- scope: {
1068
- type: "string",
1069
- enum: ["project", "user", "session"],
1070
- description: "Visibility scope. Defaults to `project`."
1071
- },
1072
- kind: {
1073
- type: "string",
1074
- enum: ["note", "fact", "summary", "snippet", "link"],
1075
- description: "Entry kind. Defaults to `note`."
1076
- },
1077
- metadata: {
1078
- type: "object",
1079
- additionalProperties: true,
1080
- description: "Free-form metadata persisted as JSON."
1081
- }
1082
- },
1083
- required: ["text"],
1084
- additionalProperties: false
1085
- },
1086
- execute: async (input) => {
1087
- const entry = await store.remember(input);
1088
- return { id: entry.id, hasVector: entry.vector !== void 0 };
680
+ function upsertEmbeddingCache(db, row) {
681
+ db.prepare(
682
+ `INSERT INTO embedding_cache
683
+ (content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
684
+ VALUES (?, ?, ?, ?, ?, ?, ?, 1)
685
+ ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
686
+ last_used_at = excluded.last_used_at,
687
+ use_count = embedding_cache.use_count + 1,
688
+ vector = excluded.vector`
689
+ ).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
690
+ }
691
+ function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
692
+ const row = db.prepare(
693
+ `SELECT vector FROM embedding_cache
694
+ WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
695
+ LIMIT 1`
696
+ ).get(contentHash, providerId, dimensions);
697
+ if (!row) return void 0;
698
+ try {
699
+ db.prepare(
700
+ `UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
701
+ WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
702
+ ).run(now, contentHash, providerId, dimensions);
703
+ } catch {
704
+ }
705
+ return decodeVector(row.vector);
706
+ }
707
+ function encodeVector(vec) {
708
+ return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
709
+ }
710
+ function decodeVector(buf) {
711
+ if (buf.byteLength % 4 !== 0) {
712
+ throw new Error(
713
+ `decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
714
+ );
715
+ }
716
+ const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
717
+ const copy = new Float32Array(buf.byteLength / 4);
718
+ for (let i = 0; i < copy.length; i++) {
719
+ copy[i] = view.getFloat32(i * 4, true);
720
+ }
721
+ return copy;
722
+ }
723
+
724
+ // src/search-race.ts
725
+ function previewText(text, maxLen) {
726
+ if (text.length <= maxLen) return text;
727
+ return text.slice(0, maxLen - 1) + "\u2026";
728
+ }
729
+ async function runSearchRace(query, lexical, vectorStore, options = {}) {
730
+ const limit = options.limit ?? 20;
731
+ const threshold = options.threshold ?? 0;
732
+ let vectorHits = [];
733
+ try {
734
+ vectorHits = await vectorStore.search(query, {
735
+ limit,
736
+ ...threshold > 0 ? { threshold } : {}
737
+ });
738
+ } catch {
739
+ vectorHits = [];
740
+ }
741
+ const lexicalOnly = [];
742
+ const vectorOnly = [];
743
+ const overlap = [];
744
+ const seenIds = /* @__PURE__ */ new Set();
745
+ const lexicalCapped = lexical.slice(0, limit);
746
+ for (let i = 0; i < lexicalCapped.length; i++) {
747
+ const mem = lexicalCapped[i];
748
+ const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
749
+ const id = mem.id;
750
+ seenIds.add(id);
751
+ overlap.push({
752
+ id,
753
+ lexicalScore: score,
754
+ vectorScore: null,
755
+ // patched below when a vector hit carries this id
756
+ preview: previewText(mem.text, 140)
757
+ });
758
+ }
759
+ const overlapById = /* @__PURE__ */ new Map();
760
+ for (const row of overlap) overlapById.set(row.id, row);
761
+ for (const hit of vectorHits) {
762
+ const sageId = hit.entry.metadata?.["sageId"];
763
+ if (typeof sageId !== "string") continue;
764
+ const existing = overlapById.get(sageId);
765
+ if (existing) {
766
+ if (existing.vectorScore === null) {
767
+ existing.vectorScore = hit.score;
768
+ }
769
+ continue;
1089
770
  }
1090
- };
1091
- }
1092
- function vectorMemorySearchTool(store) {
1093
- return {
1094
- name: "vector_memory_search",
1095
- category: "Memory",
1096
- description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
1097
- usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
1098
- permission: "auto",
1099
- mutating: false,
1100
- riskTier: "safe",
1101
- timeoutMs: 5e3,
1102
- capabilities: ["memory.read"],
1103
- icon: "search",
1104
- inputSchema: {
1105
- type: "object",
1106
- properties: {
1107
- query: { type: "string", minLength: 1, description: "The natural-language query." },
1108
- limit: {
1109
- type: "number",
1110
- minimum: 1,
1111
- maximum: 100,
1112
- description: "Max results (default 10)."
1113
- },
1114
- threshold: {
1115
- type: "number",
1116
- minimum: 0,
1117
- maximum: 1,
1118
- description: "Minimum cosine similarity. Results below the floor are dropped."
1119
- },
1120
- scope: {
1121
- type: "string",
1122
- enum: ["project", "user", "session"],
1123
- description: "Restrict to a scope."
1124
- },
1125
- kind: {
1126
- type: "string",
1127
- enum: ["note", "fact", "summary", "snippet", "link"],
1128
- description: "Restrict to a kind."
1129
- }
1130
- },
1131
- required: ["query"],
1132
- additionalProperties: false
1133
- },
1134
- execute: async (input) => {
1135
- const hits = await store.search(input.query, {
1136
- limit: input.limit !== void 0 ? input.limit : void 0,
1137
- threshold: input.threshold !== void 0 ? input.threshold : void 0,
1138
- scope: input.scope,
1139
- kind: input.kind
771
+ if (seenIds.has(sageId)) continue;
772
+ seenIds.add(sageId);
773
+ vectorOnly.push({
774
+ id: sageId,
775
+ lexicalScore: null,
776
+ vectorScore: hit.score,
777
+ preview: previewText(hit.entry.text, 140)
778
+ });
779
+ }
780
+ const finalOverlap = [];
781
+ for (const row of overlap) {
782
+ if (row.vectorScore === null) {
783
+ lexicalOnly.push({
784
+ id: row.id,
785
+ lexicalScore: row.lexicalScore,
786
+ vectorScore: null,
787
+ preview: row.preview
1140
788
  });
1141
- return {
1142
- hits: hits.map((h) => ({
1143
- id: h.entry.id,
1144
- score: h.score,
1145
- text: h.entry.text,
1146
- summary: h.entry.summary ?? void 0,
1147
- tags: h.entry.tags
1148
- }))
1149
- };
789
+ } else {
790
+ finalOverlap.push(row);
1150
791
  }
1151
- };
1152
- }
1153
- function vectorMemoryStatsTool(store) {
1154
- return {
1155
- name: "vector_memory_stats",
1156
- category: "Memory",
1157
- description: "Return counts, providers, and dimensions for the local vector memory store.",
1158
- usageHint: "Cheap diagnostic \u2014 safe to call any time.",
1159
- permission: "auto",
1160
- mutating: false,
1161
- riskTier: "safe",
1162
- timeoutMs: 1e3,
1163
- capabilities: ["memory.read"],
1164
- icon: "search",
1165
- inputSchema: {
1166
- type: "object",
1167
- properties: {},
1168
- additionalProperties: false
1169
- },
1170
- execute: async () => store.stats()
1171
- };
1172
- }
1173
- function vectorMemoryForgetTool(store) {
792
+ }
793
+ const lexicalCount = lexicalOnly.length + finalOverlap.length;
794
+ const vectorCount = vectorOnly.length + finalOverlap.length;
795
+ const denom = Math.max(lexicalCount, vectorCount, 1);
1174
796
  return {
1175
- name: "vector_memory_forget",
1176
- category: "Memory",
1177
- description: "Remove an entry (and its vector) from the vector memory store.",
1178
- usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
1179
- permission: "confirm",
1180
- mutating: true,
1181
- riskTier: "standard",
1182
- timeoutMs: 1e3,
1183
- capabilities: ["memory.write"],
1184
- icon: "settings",
1185
- inputSchema: {
1186
- type: "object",
1187
- properties: {
1188
- id: {
1189
- type: "string",
1190
- minLength: 1,
1191
- description: "Entry id returned by `vector_memory_remember`."
1192
- }
1193
- },
1194
- required: ["id"],
1195
- additionalProperties: false
1196
- },
1197
- execute: async (input) => ({ removed: await store.forget(input.id) })
797
+ query,
798
+ lexicalOnly,
799
+ vectorOnly,
800
+ overlap: finalOverlap,
801
+ metrics: {
802
+ lexicalCount,
803
+ vectorCount,
804
+ overlapCount: finalOverlap.length,
805
+ lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
806
+ vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
807
+ agreementRatio: finalOverlap.length / denom
808
+ }
1198
809
  };
1199
810
  }
1200
811
 
1201
- // src/sage-fusion.ts
1202
- var DEFAULT_RRF_K = 60;
1203
- var DEFAULT_VECTOR_WEIGHT = 0.3;
1204
- function lexicalRankScore(index, total) {
1205
- if (total <= 1) return 1;
1206
- return 1 - index / Math.max(1, total - 1);
812
+ // src/store.ts
813
+ import { createHash, randomUUID } from "node:crypto";
814
+ import * as fs3 from "node:fs";
815
+ import * as path3 from "node:path";
816
+ import { withFileLock } from "@wrongstack/core/utils";
817
+ import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
818
+ import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
819
+ var DEFAULT_SEARCH_LIMIT = 10;
820
+ function normalizeLimit(limit) {
821
+ if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
822
+ const floored = Math.floor(limit);
823
+ return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
1207
824
  }
1208
- async function fuseWithVectorMemory(query, lexical, options = {}) {
1209
- const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
1210
- const k = options.rrfK ?? DEFAULT_RRF_K;
1211
- const limit = options.limit ?? 25;
1212
- const vectorOnlyThreshold = options.vectorOnlyThreshold ?? 0;
1213
- let vectorHits = [];
1214
- if (options.vectorHits) {
1215
- vectorHits = [...options.vectorHits];
1216
- } else if (options.store) {
825
+ var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
826
+ var DEFAULT_FILENAME = "vector-memory.db";
827
+ var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
828
+ var CACHE_EVICT_BATCH = 256;
829
+ var VectorMemoryStore = class _VectorMemoryStore {
830
+ db;
831
+ dbPath;
832
+ rootDir;
833
+ provider;
834
+ closed = false;
835
+ constructor(opts) {
836
+ if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
837
+ if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
838
+ this.provider = opts.provider;
839
+ const dir = opts.directory ?? DEFAULT_DIRECTORY;
840
+ const filename = opts.filename ?? DEFAULT_FILENAME;
841
+ if (path3.isAbsolute(dir)) {
842
+ throw new Error("Vector memory directory must be project-relative.");
843
+ }
844
+ const rootDir = path3.resolve(opts.projectRoot, dir);
845
+ const rel = path3.relative(path3.resolve(opts.projectRoot), rootDir);
846
+ if (rel.startsWith("..") || path3.isAbsolute(rel)) {
847
+ throw new Error("Vector memory directory must stay inside the project root.");
848
+ }
849
+ fs3.mkdirSync(rootDir, { recursive: true });
850
+ this.rootDir = rootDir;
851
+ this.dbPath = path3.join(rootDir, filename);
852
+ const Database = loadRuntimeDatabaseSync();
853
+ this.db = new Database(this.dbPath);
854
+ initVectorSchema(this.db);
855
+ this.recordActiveProvider();
856
+ }
857
+ /**
858
+ * Absolute path of the store's data directory (the resolved
859
+ * `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
860
+ * to place sidecar state (e.g. the first-boot SAGE sync marker) next to
861
+ * the db instead of re-deriving the path and drifting from it.
862
+ */
863
+ get directory() {
864
+ return this.rootDir;
865
+ }
866
+ /**
867
+ * Absolute path of the SQLite database file. Hosts use this to take a
868
+ * file-level lock that covers all mutating operations (see
869
+ * `withFileLock(this.dbPath + '.lock', …)`).
870
+ */
871
+ get databasePath() {
872
+ return this.dbPath;
873
+ }
874
+ /** The lockfile path used to serialize mutating operations. */
875
+ get lockPath() {
876
+ return `${this.dbPath}.lock`;
877
+ }
878
+ get activeProviderId() {
879
+ const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
880
+ return row?.value ?? this.provider.id;
881
+ }
882
+ recordActiveProvider() {
883
+ this.db.exec("BEGIN");
884
+ try {
885
+ this.db.prepare(
886
+ `INSERT INTO schema_meta (key, value) VALUES (?, ?)
887
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value`
888
+ ).run(VECTOR_PROVIDER_KEY, this.provider.id);
889
+ this.db.prepare(
890
+ `INSERT INTO schema_meta (key, value) VALUES (?, ?)
891
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value`
892
+ ).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
893
+ this.db.exec("COMMIT");
894
+ } catch (e) {
895
+ this.db.exec("ROLLBACK");
896
+ throw e;
897
+ }
898
+ }
899
+ static contentHash(text) {
900
+ return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
901
+ }
902
+ /**
903
+ * Look up a vector for `text` in the provider-level embedding cache.
904
+ * Cache hit returns the cached vector (no ONNX pass). Miss returns
905
+ * `undefined`.
906
+ */
907
+ cachedVector(text, now) {
908
+ return lookupEmbeddingCache(
909
+ this.db,
910
+ _VectorMemoryStore.contentHash(text),
911
+ this.provider.id,
912
+ this.provider.dimensions,
913
+ now
914
+ );
915
+ }
916
+ /** Persist `vec` for `text` to the embedding cache. */
917
+ cacheVector(text, vec, now) {
918
+ upsertEmbeddingCache(this.db, {
919
+ contentHash: _VectorMemoryStore.contentHash(text),
920
+ providerId: this.provider.id,
921
+ dimensions: vec.length,
922
+ vector: encodeVector(vec),
923
+ text,
924
+ now
925
+ });
926
+ }
927
+ /**
928
+ * Embed `text`, hitting the provider-level cache first. Cache miss falls
929
+ * through to the configured provider and writes the result back. Returns
930
+ * `undefined` when the provider fails — the caller can persist the entry
931
+ * without a vector (fail-open).
932
+ */
933
+ async embedWithCache(text) {
934
+ const now = (/* @__PURE__ */ new Date()).toISOString();
935
+ const cached = this.cachedVector(text, now);
936
+ if (cached) return cached;
1217
937
  try {
1218
- vectorHits = await options.store.search(query, {
1219
- ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
1220
- limit: Math.max(limit * 2, 50)
1221
- });
938
+ const result = await this.provider.embed([text]);
939
+ const vec = result[0];
940
+ if (vec) this.cacheVector(text, vec, now);
941
+ return vec;
1222
942
  } catch {
1223
- vectorHits = [];
943
+ return void 0;
1224
944
  }
1225
945
  }
1226
- const sageById = /* @__PURE__ */ new Map();
1227
- for (const memory of lexical) sageById.set(memory.id, memory);
1228
- const lexicalRanked = lexical.map((memory, index) => ({
1229
- memory,
1230
- rankScore: lexicalRankScore(index, lexical.length),
1231
- lexicalScore: lexicalRankScore(index, lexical.length)
1232
- }));
1233
- const vectorRanked = [];
1234
- for (let i = 0; i < vectorHits.length; i++) {
1235
- const hit = vectorHits[i];
1236
- const sageId = hit.entry.metadata?.["sageId"];
1237
- if (typeof sageId !== "string") continue;
1238
- const memory = sageById.get(sageId);
1239
- if (!memory) continue;
1240
- vectorRanked.push({ memory, vectorScore: hit.score });
946
+ /**
947
+ * Look up an existing entry by `content_hash`. Returns `undefined` when
948
+ * the entry is not present. Used by `remember()` to make writes idempotent
949
+ * and by `syncFromSage()` to skip already-indexed SAGE memories.
950
+ */
951
+ findByContentHash(contentHash) {
952
+ this.assertOpen();
953
+ const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
954
+ if (!row) return void 0;
955
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
956
+ return this.rowToEntry(row, vectorRow);
1241
957
  }
1242
- const fused = /* @__PURE__ */ new Map();
1243
- for (let i = 0; i < lexicalRanked.length; i++) {
1244
- const c = lexicalRanked[i];
1245
- const rrf = (1 - weight) * (1 / (k + i + 1));
1246
- fused.set(c.memory.id, {
1247
- memory: c.memory,
1248
- vectorScore: null,
1249
- lexicalScore: c.lexicalScore,
1250
- finalScore: rrf,
1251
- source: "lexical"
958
+ /**
959
+ * Look up the entry mirroring a given SAGE memory id (i.e. the entry
960
+ * whose `metadata.sageId` equals `sageId`). Returns `undefined` when
961
+ * no such entry exists. Used by the event-driven mirror to delete
962
+ * vector entries on SAGE delete events (the emitter knows the SAGE
963
+ * id, not the vector entry id).
964
+ *
965
+ * Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
966
+ * column is the JSON blob `syncFromSage` writes, so this avoids a
967
+ * full table scan.
968
+ */
969
+ findBySageId(sageId) {
970
+ this.assertOpen();
971
+ const row = this.db.prepare(
972
+ `SELECT * FROM entries
973
+ WHERE json_extract(metadata, '$.sageId') = ?
974
+ ORDER BY updated_at DESC
975
+ LIMIT 1`
976
+ ).get(sageId);
977
+ if (!row) return void 0;
978
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
979
+ return this.rowToEntry(row, vectorRow);
980
+ }
981
+ /**
982
+ * Persist a new entry. Idempotent: if an entry with the same
983
+ * `content_hash` already exists, that entry is returned unchanged
984
+ * instead of inserting a duplicate. Mutating ops are wrapped in
985
+ * `withFileLock` so two processes cannot race the dedup check.
986
+ */
987
+ async remember(input) {
988
+ this.assertOpen();
989
+ if (!input.text || input.text.trim().length === 0) {
990
+ throw new Error("VectorMemoryStore.remember: text must be non-empty");
991
+ }
992
+ return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
993
+ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
1252
994
  });
1253
995
  }
1254
- for (let i = 0; i < vectorRanked.length; i++) {
1255
- const v = vectorRanked[i];
1256
- const rrf = weight * (1 / (k + i + 1));
1257
- const existing = fused.get(v.memory.id);
1258
- if (existing) {
1259
- existing.finalScore += rrf;
1260
- existing.vectorScore = v.vectorScore;
1261
- existing.source = "both";
1262
- } else if (v.vectorScore >= vectorOnlyThreshold) {
1263
- fused.set(v.memory.id, {
1264
- memory: v.memory,
1265
- vectorScore: v.vectorScore,
1266
- lexicalScore: null,
1267
- finalScore: rrf,
1268
- source: "vector"
1269
- });
996
+ async rememberUnlocked(input) {
997
+ const now = (/* @__PURE__ */ new Date()).toISOString();
998
+ const contentHash = _VectorMemoryStore.contentHash(input.text);
999
+ const existing = this.findByContentHash(contentHash);
1000
+ if (existing) return existing;
1001
+ const metadata = input.metadata ?? {};
1002
+ const tags = input.tags ?? [];
1003
+ const scope = input.scope ?? "project";
1004
+ const kind = input.kind ?? "note";
1005
+ const id = randomUUID();
1006
+ const vector = await this.embedWithCache(input.text);
1007
+ const providerId = vector ? this.provider.id : void 0;
1008
+ this.db.exec("BEGIN");
1009
+ try {
1010
+ this.db.prepare(
1011
+ `INSERT INTO entries
1012
+ (id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
1013
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
1014
+ ).run(
1015
+ id,
1016
+ input.text,
1017
+ input.summary ?? null,
1018
+ JSON.stringify(metadata),
1019
+ JSON.stringify(tags),
1020
+ scope,
1021
+ kind,
1022
+ contentHash,
1023
+ now,
1024
+ now
1025
+ );
1026
+ if (vector && providerId) {
1027
+ this.db.prepare(
1028
+ `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
1029
+ VALUES (?, ?, ?, ?, ?)
1030
+ ON CONFLICT(entry_id, provider_id) DO UPDATE SET
1031
+ vector = excluded.vector,
1032
+ dimensions = excluded.dimensions,
1033
+ created_at = excluded.created_at`
1034
+ ).run(id, providerId, vector.length, encodeVector(vector), now);
1035
+ }
1036
+ this.db.exec("COMMIT");
1037
+ } catch (e) {
1038
+ this.db.exec("ROLLBACK");
1039
+ throw e;
1040
+ }
1041
+ const result = {
1042
+ id,
1043
+ text: input.text,
1044
+ summary: input.summary ?? void 0,
1045
+ metadata,
1046
+ tags,
1047
+ scope,
1048
+ kind,
1049
+ contentHash,
1050
+ createdAt: now,
1051
+ updatedAt: now,
1052
+ providerId: providerId ?? "",
1053
+ dimensions: vector?.length ?? 0
1054
+ };
1055
+ if (vector) result.vector = vector;
1056
+ return result;
1057
+ }
1058
+ get(id) {
1059
+ this.assertOpen();
1060
+ const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
1061
+ if (!row) return void 0;
1062
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
1063
+ return this.rowToEntry(row, vectorRow);
1064
+ }
1065
+ /** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
1066
+ async forget(id) {
1067
+ this.assertOpen();
1068
+ return withFileLock(
1069
+ this.lockPath,
1070
+ async () => {
1071
+ this.db.exec("BEGIN");
1072
+ try {
1073
+ const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
1074
+ this.db.exec("COMMIT");
1075
+ return info.changes > 0;
1076
+ } catch (e) {
1077
+ this.db.exec("ROLLBACK");
1078
+ throw e;
1079
+ }
1080
+ },
1081
+ { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
1082
+ );
1083
+ }
1084
+ async search(query, opts = {}) {
1085
+ this.assertOpen();
1086
+ const limit = normalizeLimit(opts.limit);
1087
+ const threshold = opts.threshold ?? 0;
1088
+ const includeVectors = opts.includeVectors === true;
1089
+ if (typeof query !== "string" || query.trim().length === 0) return [];
1090
+ const queryVec = await this.embedWithCache(query);
1091
+ if (!queryVec || queryVec.length === 0) return [];
1092
+ const providerId = this.provider.id;
1093
+ const dimensions = this.provider.dimensions;
1094
+ const filters = ["v.provider_id = ?", "v.dimensions = ?"];
1095
+ const params = [providerId, dimensions];
1096
+ if (opts.scope !== void 0) {
1097
+ filters.push("e.scope = ?");
1098
+ params.push(opts.scope);
1099
+ }
1100
+ if (opts.kind !== void 0) {
1101
+ filters.push("e.kind = ?");
1102
+ params.push(opts.kind);
1103
+ }
1104
+ const scanRows = this.db.prepare(
1105
+ `SELECT e.id AS id, v.vector AS vec_blob
1106
+ FROM entries e
1107
+ JOIN vectors v ON v.entry_id = e.id
1108
+ WHERE ${filters.join(" AND ")}`
1109
+ ).all(...params);
1110
+ const top = [];
1111
+ for (const row of scanRows) {
1112
+ const vec = decodeVector(row.vec_blob);
1113
+ const raw = cosineSimilarity(queryVec, vec);
1114
+ const score = Math.max(0, Math.min(1, raw));
1115
+ if (!Number.isFinite(score) || score < threshold) continue;
1116
+ if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
1117
+ let at = top.length;
1118
+ while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
1119
+ top.splice(at, 0, { id: row.id, score, vector: vec });
1120
+ if (top.length > limit) top.length = limit;
1121
+ }
1122
+ if (top.length === 0) return [];
1123
+ const placeholders = top.map(() => "?").join(",");
1124
+ const hydrated = this.db.prepare(
1125
+ `SELECT id, text, summary, metadata, tags, scope, kind,
1126
+ content_hash, created_at, updated_at
1127
+ FROM entries WHERE id IN (${placeholders})`
1128
+ ).all(...top.map((t) => t.id));
1129
+ const entryById = /* @__PURE__ */ new Map();
1130
+ for (const row of hydrated) {
1131
+ entryById.set(row.id, this.rowToEntry(row));
1132
+ }
1133
+ const scored = [];
1134
+ for (const candidate of top) {
1135
+ const entry = entryById.get(candidate.id);
1136
+ if (!entry) continue;
1137
+ const hit = { entry, score: candidate.score, providerId };
1138
+ if (includeVectors) hit.vector = candidate.vector;
1139
+ scored.push(hit);
1140
+ }
1141
+ return scored;
1142
+ }
1143
+ /**
1144
+ * Page through entries, newest first.
1145
+ *
1146
+ * Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
1147
+ * without the id tiebreak two entries written in the same millisecond can
1148
+ * swap places between calls and a paging caller silently skips one.
1149
+ *
1150
+ * Pagination is keyset (`after`), not offset, because the only caller that
1151
+ * pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
1152
+ * `OFFSET` every deletion shifts the remaining rows left and the next page
1153
+ * skips exactly as many entries as were removed. Keyset is immune: it
1154
+ * resumes from a position, and the rows a deletion removes are ones the
1155
+ * sweep has already passed.
1156
+ */
1157
+ list(opts = {}) {
1158
+ this.assertOpen();
1159
+ const where = [];
1160
+ const params = [];
1161
+ if (opts.scope !== void 0) {
1162
+ where.push("scope = ?");
1163
+ params.push(opts.scope);
1270
1164
  }
1271
- }
1272
- const out = Array.from(fused.values());
1273
- out.sort((a, b) => b.finalScore - a.finalScore);
1274
- return out.slice(0, limit);
1275
- }
1276
- function asVectorRecallProvider(store) {
1277
- return {
1278
- async search(query, opts) {
1279
- const hits = await store.search(query, {
1280
- limit: opts.limit,
1281
- ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
1282
- });
1283
- return hits.map((h) => ({
1284
- id: h.entry.id,
1285
- score: h.score,
1286
- text: h.entry.text,
1287
- ...h.entry.summary ? { summary: h.entry.summary } : {},
1288
- tags: h.entry.tags,
1289
- ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
1290
- }));
1165
+ if (opts.kind !== void 0) {
1166
+ where.push("kind = ?");
1167
+ params.push(opts.kind);
1291
1168
  }
1292
- };
1293
- }
1294
- function clamp01(value) {
1295
- if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
1296
- if (value < 0) return 0;
1297
- if (value > 1) return 1;
1298
- return value;
1299
- }
1300
-
1301
- // src/sage-port-wrapper.ts
1302
- import {
1303
- augmentLexicalWithVectorRecall,
1304
- isSageVisibleForSearch,
1305
- SAGE_RETRIEVAL_CAPABILITY,
1306
- SAGE_SURFACE_CAPABILITY
1307
- } from "@wrongstack/sage";
1308
- function asVectorRecallProviderAdapter(store) {
1309
- return {
1310
- async search(query, opts) {
1311
- const hits = await store.search(query, {
1312
- limit: opts.limit,
1313
- ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
1314
- });
1315
- return hits.map((h) => ({
1316
- id: h.entry.id,
1317
- score: h.score,
1318
- text: h.entry.text,
1319
- ...h.entry.summary ? { summary: h.entry.summary } : {},
1320
- tags: h.entry.tags,
1321
- ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
1322
- }));
1169
+ if (opts.after) {
1170
+ where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
1171
+ params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
1323
1172
  }
1324
- };
1325
- }
1326
- function wrapMemoryPortWithVectorRecall(port, options) {
1327
- const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
1328
- const materializeFor = (searchOpts) => async (sageId) => {
1329
- const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
1330
- if (!surface?.getSage) return void 0;
1331
- const memory = await surface.getSage(sageId);
1332
- if (!memory) return void 0;
1333
- return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
1334
- };
1335
- const fusionOptions = (searchOpts) => ({
1336
- vectorRecall: recall,
1337
- materializeVectorOnly: materializeFor(searchOpts),
1338
- ...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
1339
- ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
1340
- ...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
1341
- ...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
1342
- ...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
1343
- });
1344
- const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
1345
- const wrapSearchSage = (original) => async (query, searchOpts) => {
1346
- const opts = searchOpts;
1347
- const lexical = await original(query, searchOpts);
1348
- if (callerOwnsFusion(opts)) return lexical;
1349
- const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
1350
- return fused.map((hit) => hit.memory);
1351
- };
1352
- const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
1353
- const opts = searchOpts;
1354
- const lexicalHits = await original(query, searchOpts);
1355
- if (callerOwnsFusion(opts)) return lexicalHits;
1356
- return augmentLexicalWithVectorRecall(
1357
- query,
1358
- lexicalHits.map((hit) => hit.memory),
1359
- fusionOptions(opts)
1173
+ const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
1174
+ ORDER BY updated_at DESC, id DESC LIMIT ?`;
1175
+ params.push(opts.limit ?? 100);
1176
+ const rows = this.db.prepare(sql).all(...params);
1177
+ return rows.map((r) => this.rowToEntry(r));
1178
+ }
1179
+ async reindexAll() {
1180
+ this.assertOpen();
1181
+ return withFileLock(
1182
+ this.lockPath,
1183
+ async () => {
1184
+ const rows = this.db.prepare("SELECT id, text FROM entries").all();
1185
+ let processed = 0;
1186
+ let errors = 0;
1187
+ for (const row of rows) {
1188
+ try {
1189
+ const result = await this.provider.embed([row.text]);
1190
+ const v = result[0];
1191
+ if (!v) {
1192
+ errors++;
1193
+ continue;
1194
+ }
1195
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1196
+ this.db.prepare(
1197
+ `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
1198
+ VALUES (?, ?, ?, ?, ?)
1199
+ ON CONFLICT(entry_id, provider_id) DO UPDATE SET
1200
+ vector = excluded.vector,
1201
+ dimensions = excluded.dimensions,
1202
+ created_at = excluded.created_at`
1203
+ ).run(row.id, this.provider.id, v.length, encodeVector(v), now);
1204
+ this.cacheVector(row.text, v, now);
1205
+ processed++;
1206
+ } catch {
1207
+ errors++;
1208
+ }
1209
+ }
1210
+ return { processed, errors };
1211
+ },
1212
+ { timeoutMs: 6e4 }
1360
1213
  );
1361
- };
1362
- const wrapped = Object.create(
1363
- Object.getPrototypeOf(port),
1364
- Object.getOwnPropertyDescriptors(port)
1365
- );
1366
- wrapped.getCapability = (capability) => {
1367
- if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
1368
- const original = port.getCapability(capability);
1369
- if (!original) return void 0;
1370
- return {
1371
- ...original,
1372
- searchSage: wrapSearchSage(original.searchSage),
1373
- ...original.searchSageWithBreakdown ? {
1374
- searchSageWithBreakdown: wrapSearchWithBreakdown(
1375
- original.searchSageWithBreakdown
1376
- )
1377
- } : {}
1378
- };
1379
- }
1380
- if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
1381
- const original = port.getCapability(capability);
1382
- if (!original) return void 0;
1383
- return {
1384
- ...original,
1385
- searchSage: wrapSearchSage(original.searchSage),
1386
- ...original.searchSageWithBreakdown ? {
1387
- searchSageWithBreakdown: wrapSearchWithBreakdown(
1388
- original.searchSageWithBreakdown
1389
- )
1390
- } : {}
1391
- };
1214
+ }
1215
+ stats() {
1216
+ this.assertOpen();
1217
+ const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
1218
+ const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
1219
+ const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
1220
+ return {
1221
+ entries: entryCount,
1222
+ vectors: vectorCount,
1223
+ providers: providerRows.map((r) => r.provider_id),
1224
+ modelAvailable: true,
1225
+ modelId: this.provider.id,
1226
+ dimensions: this.provider.dimensions
1227
+ };
1228
+ }
1229
+ /**
1230
+ * Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
1231
+ * Useful for the WebUI's vector-memory panel and for diagnosing the
1232
+ * "why is search slow?" question.
1233
+ */
1234
+ cacheStats() {
1235
+ this.assertOpen();
1236
+ const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
1237
+ const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
1238
+ const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
1239
+ const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
1240
+ return {
1241
+ entries,
1242
+ providers,
1243
+ totalUseCount,
1244
+ oldestLastUsedAt: oldest?.t ?? null
1245
+ };
1246
+ }
1247
+ /**
1248
+ * LRU-evict the embedding cache down to `keepMostRecent` rows. The
1249
+ * `embedding_cache` table is independent of `entries`, so a sweep here
1250
+ * only removes cached vectors — never stored entries. Called by hosts
1251
+ * that want to bound cache growth on long-lived processes.
1252
+ */
1253
+ async evictCache(keepMostRecent) {
1254
+ this.assertOpen();
1255
+ if (keepMostRecent < 0) {
1256
+ throw new Error("evictCache: keepMostRecent must be >= 0");
1392
1257
  }
1393
- return port.getCapability(capability);
1394
- };
1395
- return wrapped;
1396
- }
1397
-
1398
- // src/sage-event-mirror.ts
1399
- import * as fs3 from "node:fs";
1400
- import * as path3 from "node:path";
1401
- import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
1402
- function subscribeVectorMemoryToSage(opts) {
1403
- const { store, memoryStore } = opts;
1404
- const log = opts.logger;
1405
- if (opts.enabled === false) {
1406
- return { dispose: () => void 0 };
1258
+ return withFileLock(
1259
+ this.lockPath,
1260
+ async () => {
1261
+ const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
1262
+ if (total <= keepMostRecent) return { removed: 0 };
1263
+ const toRemove = total - keepMostRecent;
1264
+ const stmt = this.db.prepare(
1265
+ `DELETE FROM embedding_cache
1266
+ WHERE content_hash IN (
1267
+ SELECT content_hash FROM embedding_cache
1268
+ ORDER BY last_used_at ASC
1269
+ LIMIT ?
1270
+ )`
1271
+ );
1272
+ const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
1273
+ return { removed: Number(info.changes) };
1274
+ },
1275
+ { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
1276
+ );
1277
+ }
1278
+ close() {
1279
+ if (this.closed) return;
1280
+ this.closed = true;
1281
+ this.db.close();
1407
1282
  }
1408
- const surface = getSageSurface2(memoryStore);
1409
- if (!surface) {
1410
- log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
1411
- return { dispose: () => void 0 };
1283
+ assertOpen() {
1284
+ if (this.closed) throw new Error("VectorMemoryStore is closed");
1412
1285
  }
1413
- const events = memoryStore.events;
1414
- if (!events) {
1415
- log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
1416
- return { dispose: () => void 0 };
1286
+ rowToEntry(row, vectorRow) {
1287
+ const summaryValue = row.summary;
1288
+ const entry = {
1289
+ id: row.id,
1290
+ text: row.text,
1291
+ summary: summaryValue ?? void 0,
1292
+ metadata: safeParseJson(row.metadata, {}),
1293
+ tags: safeParseJson(row.tags, []),
1294
+ scope: row.scope,
1295
+ kind: row.kind,
1296
+ contentHash: row.content_hash,
1297
+ createdAt: row.created_at,
1298
+ updatedAt: row.updated_at,
1299
+ providerId: vectorRow?.provider_id ?? "",
1300
+ dimensions: vectorRow?.dimensions ?? 0
1301
+ };
1302
+ if (vectorRow?.vector) {
1303
+ entry.vector = decodeVector(vectorRow.vector);
1304
+ }
1305
+ return entry;
1417
1306
  }
1418
- const fetch = async (memoryId) => {
1419
- try {
1420
- return await surface.getSage(memoryId);
1421
- } catch (err) {
1422
- log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg2(err)}`);
1423
- return null;
1307
+ async syncFromSage(sage) {
1308
+ this.assertOpen();
1309
+ const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
1310
+ let indexed = 0;
1311
+ let skipped = 0;
1312
+ let failed = 0;
1313
+ const errors = [];
1314
+ for (const memory of memories) {
1315
+ try {
1316
+ const hash = _VectorMemoryStore.contentHash(memory.text);
1317
+ const existing = this.findByContentHash(hash);
1318
+ if (existing) {
1319
+ skipped++;
1320
+ continue;
1321
+ }
1322
+ await this.rememberUnlocked({
1323
+ text: memory.text,
1324
+ summary: memory.summary ?? void 0,
1325
+ metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
1326
+ tags: memory.tags ?? [],
1327
+ scope: "project",
1328
+ kind: "note"
1329
+ });
1330
+ indexed++;
1331
+ } catch (err) {
1332
+ failed++;
1333
+ errors.push({ memoryId: memory.id, message: errMsg2(err) });
1334
+ }
1424
1335
  }
1425
- };
1426
- const mirror = async (memoryId) => {
1427
- const memory = await fetch(memoryId);
1428
- if (!memory) return;
1429
- if (memory.scope === "session") return;
1430
- try {
1431
- const existing = store.findBySageId(memoryId);
1432
- if (existing) await store.forget(existing.id);
1433
- await store.remember({
1434
- text: memory.text,
1435
- ...memory.summary ? { summary: memory.summary } : {},
1436
- tags: memory.tags ?? [],
1437
- scope: "project",
1438
- kind: "note",
1336
+ return { scanned: memories.length, indexed, skipped, failed, errors };
1337
+ }
1338
+ };
1339
+ function fallbackHashingProvider(dimensions) {
1340
+ return new HashingEmbeddingProvider({ dimensions });
1341
+ }
1342
+ function safeParseJson(value, fallback) {
1343
+ if (typeof value !== "string") return fallback;
1344
+ try {
1345
+ return JSON.parse(value);
1346
+ } catch {
1347
+ return fallback;
1348
+ }
1349
+ }
1350
+ function errMsg2(err) {
1351
+ return err instanceof Error ? err.message : String(err);
1352
+ }
1353
+
1354
+ // src/tools.ts
1355
+ function createVectorMemoryTools(store) {
1356
+ return [
1357
+ vectorMemoryRememberTool(store),
1358
+ vectorMemorySearchTool(store),
1359
+ vectorMemoryStatsTool(store),
1360
+ vectorMemoryForgetTool(store)
1361
+ ];
1362
+ }
1363
+ function vectorMemoryRememberTool(store) {
1364
+ return {
1365
+ name: "vector_memory_remember",
1366
+ category: "Memory",
1367
+ description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
1368
+ usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
1369
+ permission: "confirm",
1370
+ mutating: true,
1371
+ riskTier: "standard",
1372
+ timeoutMs: 5e3,
1373
+ capabilities: ["memory.write"],
1374
+ icon: "settings",
1375
+ inputSchema: {
1376
+ type: "object",
1377
+ properties: {
1378
+ text: { type: "string", minLength: 1, description: "The text to embed and store." },
1379
+ summary: { type: "string", description: "Optional short label." },
1380
+ tags: {
1381
+ type: "array",
1382
+ items: { type: "string" },
1383
+ description: "Optional tags for later filtering."
1384
+ },
1385
+ scope: {
1386
+ type: "string",
1387
+ enum: ["project", "user", "session"],
1388
+ description: "Visibility scope. Defaults to `project`."
1389
+ },
1390
+ kind: {
1391
+ type: "string",
1392
+ enum: ["note", "fact", "summary", "snippet", "link"],
1393
+ description: "Entry kind. Defaults to `note`."
1394
+ },
1439
1395
  metadata: {
1440
- source: "sage",
1441
- sageId: memory.id,
1442
- sageKind: memory.kind,
1443
- sageScope: memory.scope,
1444
- importance: memory.importance,
1445
- confidence: memory.confidence
1396
+ type: "object",
1397
+ additionalProperties: true,
1398
+ description: "Free-form metadata persisted as JSON."
1446
1399
  }
1447
- });
1448
- } catch (err) {
1449
- log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg2(err)}`);
1400
+ },
1401
+ required: ["text"],
1402
+ additionalProperties: false
1403
+ },
1404
+ execute: async (input) => {
1405
+ const entry = await store.remember(input);
1406
+ return { id: entry.id, hasVector: entry.vector !== void 0 };
1450
1407
  }
1451
1408
  };
1452
- const forgetMirror = async (memoryId) => {
1453
- try {
1454
- const existing = store.findBySageId(memoryId);
1455
- if (existing) await store.forget(existing.id);
1456
- } catch (err) {
1457
- log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg2(err)}`);
1409
+ }
1410
+ function vectorMemorySearchTool(store) {
1411
+ return {
1412
+ name: "vector_memory_search",
1413
+ category: "Memory",
1414
+ description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
1415
+ usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
1416
+ permission: "auto",
1417
+ mutating: false,
1418
+ riskTier: "safe",
1419
+ timeoutMs: 5e3,
1420
+ capabilities: ["memory.read"],
1421
+ icon: "search",
1422
+ inputSchema: {
1423
+ type: "object",
1424
+ properties: {
1425
+ query: { type: "string", minLength: 1, description: "The natural-language query." },
1426
+ limit: {
1427
+ type: "number",
1428
+ minimum: 1,
1429
+ maximum: 100,
1430
+ description: "Max results (default 10)."
1431
+ },
1432
+ threshold: {
1433
+ type: "number",
1434
+ minimum: 0,
1435
+ maximum: 1,
1436
+ description: "Minimum cosine similarity. Results below the floor are dropped."
1437
+ },
1438
+ scope: {
1439
+ type: "string",
1440
+ enum: ["project", "user", "session"],
1441
+ description: "Restrict to a scope."
1442
+ },
1443
+ kind: {
1444
+ type: "string",
1445
+ enum: ["note", "fact", "summary", "snippet", "link"],
1446
+ description: "Restrict to a kind."
1447
+ }
1448
+ },
1449
+ required: ["query"],
1450
+ additionalProperties: false
1451
+ },
1452
+ execute: async (input) => {
1453
+ const hits = await store.search(input.query, {
1454
+ limit: input.limit !== void 0 ? input.limit : void 0,
1455
+ threshold: input.threshold !== void 0 ? input.threshold : void 0,
1456
+ scope: input.scope,
1457
+ kind: input.kind
1458
+ });
1459
+ return {
1460
+ hits: hits.map((h) => ({
1461
+ id: h.entry.id,
1462
+ score: h.score,
1463
+ text: h.entry.text,
1464
+ summary: h.entry.summary ?? void 0,
1465
+ tags: h.entry.tags
1466
+ }))
1467
+ };
1458
1468
  }
1459
1469
  };
1460
- const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
1461
- const memoryId = payload?.memoryId;
1462
- if (typeof memoryId !== "string") return;
1463
- void mirror(memoryId);
1464
- });
1465
- const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
1466
- const memoryId = payload?.memoryId;
1467
- if (typeof memoryId !== "string") return;
1468
- void mirror(memoryId);
1469
- });
1470
- const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
1471
- const memoryId = payload?.memoryId;
1472
- if (typeof memoryId !== "string") return;
1473
- const status = payload?.status;
1474
- if (status === "deleted") return;
1475
- void mirror(memoryId);
1476
- });
1477
- const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
1478
- const memoryId = payload?.memoryId;
1479
- if (typeof memoryId !== "string") return;
1480
- void forgetMirror(memoryId);
1481
- });
1470
+ }
1471
+ function vectorMemoryStatsTool(store) {
1472
+ return {
1473
+ name: "vector_memory_stats",
1474
+ category: "Memory",
1475
+ description: "Return counts, providers, and dimensions for the local vector memory store.",
1476
+ usageHint: "Cheap diagnostic \u2014 safe to call any time.",
1477
+ permission: "auto",
1478
+ mutating: false,
1479
+ riskTier: "safe",
1480
+ timeoutMs: 1e3,
1481
+ capabilities: ["memory.read"],
1482
+ icon: "search",
1483
+ inputSchema: {
1484
+ type: "object",
1485
+ properties: {},
1486
+ additionalProperties: false
1487
+ },
1488
+ execute: async () => store.stats()
1489
+ };
1490
+ }
1491
+ function vectorMemoryForgetTool(store) {
1482
1492
  return {
1483
- dispose: () => {
1484
- offAccepted();
1485
- offRecovered();
1486
- offUpdated();
1487
- offDeleted();
1488
- }
1493
+ name: "vector_memory_forget",
1494
+ category: "Memory",
1495
+ description: "Remove an entry (and its vector) from the vector memory store.",
1496
+ usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
1497
+ permission: "confirm",
1498
+ mutating: true,
1499
+ riskTier: "standard",
1500
+ timeoutMs: 1e3,
1501
+ capabilities: ["memory.write"],
1502
+ icon: "settings",
1503
+ inputSchema: {
1504
+ type: "object",
1505
+ properties: {
1506
+ id: {
1507
+ type: "string",
1508
+ minLength: 1,
1509
+ description: "Entry id returned by `vector_memory_remember`."
1510
+ }
1511
+ },
1512
+ required: ["id"],
1513
+ additionalProperties: false
1514
+ },
1515
+ execute: async (input) => ({ removed: await store.forget(input.id) })
1489
1516
  };
1490
1517
  }
1491
- async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
1492
- const surface = getSageSurface2(memoryStore);
1493
- if (!surface) return { scanned: 0, removed: 0 };
1494
- let scanned = 0;
1495
- let removed = 0;
1496
- const PAGE = Math.max(1, options?.pageSize ?? 500);
1497
- let after;
1498
- for (; ; ) {
1499
- const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
1500
- if (page.length === 0) break;
1501
- const last = page[page.length - 1];
1502
- after = { updatedAt: last.updatedAt, id: last.id };
1503
- for (const entry of page) {
1504
- scanned++;
1505
- const sageId = entry.metadata?.sageId;
1506
- if (typeof sageId !== "string") continue;
1507
- try {
1508
- const memory = await surface.getSage(sageId);
1509
- if (memory !== null && memory.status !== "deleted") continue;
1510
- await store.forget(entry.id);
1511
- removed++;
1512
- } catch (err) {
1513
- logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg2(err)}`);
1514
- }
1515
- }
1516
- if (page.length < PAGE) break;
1518
+
1519
+ // src/transformers-provider.ts
1520
+ var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
1521
+ var DEFAULT_VECTOR_DIMENSIONS = 384;
1522
+ var DEFAULT_VECTOR_DTYPE = "q8";
1523
+ var TransformersEmbeddingProvider = class {
1524
+ id;
1525
+ dimensions;
1526
+ modelId;
1527
+ cacheDir;
1528
+ dtype;
1529
+ device;
1530
+ batchSize;
1531
+ maxChars;
1532
+ allowRemote;
1533
+ extractor;
1534
+ loadPromise;
1535
+ constructor(opts = {}) {
1536
+ this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
1537
+ this.cacheDir = opts.cacheDir;
1538
+ this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
1539
+ this.device = opts.device ?? "cpu";
1540
+ this.batchSize = opts.batchSize ?? 16;
1541
+ this.maxChars = opts.maxChars ?? 2048;
1542
+ this.allowRemote = opts.allowRemoteModels ?? true;
1543
+ this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
1544
+ this.id = `transformers-js:${this.modelId}:${this.dtype}`;
1517
1545
  }
1518
- return { scanned, removed };
1519
- }
1520
- function errMsg2(err) {
1521
- return err instanceof Error ? err.message : String(err);
1522
- }
1523
- var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
1524
- var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
1525
- async function sweepStaleSageMirrors(opts) {
1526
- const markerPath2 = path3.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
1527
- const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
1528
- if (!opts.force) {
1546
+ /**
1547
+ * Synchronous capability check. Returns false when the optional
1548
+ * `@huggingface/transformers` dependency is not installed.
1549
+ *
1550
+ * NOTE: this probes via dynamic import and caches the result, but does
1551
+ * NOT load the model itself — model loading is deferred to `embed()`.
1552
+ */
1553
+ async isAvailable() {
1529
1554
  try {
1530
- const raw = JSON.parse(fs3.readFileSync(markerPath2, "utf8"));
1531
- const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
1532
- if (Number.isFinite(at) && Date.now() - at < interval) {
1533
- return { swept: false, reason: "throttled" };
1534
- }
1555
+ await this.loadModule();
1556
+ return true;
1535
1557
  } catch {
1558
+ return false;
1536
1559
  }
1537
1560
  }
1538
- try {
1539
- fs3.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
1540
- } catch {
1561
+ async embed(texts) {
1562
+ if (texts.length === 0) return [];
1563
+ const extractor = await this.getExtractor();
1564
+ const prepared = texts.map((t) => this.prepare(t));
1565
+ const batches = [];
1566
+ for (let i = 0; i < prepared.length; i += this.batchSize) {
1567
+ batches.push(prepared.slice(i, i + this.batchSize));
1568
+ }
1569
+ const results = [];
1570
+ for (const batch of batches) {
1571
+ const out = await extractor(batch, { pooling: "mean", normalize: true });
1572
+ results.push(...this.tensorToVectors(out, batch.length));
1573
+ }
1574
+ return results;
1541
1575
  }
1542
- try {
1543
- const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
1544
- opts.logger?.debug?.(
1545
- `vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
1546
- );
1547
- return { swept: true, ...result };
1548
- } catch (err) {
1549
- opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg2(err)}`);
1550
- return { swept: false, reason: errMsg2(err) };
1576
+ /** Truncate + normalize text before embedding. */
1577
+ prepare(text) {
1578
+ if (!text) return "";
1579
+ const normalized = text.normalize("NFKC").trim();
1580
+ return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
1551
1581
  }
1552
- }
1553
-
1554
- // src/search-race.ts
1555
- function previewText(text, maxLen) {
1556
- if (text.length <= maxLen) return text;
1557
- return text.slice(0, maxLen - 1) + "\u2026";
1558
- }
1559
- async function runSearchRace(query, lexical, vectorStore, options = {}) {
1560
- const limit = options.limit ?? 20;
1561
- const threshold = options.threshold ?? 0;
1562
- let vectorHits = [];
1563
- try {
1564
- vectorHits = await vectorStore.search(query, {
1565
- limit,
1566
- ...threshold > 0 ? { threshold } : {}
1567
- });
1568
- } catch {
1569
- vectorHits = [];
1582
+ tensorToVectors(out, batchSize) {
1583
+ if (typeof out.tolist === "function") {
1584
+ const nested = out.tolist();
1585
+ if (Array.isArray(nested) && Array.isArray(nested[0])) {
1586
+ return nested.map((row) => Float32Array.from(row));
1587
+ }
1588
+ return [Float32Array.from(nested)];
1589
+ }
1590
+ const flat = out.data;
1591
+ if (flat instanceof Float32Array) {
1592
+ if (batchSize === 1) return [flat];
1593
+ const dim = flat.length / batchSize;
1594
+ const vectors = [];
1595
+ for (let i = 0; i < batchSize; i++) {
1596
+ vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
1597
+ }
1598
+ return vectors;
1599
+ }
1600
+ if (Array.isArray(flat) && Array.isArray(flat[0])) {
1601
+ return flat.map((row) => Float32Array.from(row));
1602
+ }
1603
+ if (Array.isArray(flat)) {
1604
+ return [Float32Array.from(flat)];
1605
+ }
1606
+ throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
1570
1607
  }
1571
- const lexicalOnly = [];
1572
- const vectorOnly = [];
1573
- const overlap = [];
1574
- const seenIds = /* @__PURE__ */ new Set();
1575
- const lexicalCapped = lexical.slice(0, limit);
1576
- for (let i = 0; i < lexicalCapped.length; i++) {
1577
- const mem = lexicalCapped[i];
1578
- const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
1579
- const id = mem.id;
1580
- seenIds.add(id);
1581
- overlap.push({
1582
- id,
1583
- lexicalScore: score,
1584
- vectorScore: null,
1585
- // patched below when a vector hit carries this id
1586
- preview: previewText(mem.text, 140)
1587
- });
1608
+ async getExtractor() {
1609
+ if (this.extractor) return this.extractor;
1610
+ if (!this.loadPromise) this.loadPromise = this.loadExtractor();
1611
+ this.extractor = await this.loadPromise;
1612
+ return this.extractor;
1588
1613
  }
1589
- for (const hit of vectorHits) {
1590
- const sageId = hit.entry.metadata?.["sageId"];
1591
- if (typeof sageId !== "string") continue;
1592
- const existing = overlap.find((row) => row.id === sageId);
1593
- if (existing) {
1594
- existing.vectorScore = hit.score;
1595
- continue;
1596
- }
1597
- if (seenIds.has(sageId)) continue;
1598
- seenIds.add(sageId);
1599
- vectorOnly.push({
1600
- id: sageId,
1601
- lexicalScore: null,
1602
- vectorScore: hit.score,
1603
- preview: previewText(hit.entry.text, 140)
1614
+ async loadExtractor() {
1615
+ const mod = await this.loadModule();
1616
+ if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
1617
+ mod.env.allowRemoteModels = this.allowRemote;
1618
+ if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
1619
+ const pipe = await mod.pipeline("feature-extraction", this.modelId, {
1620
+ dtype: this.dtype,
1621
+ device: this.device
1604
1622
  });
1623
+ return pipe;
1605
1624
  }
1606
- for (let i = overlap.length - 1; i >= 0; i--) {
1607
- const row = overlap[i];
1608
- if (row.vectorScore === null) {
1609
- lexicalOnly.push({
1610
- id: row.id,
1611
- lexicalScore: row.lexicalScore,
1612
- vectorScore: null,
1613
- preview: row.preview
1614
- });
1615
- overlap.splice(i, 1);
1625
+ async loadModule() {
1626
+ try {
1627
+ return await import("@huggingface/transformers");
1628
+ } catch (err) {
1629
+ throw new VectorMemoryProviderUnavailableError(
1630
+ "@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
1631
+ err
1632
+ );
1616
1633
  }
1617
1634
  }
1618
- const lexicalCount = lexicalOnly.length + overlap.length;
1619
- const vectorCount = vectorOnly.length + overlap.length;
1620
- const denom = Math.max(lexicalCount, vectorCount, 1);
1621
- return {
1622
- query,
1623
- lexicalOnly,
1624
- vectorOnly,
1625
- // The sweep above removed every row still carrying a null vectorScore,
1626
- // so every remaining row holds a real number here.
1627
- overlap,
1628
- metrics: {
1629
- lexicalCount,
1630
- vectorCount,
1631
- overlapCount: overlap.length,
1632
- lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
1633
- vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
1634
- agreementRatio: overlap.length / denom
1635
- }
1636
- };
1637
- }
1635
+ };
1638
1636
  export {
1639
1637
  DEFAULT_SWEEP_INTERVAL_MS,
1640
1638
  DEFAULT_VECTOR_DIMENSIONS,