@wrongstack/vector-memory 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -14,1632 +14,1625 @@ var VectorMemoryProviderUnavailableError = class extends VectorMemoryError {
14
14
  }
15
15
  };
16
16
 
17
- // src/transformers-provider.ts
18
- var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
19
- var DEFAULT_VECTOR_DIMENSIONS = 384;
20
- var DEFAULT_VECTOR_DTYPE = "q8";
21
- var TransformersEmbeddingProvider = class {
22
- id;
23
- dimensions;
24
- modelId;
25
- cacheDir;
26
- dtype;
27
- device;
28
- batchSize;
29
- maxChars;
30
- allowRemote;
31
- extractor;
32
- loadPromise;
33
- constructor(opts = {}) {
34
- this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
35
- this.cacheDir = opts.cacheDir;
36
- this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
37
- this.device = opts.device ?? "cpu";
38
- this.batchSize = opts.batchSize ?? 16;
39
- this.maxChars = opts.maxChars ?? 2048;
40
- this.allowRemote = opts.allowRemoteModels ?? true;
41
- this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
42
- this.id = `transformers-js:${this.modelId}:${this.dtype}`;
17
+ // src/sage-event-mirror.ts
18
+ import * as fs from "node:fs";
19
+ import * as path from "node:path";
20
+ import { getSageSurface } from "@wrongstack/sage";
21
+ function subscribeVectorMemoryToSage(opts) {
22
+ const { store, memoryStore } = opts;
23
+ const log = opts.logger;
24
+ if (opts.enabled === false) {
25
+ return { dispose: () => void 0 };
43
26
  }
44
- /**
45
- * Synchronous capability check. Returns false when the optional
46
- * `@huggingface/transformers` dependency is not installed.
47
- *
48
- * NOTE: this probes via dynamic import and caches the result, but does
49
- * NOT load the model itself — model loading is deferred to `embed()`.
50
- */
51
- async isAvailable() {
27
+ const surface = getSageSurface(memoryStore);
28
+ if (!surface) {
29
+ log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
30
+ return { dispose: () => void 0 };
31
+ }
32
+ const events = memoryStore.events;
33
+ if (!events) {
34
+ log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
35
+ return { dispose: () => void 0 };
36
+ }
37
+ const fetch = async (memoryId) => {
52
38
  try {
53
- await this.loadModule();
54
- return true;
55
- } catch {
56
- return false;
39
+ return await surface.getSage(memoryId);
40
+ } catch (err) {
41
+ log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg(err)}`);
42
+ return null;
57
43
  }
58
- }
59
- async embed(texts) {
60
- if (texts.length === 0) return [];
61
- const extractor = await this.getExtractor();
62
- const prepared = texts.map((t) => this.prepare(t));
63
- const batches = [];
64
- for (let i = 0; i < prepared.length; i += this.batchSize) {
65
- batches.push(prepared.slice(i, i + this.batchSize));
44
+ };
45
+ const mirror = async (memoryId) => {
46
+ const memory = await fetch(memoryId);
47
+ if (!memory) return;
48
+ if (memory.scope === "session") return;
49
+ try {
50
+ const existing = store.findBySageId(memoryId);
51
+ if (existing) await store.forget(existing.id);
52
+ await store.remember({
53
+ text: memory.text,
54
+ ...memory.summary ? { summary: memory.summary } : {},
55
+ tags: memory.tags ?? [],
56
+ scope: "project",
57
+ kind: "note",
58
+ metadata: {
59
+ source: "sage",
60
+ sageId: memory.id,
61
+ sageKind: memory.kind,
62
+ sageScope: memory.scope,
63
+ importance: memory.importance,
64
+ confidence: memory.confidence
65
+ }
66
+ });
67
+ } catch (err) {
68
+ log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg(err)}`);
66
69
  }
67
- const results = [];
68
- for (const batch of batches) {
69
- const out = await extractor(batch, { pooling: "mean", normalize: true });
70
- results.push(...this.tensorToVectors(out, batch.length));
70
+ };
71
+ const forgetMirror = async (memoryId) => {
72
+ try {
73
+ const existing = store.findBySageId(memoryId);
74
+ if (existing) await store.forget(existing.id);
75
+ } catch (err) {
76
+ log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg(err)}`);
71
77
  }
72
- return results;
73
- }
74
- /** Truncate + normalize text before embedding. */
75
- prepare(text) {
76
- if (!text) return "";
77
- const normalized = text.normalize("NFKC").trim();
78
- return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
79
- }
80
- tensorToVectors(out, batchSize) {
81
- if (typeof out.tolist === "function") {
82
- const nested = out.tolist();
83
- if (Array.isArray(nested) && Array.isArray(nested[0])) {
84
- return nested.map((row) => Float32Array.from(row));
85
- }
86
- return [Float32Array.from(nested)];
78
+ };
79
+ const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
80
+ const memoryId = payload?.memoryId;
81
+ if (typeof memoryId !== "string") return;
82
+ void mirror(memoryId);
83
+ });
84
+ const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
85
+ const memoryId = payload?.memoryId;
86
+ if (typeof memoryId !== "string") return;
87
+ void mirror(memoryId);
88
+ });
89
+ const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
90
+ const memoryId = payload?.memoryId;
91
+ if (typeof memoryId !== "string") return;
92
+ const status = payload?.status;
93
+ if (status === "deleted") return;
94
+ void mirror(memoryId);
95
+ });
96
+ const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
97
+ const memoryId = payload?.memoryId;
98
+ if (typeof memoryId !== "string") return;
99
+ void forgetMirror(memoryId);
100
+ });
101
+ return {
102
+ dispose: () => {
103
+ offAccepted();
104
+ offRecovered();
105
+ offUpdated();
106
+ offDeleted();
87
107
  }
88
- const flat = out.data;
89
- if (flat instanceof Float32Array) {
90
- if (batchSize === 1) return [flat];
91
- const dim = flat.length / batchSize;
92
- const vectors = [];
93
- for (let i = 0; i < batchSize; i++) {
94
- vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
108
+ };
109
+ }
110
+ async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
111
+ const surface = getSageSurface(memoryStore);
112
+ if (!surface) return { scanned: 0, removed: 0 };
113
+ let scanned = 0;
114
+ let removed = 0;
115
+ const PAGE = Math.max(1, options?.pageSize ?? 500);
116
+ let after;
117
+ for (; ; ) {
118
+ const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
119
+ if (page.length === 0) break;
120
+ const last = page[page.length - 1];
121
+ after = { updatedAt: last.updatedAt, id: last.id };
122
+ for (const entry of page) {
123
+ scanned++;
124
+ const sageId = entry.metadata?.sageId;
125
+ if (typeof sageId !== "string") continue;
126
+ try {
127
+ const memory = await surface.getSage(sageId);
128
+ if (memory !== null && memory.status !== "deleted") continue;
129
+ await store.forget(entry.id);
130
+ removed++;
131
+ } catch (err) {
132
+ logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg(err)}`);
95
133
  }
96
- return vectors;
97
- }
98
- if (Array.isArray(flat) && Array.isArray(flat[0])) {
99
- return flat.map((row) => Float32Array.from(row));
100
134
  }
101
- if (Array.isArray(flat)) {
102
- return [Float32Array.from(flat)];
135
+ if (page.length < PAGE) break;
136
+ }
137
+ return { scanned, removed };
138
+ }
139
+ function errMsg(err) {
140
+ return err instanceof Error ? err.message : String(err);
141
+ }
142
+ var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
143
+ var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
144
+ async function sweepStaleSageMirrors(opts) {
145
+ const markerPath2 = path.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
146
+ const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
147
+ if (!opts.force) {
148
+ try {
149
+ const raw = JSON.parse(fs.readFileSync(markerPath2, "utf8"));
150
+ const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
151
+ if (Number.isFinite(at) && Date.now() - at < interval) {
152
+ return { swept: false, reason: "throttled" };
153
+ }
154
+ } catch {
103
155
  }
104
- throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
105
156
  }
106
- async getExtractor() {
107
- if (this.extractor) return this.extractor;
108
- if (!this.loadPromise) this.loadPromise = this.loadExtractor();
109
- this.extractor = await this.loadPromise;
110
- return this.extractor;
157
+ try {
158
+ fs.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
159
+ } catch {
111
160
  }
112
- async loadExtractor() {
113
- const mod = await this.loadModule();
114
- if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
115
- mod.env.allowRemoteModels = this.allowRemote;
116
- if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
117
- const pipe = await mod.pipeline("feature-extraction", this.modelId, {
118
- dtype: this.dtype,
119
- device: this.device
120
- });
121
- return pipe;
161
+ try {
162
+ const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
163
+ opts.logger?.debug?.(
164
+ `vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
165
+ );
166
+ return { swept: true, ...result };
167
+ } catch (err) {
168
+ opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg(err)}`);
169
+ return { swept: false, reason: errMsg(err) };
122
170
  }
123
- async loadModule() {
171
+ }
172
+
173
+ // src/sage-fusion.ts
174
+ var DEFAULT_RRF_K = 60;
175
+ var DEFAULT_VECTOR_WEIGHT = 0.3;
176
+ function lexicalRankScore(index, total) {
177
+ if (total <= 1) return 1;
178
+ return 1 - index / Math.max(1, total - 1);
179
+ }
180
+ async function fuseWithVectorMemory(query, lexical, options = {}) {
181
+ const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
182
+ const k = options.rrfK ?? DEFAULT_RRF_K;
183
+ const limit = options.limit ?? 25;
184
+ let vectorHits = [];
185
+ if (options.vectorHits) {
186
+ vectorHits = [...options.vectorHits];
187
+ } else if (options.store) {
124
188
  try {
125
- return await import("@huggingface/transformers");
126
- } catch (err) {
127
- throw new VectorMemoryProviderUnavailableError(
128
- "@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
129
- err
130
- );
189
+ vectorHits = await options.store.search(query, {
190
+ ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
191
+ limit: Math.max(limit * 2, 50)
192
+ });
193
+ } catch {
194
+ vectorHits = [];
131
195
  }
132
196
  }
133
- };
134
-
135
- // src/schema.ts
136
- var VECTOR_SCHEMA_VERSION = 2;
137
- var VECTOR_PROVIDER_KEY = "active_provider_id";
138
- var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
139
- function initVectorSchema(db) {
140
- db.exec("PRAGMA journal_mode = WAL");
141
- db.exec("PRAGMA synchronous = NORMAL");
142
- db.exec("PRAGMA busy_timeout = 30000");
143
- db.exec("PRAGMA temp_store = MEMORY");
144
- db.exec("PRAGMA foreign_keys = ON");
145
- db.exec(`
146
- CREATE TABLE IF NOT EXISTS schema_meta (
147
- key TEXT PRIMARY KEY,
148
- value TEXT NOT NULL
149
- );
150
- `);
151
- db.exec(`
152
- CREATE TABLE IF NOT EXISTS entries (
153
- id TEXT PRIMARY KEY,
154
- text TEXT NOT NULL,
155
- summary TEXT,
156
- metadata TEXT NOT NULL DEFAULT '{}',
157
- tags TEXT NOT NULL DEFAULT '[]',
158
- scope TEXT NOT NULL DEFAULT 'project',
159
- kind TEXT NOT NULL DEFAULT 'note',
160
- content_hash TEXT NOT NULL,
161
- created_at TEXT NOT NULL,
162
- updated_at TEXT NOT NULL
163
- );
164
- `);
165
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
166
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
167
- db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
168
- db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
169
- db.exec(`
170
- CREATE TABLE IF NOT EXISTS vectors (
171
- entry_id TEXT NOT NULL,
172
- provider_id TEXT NOT NULL,
173
- dimensions INTEGER NOT NULL,
174
- vector BLOB NOT NULL,
175
- created_at TEXT NOT NULL,
176
- PRIMARY KEY (entry_id, provider_id),
177
- FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
178
- );
179
- `);
180
- db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
181
- db.exec(`
182
- CREATE TABLE IF NOT EXISTS embedding_cache (
183
- content_hash TEXT NOT NULL,
184
- provider_id TEXT NOT NULL,
185
- dimensions INTEGER NOT NULL,
186
- vector BLOB NOT NULL,
187
- text TEXT NOT NULL,
188
- created_at TEXT NOT NULL,
189
- last_used_at TEXT NOT NULL,
190
- use_count INTEGER NOT NULL DEFAULT 0,
191
- PRIMARY KEY (content_hash, provider_id, dimensions)
192
- );
193
- `);
194
- db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
195
- db.exec(`
196
- CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
197
- id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
198
- );
199
- `);
200
- db.exec(`
201
- CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
202
- INSERT INTO entries_fts(rowid, text, tags)
203
- VALUES (new.rowid, new.text, new.tags);
204
- END;
205
- `);
206
- db.exec(`
207
- CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
208
- INSERT INTO entries_fts(entries_fts, rowid, text, tags)
209
- VALUES('delete', old.rowid, old.text, old.tags);
210
- END;
211
- `);
212
- db.exec(`
213
- CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
214
- INSERT INTO entries_fts(entries_fts, rowid, text, tags)
215
- VALUES('delete', old.rowid, old.text, old.tags);
216
- INSERT INTO entries_fts(rowid, text, tags)
217
- VALUES (new.rowid, new.text, new.tags);
218
- END;
219
- `);
220
- }
221
- function upsertEmbeddingCache(db, row) {
222
- db.prepare(
223
- `INSERT INTO embedding_cache
224
- (content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
225
- VALUES (?, ?, ?, ?, ?, ?, ?, 1)
226
- ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
227
- last_used_at = excluded.last_used_at,
228
- use_count = embedding_cache.use_count + 1,
229
- vector = excluded.vector`
230
- ).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
231
- }
232
- function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
233
- const row = db.prepare(
234
- `SELECT vector FROM embedding_cache
235
- WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
236
- LIMIT 1`
237
- ).get(contentHash, providerId, dimensions);
238
- if (!row) return void 0;
239
- try {
240
- db.prepare(
241
- `UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
242
- WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
243
- ).run(now, contentHash, providerId, dimensions);
244
- } catch {
197
+ const sageById = /* @__PURE__ */ new Map();
198
+ for (const memory of lexical) sageById.set(memory.id, memory);
199
+ const lexicalRanked = lexical.map((memory, index) => ({
200
+ memory,
201
+ rankScore: lexicalRankScore(index, lexical.length),
202
+ lexicalScore: lexicalRankScore(index, lexical.length)
203
+ }));
204
+ const vectorRanked = [];
205
+ const seenVectorSageIds = /* @__PURE__ */ new Set();
206
+ for (let i = 0; i < vectorHits.length; i++) {
207
+ const hit = vectorHits[i];
208
+ const sageId = hit.entry.metadata?.["sageId"];
209
+ if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
210
+ const memory = sageById.get(sageId);
211
+ if (!memory) continue;
212
+ seenVectorSageIds.add(sageId);
213
+ vectorRanked.push({ memory, vectorScore: hit.score });
245
214
  }
246
- return decodeVector(row.vector);
247
- }
248
- function encodeVector(vec) {
249
- return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
250
- }
251
- function decodeVector(buf) {
252
- if (buf.byteLength % 4 !== 0) {
253
- throw new Error(
254
- `decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
255
- );
215
+ const fused = /* @__PURE__ */ new Map();
216
+ for (let i = 0; i < lexicalRanked.length; i++) {
217
+ const c = lexicalRanked[i];
218
+ const rrf = (1 - weight) * (1 / (k + i + 1));
219
+ fused.set(c.memory.id, {
220
+ memory: c.memory,
221
+ vectorScore: null,
222
+ lexicalScore: c.lexicalScore,
223
+ finalScore: rrf,
224
+ source: "lexical"
225
+ });
256
226
  }
257
- const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
258
- const copy = new Float32Array(buf.byteLength / 4);
259
- for (let i = 0; i < copy.length; i++) {
260
- copy[i] = view.getFloat32(i * 4, true);
227
+ for (let i = 0; i < vectorRanked.length; i++) {
228
+ const v = vectorRanked[i];
229
+ const rrf = weight * (1 / (k + i + 1));
230
+ const existing = fused.get(v.memory.id);
231
+ if (existing) {
232
+ existing.finalScore += rrf;
233
+ existing.vectorScore = v.vectorScore;
234
+ existing.source = "both";
235
+ }
261
236
  }
262
- return copy;
263
- }
264
-
265
- // src/store.ts
266
- import { createHash, randomUUID } from "node:crypto";
267
- import * as fs from "node:fs";
268
- import * as path from "node:path";
269
- import { withFileLock } from "@wrongstack/core/utils";
270
- import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
271
- import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
272
- var DEFAULT_SEARCH_LIMIT = 10;
273
- function normalizeLimit(limit) {
274
- if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
275
- const floored = Math.floor(limit);
276
- return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
237
+ const out = Array.from(fused.values());
238
+ out.sort((a, b) => b.finalScore - a.finalScore);
239
+ return out.slice(0, limit);
277
240
  }
278
- var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
279
- var DEFAULT_FILENAME = "vector-memory.db";
280
- var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
281
- var CACHE_EVICT_BATCH = 256;
282
- var VectorMemoryStore = class _VectorMemoryStore {
283
- db;
284
- dbPath;
285
- rootDir;
286
- provider;
287
- closed = false;
288
- constructor(opts) {
289
- if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
290
- if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
291
- this.provider = opts.provider;
292
- const dir = opts.directory ?? DEFAULT_DIRECTORY;
293
- const filename = opts.filename ?? DEFAULT_FILENAME;
294
- if (path.isAbsolute(dir)) {
295
- throw new Error("Vector memory directory must be project-relative.");
241
+ function asVectorRecallProvider(store) {
242
+ return {
243
+ async search(query, opts) {
244
+ const hits = await store.search(query, {
245
+ limit: opts.limit,
246
+ ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
247
+ });
248
+ return hits.map((h) => ({
249
+ id: h.entry.id,
250
+ score: h.score,
251
+ text: h.entry.text,
252
+ ...h.entry.summary ? { summary: h.entry.summary } : {},
253
+ tags: h.entry.tags,
254
+ ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
255
+ }));
296
256
  }
297
- const rootDir = path.resolve(opts.projectRoot, dir);
298
- const rel = path.relative(path.resolve(opts.projectRoot), rootDir);
299
- if (rel.startsWith("..") || path.isAbsolute(rel)) {
300
- throw new Error("Vector memory directory must stay inside the project root.");
257
+ };
258
+ }
259
+ function clamp01(value) {
260
+ if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
261
+ if (value < 0) return 0;
262
+ if (value > 1) return 1;
263
+ return value;
264
+ }
265
+
266
+ // src/sage-port-wrapper.ts
267
+ import {
268
+ augmentLexicalWithVectorRecall,
269
+ isSageVisibleForSearch,
270
+ SAGE_RETRIEVAL_CAPABILITY,
271
+ SAGE_SURFACE_CAPABILITY
272
+ } from "@wrongstack/sage";
273
+ function asVectorRecallProviderAdapter(store) {
274
+ return {
275
+ async search(query, opts) {
276
+ const hits = await store.search(query, {
277
+ limit: opts.limit,
278
+ ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
279
+ });
280
+ return hits.map((h) => ({
281
+ id: h.entry.id,
282
+ score: h.score,
283
+ text: h.entry.text,
284
+ ...h.entry.summary ? { summary: h.entry.summary } : {},
285
+ tags: h.entry.tags,
286
+ ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
287
+ }));
301
288
  }
302
- fs.mkdirSync(rootDir, { recursive: true });
303
- this.rootDir = rootDir;
304
- this.dbPath = path.join(rootDir, filename);
305
- const Database = loadRuntimeDatabaseSync();
306
- this.db = new Database(this.dbPath);
307
- initVectorSchema(this.db);
308
- this.recordActiveProvider();
309
- }
310
- /**
311
- * Absolute path of the store's data directory (the resolved
312
- * `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
313
- * to place sidecar state (e.g. the first-boot SAGE sync marker) next to
314
- * the db instead of re-deriving the path and drifting from it.
315
- */
316
- get directory() {
317
- return this.rootDir;
318
- }
319
- /**
320
- * Absolute path of the SQLite database file. Hosts use this to take a
321
- * file-level lock that covers all mutating operations (see
322
- * `withFileLock(this.dbPath + '.lock', …)`).
323
- */
324
- get databasePath() {
325
- return this.dbPath;
326
- }
327
- /** The lockfile path used to serialize mutating operations. */
328
- get lockPath() {
329
- return `${this.dbPath}.lock`;
330
- }
331
- get activeProviderId() {
332
- const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
333
- return row?.value ?? this.provider.id;
334
- }
335
- recordActiveProvider() {
336
- this.db.exec("BEGIN");
337
- try {
338
- this.db.prepare(
339
- `INSERT INTO schema_meta (key, value) VALUES (?, ?)
340
- ON CONFLICT(key) DO UPDATE SET value = excluded.value`
341
- ).run(VECTOR_PROVIDER_KEY, this.provider.id);
342
- this.db.prepare(
343
- `INSERT INTO schema_meta (key, value) VALUES (?, ?)
344
- ON CONFLICT(key) DO UPDATE SET value = excluded.value`
345
- ).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
346
- this.db.exec("COMMIT");
347
- } catch (e) {
348
- this.db.exec("ROLLBACK");
349
- throw e;
350
- }
351
- }
352
- static contentHash(text) {
353
- return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
354
- }
355
- /**
356
- * Look up a vector for `text` in the provider-level embedding cache.
357
- * Cache hit returns the cached vector (no ONNX pass). Miss returns
358
- * `undefined`.
359
- */
360
- cachedVector(text, now) {
361
- return lookupEmbeddingCache(
362
- this.db,
363
- _VectorMemoryStore.contentHash(text),
364
- this.provider.id,
365
- this.provider.dimensions,
366
- now
289
+ };
290
+ }
291
+ function wrapMemoryPortWithVectorRecall(port, options) {
292
+ const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
293
+ const materializeFor = (searchOpts) => async (sageId) => {
294
+ const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
295
+ if (!surface?.getSage) return void 0;
296
+ const memory = await surface.getSage(sageId);
297
+ if (!memory) return void 0;
298
+ return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
299
+ };
300
+ const fusionOptions = (searchOpts) => ({
301
+ vectorRecall: recall,
302
+ materializeVectorOnly: materializeFor(searchOpts),
303
+ ...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
304
+ ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
305
+ ...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
306
+ ...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
307
+ ...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
308
+ });
309
+ const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
310
+ const wrapSearchSage = (original) => async (query, searchOpts) => {
311
+ const opts = searchOpts;
312
+ const lexical = await original(query, searchOpts);
313
+ if (callerOwnsFusion(opts)) return lexical;
314
+ const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
315
+ return fused.map((hit) => hit.memory);
316
+ };
317
+ const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
318
+ const opts = searchOpts;
319
+ const lexicalHits = await original(query, searchOpts);
320
+ if (callerOwnsFusion(opts)) return lexicalHits;
321
+ return augmentLexicalWithVectorRecall(
322
+ query,
323
+ lexicalHits.map((hit) => hit.memory),
324
+ fusionOptions(opts)
367
325
  );
368
- }
369
- /** Persist `vec` for `text` to the embedding cache. */
370
- cacheVector(text, vec, now) {
371
- upsertEmbeddingCache(this.db, {
372
- contentHash: _VectorMemoryStore.contentHash(text),
373
- providerId: this.provider.id,
374
- dimensions: vec.length,
375
- vector: encodeVector(vec),
376
- text,
377
- now
378
- });
379
- }
380
- /**
381
- * Embed `text`, hitting the provider-level cache first. Cache miss falls
382
- * through to the configured provider and writes the result back. Returns
383
- * `undefined` when the provider fails — the caller can persist the entry
384
- * without a vector (fail-open).
385
- */
386
- async embedWithCache(text) {
387
- const now = (/* @__PURE__ */ new Date()).toISOString();
388
- const cached = this.cachedVector(text, now);
389
- if (cached) return cached;
390
- try {
391
- const result = await this.provider.embed([text]);
392
- const vec = result[0];
393
- if (vec) this.cacheVector(text, vec, now);
394
- return vec;
395
- } catch {
396
- return void 0;
397
- }
398
- }
399
- /**
400
- * Look up an existing entry by `content_hash`. Returns `undefined` when
401
- * the entry is not present. Used by `remember()` to make writes idempotent
402
- * and by `syncFromSage()` to skip already-indexed SAGE memories.
403
- */
404
- findByContentHash(contentHash) {
405
- this.assertOpen();
406
- const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
407
- if (!row) return void 0;
408
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
409
- return this.rowToEntry(row, vectorRow);
410
- }
411
- /**
412
- * Look up the entry mirroring a given SAGE memory id (i.e. the entry
413
- * whose `metadata.sageId` equals `sageId`). Returns `undefined` when
414
- * no such entry exists. Used by the event-driven mirror to delete
415
- * vector entries on SAGE delete events (the emitter knows the SAGE
416
- * id, not the vector entry id).
417
- *
418
- * Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
419
- * column is the JSON blob `syncFromSage` writes, so this avoids a
420
- * full table scan.
421
- */
422
- findBySageId(sageId) {
423
- this.assertOpen();
424
- const row = this.db.prepare(
425
- `SELECT * FROM entries
426
- WHERE json_extract(metadata, '$.sageId') = ?
427
- ORDER BY updated_at DESC
428
- LIMIT 1`
429
- ).get(sageId);
430
- if (!row) return void 0;
431
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
432
- return this.rowToEntry(row, vectorRow);
433
- }
434
- /**
435
- * Persist a new entry. Idempotent: if an entry with the same
436
- * `content_hash` already exists, that entry is returned unchanged
437
- * instead of inserting a duplicate. Mutating ops are wrapped in
438
- * `withFileLock` so two processes cannot race the dedup check.
439
- */
440
- async remember(input) {
441
- this.assertOpen();
442
- if (!input.text || input.text.trim().length === 0) {
443
- throw new Error("VectorMemoryStore.remember: text must be non-empty");
326
+ };
327
+ const wrapped = Object.create(
328
+ Object.getPrototypeOf(port),
329
+ Object.getOwnPropertyDescriptors(port)
330
+ );
331
+ wrapped.getCapability = (capability) => {
332
+ if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
333
+ const original = port.getCapability(capability);
334
+ if (!original) return void 0;
335
+ return {
336
+ ...original,
337
+ searchSage: wrapSearchSage(original.searchSage),
338
+ ...original.searchSageWithBreakdown ? {
339
+ searchSageWithBreakdown: wrapSearchWithBreakdown(
340
+ original.searchSageWithBreakdown
341
+ )
342
+ } : {}
343
+ };
444
344
  }
445
- return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
446
- timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
447
- });
448
- }
449
- async rememberUnlocked(input) {
450
- const now = (/* @__PURE__ */ new Date()).toISOString();
451
- const contentHash = _VectorMemoryStore.contentHash(input.text);
452
- const existing = this.findByContentHash(contentHash);
453
- if (existing) return existing;
454
- const metadata = input.metadata ?? {};
455
- const tags = input.tags ?? [];
456
- const scope = input.scope ?? "project";
457
- const kind = input.kind ?? "note";
458
- const id = randomUUID();
459
- const vector = await this.embedWithCache(input.text);
460
- const providerId = vector ? this.provider.id : void 0;
461
- this.db.exec("BEGIN");
462
- try {
463
- this.db.prepare(
464
- `INSERT INTO entries
465
- (id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
466
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
467
- ).run(
468
- id,
469
- input.text,
470
- input.summary ?? null,
471
- JSON.stringify(metadata),
472
- JSON.stringify(tags),
473
- scope,
474
- kind,
475
- contentHash,
476
- now,
477
- now
478
- );
479
- if (vector && providerId) {
480
- this.db.prepare(
481
- `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
482
- VALUES (?, ?, ?, ?, ?)
483
- ON CONFLICT(entry_id, provider_id) DO UPDATE SET
484
- vector = excluded.vector,
485
- dimensions = excluded.dimensions,
486
- created_at = excluded.created_at`
487
- ).run(id, providerId, vector.length, encodeVector(vector), now);
488
- }
489
- this.db.exec("COMMIT");
490
- } catch (e) {
491
- this.db.exec("ROLLBACK");
492
- throw e;
345
+ if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
346
+ const original = port.getCapability(capability);
347
+ if (!original) return void 0;
348
+ return {
349
+ ...original,
350
+ searchSage: wrapSearchSage(original.searchSage),
351
+ ...original.searchSageWithBreakdown ? {
352
+ searchSageWithBreakdown: wrapSearchWithBreakdown(
353
+ original.searchSageWithBreakdown
354
+ )
355
+ } : {}
356
+ };
493
357
  }
494
- const result = {
495
- id,
496
- text: input.text,
497
- summary: input.summary ?? void 0,
498
- metadata,
499
- tags,
500
- scope,
501
- kind,
502
- contentHash,
503
- createdAt: now,
504
- updatedAt: now,
505
- providerId: providerId ?? "",
506
- dimensions: vector?.length ?? 0
507
- };
508
- if (vector) result.vector = vector;
509
- return result;
510
- }
511
- get(id) {
512
- this.assertOpen();
513
- const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
514
- if (!row) return void 0;
515
- const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
516
- return this.rowToEntry(row, vectorRow);
517
- }
518
- /** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
519
- async forget(id) {
520
- this.assertOpen();
521
- return withFileLock(
522
- this.lockPath,
523
- async () => {
524
- this.db.exec("BEGIN");
525
- try {
526
- const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
527
- this.db.exec("COMMIT");
528
- return info.changes > 0;
529
- } catch (e) {
530
- this.db.exec("ROLLBACK");
531
- throw e;
358
+ return port.getCapability(capability);
359
+ };
360
+ return wrapped;
361
+ }
362
+
363
+ // src/sage-sync.ts
364
+ import * as fs2 from "node:fs";
365
+ import * as path2 from "node:path";
366
+ import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
367
+
368
+ // src/sage-sync-source.ts
369
+ var DEFAULT_PAGE_SIZE = 500;
370
+ var HARD_FLOOR_PAGE = 1;
371
+ var HARD_CEILING_PAGE = 500;
372
+ var HARD_CEILING_TOTAL = 1e6;
373
+ function createSageSurfaceSyncSource(sage, opts = {}) {
374
+ const pageSize = clamp(opts.pageSize ?? DEFAULT_PAGE_SIZE, HARD_FLOOR_PAGE, HARD_CEILING_PAGE);
375
+ const maxTotal = opts.maxTotal === void 0 ? Number.POSITIVE_INFINITY : clamp(opts.maxTotal, 1, HARD_CEILING_TOTAL);
376
+ return {
377
+ async listActiveMemories({ limit }) {
378
+ const requested = limit === void 0 ? maxTotal : clamp(limit, 1, maxTotal === Number.POSITIVE_INFINITY ? limit : maxTotal);
379
+ const memories = [];
380
+ let cursor;
381
+ let noProgressPages = 0;
382
+ while (memories.length < requested) {
383
+ const remaining = requested - memories.length;
384
+ const page = await sage.listSagePage({
385
+ statuses: ["active"],
386
+ limit: Math.min(pageSize, Math.max(1, remaining)),
387
+ ...cursor ? { cursor } : {}
388
+ });
389
+ const rows = page.memories ?? [];
390
+ for (const m of rows) {
391
+ memories.push({
392
+ id: m.id,
393
+ text: m.text,
394
+ ...m.summary ? { summary: m.summary } : {},
395
+ ...m.tags && m.tags.length > 0 ? { tags: m.tags } : {},
396
+ metadata: {
397
+ sageKind: m.kind,
398
+ sageScope: m.scope,
399
+ importance: m.importance,
400
+ confidence: m.confidence
401
+ }
402
+ });
532
403
  }
533
- },
534
- { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
535
- );
536
- }
537
- async search(query, opts = {}) {
538
- this.assertOpen();
539
- const limit = normalizeLimit(opts.limit);
540
- const threshold = opts.threshold ?? 0;
541
- const includeVectors = opts.includeVectors === true;
542
- if (typeof query !== "string" || query.trim().length === 0) return [];
543
- const queryVec = await this.embedWithCache(query);
544
- if (!queryVec || queryVec.length === 0) return [];
545
- const providerId = this.provider.id;
546
- const dimensions = this.provider.dimensions;
547
- const filters = ["v.provider_id = ?", "v.dimensions = ?"];
548
- const params = [providerId, dimensions];
549
- if (opts.scope !== void 0) {
550
- filters.push("e.scope = ?");
551
- params.push(opts.scope);
404
+ if (!page.nextCursor) break;
405
+ if (rows.length === 0) {
406
+ noProgressPages++;
407
+ if (noProgressPages > 3) break;
408
+ } else {
409
+ noProgressPages = 0;
410
+ }
411
+ cursor = page.nextCursor;
412
+ }
413
+ return memories.slice(0, requested);
552
414
  }
553
- if (opts.kind !== void 0) {
554
- filters.push("e.kind = ?");
555
- params.push(opts.kind);
415
+ };
416
+ }
417
+ function clamp(value, min, max) {
418
+ return Math.max(min, Math.min(max, value));
419
+ }
420
+
421
+ // src/sage-sync.ts
422
+ var SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
423
+ var RUNNING_STALE_MS = 10 * 60 * 1e3;
424
+ async function startFirstBootSageSync(opts) {
425
+ const { store, memoryStore } = opts;
426
+ const log = opts.logger;
427
+ try {
428
+ if (opts.force) {
429
+ try {
430
+ fs2.unlinkSync(markerPath(store));
431
+ } catch {
432
+ }
556
433
  }
557
- const scanRows = this.db.prepare(
558
- `SELECT e.id AS id, v.vector AS vec_blob
559
- FROM entries e
560
- JOIN vectors v ON v.entry_id = e.id
561
- WHERE ${filters.join(" AND ")}`
562
- ).all(...params);
563
- const top = [];
564
- for (const row of scanRows) {
565
- const vec = decodeVector(row.vec_blob);
566
- const raw = cosineSimilarity(queryVec, vec);
567
- const score = Math.max(0, Math.min(1, raw));
568
- if (!Number.isFinite(score) || score < threshold) continue;
569
- if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
570
- let at = top.length;
571
- while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
572
- top.splice(at, 0, { id: row.id, score, vector: vec });
573
- if (top.length > limit) top.length = limit;
434
+ const decision = decideWhetherToSync(
435
+ store,
436
+ opts.staleAfterMs ?? RUNNING_STALE_MS,
437
+ void 0,
438
+ opts.pidAlive
439
+ );
440
+ if (!decision.run) {
441
+ log?.debug?.(`vector-memory sage sync skipped: ${decision.reason}`);
442
+ return { synced: false, reason: decision.reason };
574
443
  }
575
- if (top.length === 0) return [];
576
- const placeholders = top.map(() => "?").join(",");
577
- const hydrated = this.db.prepare(
578
- `SELECT id, text, summary, metadata, tags, scope, kind,
579
- content_hash, created_at, updated_at
580
- FROM entries WHERE id IN (${placeholders})`
581
- ).all(...top.map((t) => t.id));
582
- const entryById = /* @__PURE__ */ new Map();
583
- for (const row of hydrated) {
584
- entryById.set(row.id, this.rowToEntry(row));
444
+ const provider = storeProvider(store);
445
+ if (provider && typeof provider.isAvailable === "function" && !await provider.isAvailable()) {
446
+ log?.debug?.(
447
+ "vector-memory sage sync deferred: embedding provider unavailable (optional dependency not installed?)"
448
+ );
449
+ return { synced: false, reason: "provider-unavailable" };
585
450
  }
586
- const scored = [];
587
- for (const candidate of top) {
588
- const entry = entryById.get(candidate.id);
589
- if (!entry) continue;
590
- const hit = { entry, score: candidate.score, providerId };
591
- if (includeVectors) hit.vector = candidate.vector;
592
- scored.push(hit);
451
+ if (provider && typeof provider.embed === "function") {
452
+ try {
453
+ const probe = await provider.embed(["wrongstack vector memory warmup probe"]);
454
+ if (!probe[0] || probe[0].length === 0) throw new Error("empty embedding");
455
+ } catch {
456
+ log?.debug?.(
457
+ "vector-memory sage sync deferred: embedding probe failed (model not cached / backend error)"
458
+ );
459
+ return { synced: false, reason: "provider-unavailable" };
460
+ }
593
461
  }
594
- return scored;
595
- }
596
- /**
597
- * Page through entries, newest first.
598
- *
599
- * Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
600
- * without the id tiebreak two entries written in the same millisecond can
601
- * swap places between calls and a paging caller silently skips one.
602
- *
603
- * Pagination is keyset (`after`), not offset, because the only caller that
604
- * pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
605
- * `OFFSET` every deletion shifts the remaining rows left and the next page
606
- * skips exactly as many entries as were removed. Keyset is immune: it
607
- * resumes from a position, and the rows a deletion removes are ones the
608
- * sweep has already passed.
609
- */
610
- list(opts = {}) {
611
- this.assertOpen();
612
- const where = [];
613
- const params = [];
614
- if (opts.scope !== void 0) {
615
- where.push("scope = ?");
616
- params.push(opts.scope);
462
+ const surface = getSageSurface2(memoryStore);
463
+ if (!surface) {
464
+ log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
465
+ return { synced: false, reason: "no-sage-surface" };
617
466
  }
618
- if (opts.kind !== void 0) {
619
- where.push("kind = ?");
620
- params.push(opts.kind);
467
+ writeMarker(store, {
468
+ phase: "running",
469
+ pid: process.pid,
470
+ startedAt: (/* @__PURE__ */ new Date()).toISOString()
471
+ });
472
+ const report = await store.syncFromSage(createSageSurfaceSyncSource(surface));
473
+ if (report.failed > 0) {
474
+ log?.warn?.(
475
+ `vector-memory sage sync finished with ${report.failed} failure(s) \u2014 will retry on next boot`
476
+ );
477
+ return {
478
+ synced: false,
479
+ reason: "partial-failure",
480
+ marker: { phase: "running", ...counts(report) }
481
+ };
621
482
  }
622
- if (opts.after) {
623
- where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
624
- params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
483
+ let stats = store.stats();
484
+ if (stats.vectors !== stats.entries) {
485
+ log?.warn?.(
486
+ `vector-memory sage sync healed ${stats.entries - stats.vectors} vector-less entr(ies) via reindexAll()`
487
+ );
488
+ await store.reindexAll();
489
+ stats = store.stats();
625
490
  }
626
- const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
627
- ORDER BY updated_at DESC, id DESC LIMIT ?`;
628
- params.push(opts.limit ?? 100);
629
- const rows = this.db.prepare(sql).all(...params);
630
- return rows.map((r) => this.rowToEntry(r));
631
- }
632
- async reindexAll() {
633
- this.assertOpen();
634
- return withFileLock(
635
- this.lockPath,
636
- async () => {
637
- const rows = this.db.prepare("SELECT id, text FROM entries").all();
638
- let processed = 0;
639
- let errors = 0;
640
- for (const row of rows) {
641
- try {
642
- const result = await this.provider.embed([row.text]);
643
- const v = result[0];
644
- if (!v) {
645
- errors++;
646
- continue;
647
- }
648
- const now = (/* @__PURE__ */ new Date()).toISOString();
649
- this.db.prepare(
650
- `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
651
- VALUES (?, ?, ?, ?, ?)
652
- ON CONFLICT(entry_id, provider_id) DO UPDATE SET
653
- vector = excluded.vector,
654
- dimensions = excluded.dimensions,
655
- created_at = excluded.created_at`
656
- ).run(row.id, this.provider.id, v.length, encodeVector(v), now);
657
- this.cacheVector(row.text, v, now);
658
- processed++;
659
- } catch {
660
- errors++;
661
- }
662
- }
663
- return { processed, errors };
664
- },
665
- { timeoutMs: 6e4 }
491
+ if (stats.vectors !== stats.entries) {
492
+ log?.warn?.(
493
+ `vector-memory sage sync incomplete: ${stats.entries - stats.vectors} entr(ies) still vector-less \u2014 will retry on next boot`
494
+ );
495
+ return {
496
+ synced: false,
497
+ reason: "vector-incomplete",
498
+ marker: { phase: "running", ...counts(report) }
499
+ };
500
+ }
501
+ const marker = {
502
+ phase: "complete",
503
+ completedAt: (/* @__PURE__ */ new Date()).toISOString(),
504
+ ...provider ? { providerId: provider.id } : {},
505
+ ...counts(report)
506
+ };
507
+ writeMarker(store, marker);
508
+ log?.info?.(
509
+ `vector-memory sage sync complete: ${report.indexed} indexed, ${report.skipped} skipped (already present)`
510
+ );
511
+ return { synced: true, reason: "synced", marker };
512
+ } catch (error) {
513
+ log?.warn?.(
514
+ `vector-memory sage sync failed: ${error instanceof Error ? error.message : String(error)} \u2014 will retry on next boot`
666
515
  );
516
+ return { synced: false, reason: "error" };
667
517
  }
668
- stats() {
669
- this.assertOpen();
670
- const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
671
- const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
672
- const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
673
- return {
674
- entries: entryCount,
675
- vectors: vectorCount,
676
- providers: providerRows.map((r) => r.provider_id),
677
- modelAvailable: true,
678
- modelId: this.provider.id,
679
- dimensions: this.provider.dimensions
680
- };
518
+ }
519
+ function decideWhetherToSync(store, staleAfterMs, now = /* @__PURE__ */ new Date(), pidAlive = defaultPidAlive) {
520
+ const existing = readMarker(store);
521
+ if (!existing) return { run: true, reason: "no-marker" };
522
+ if (existing.phase === "complete") return { run: false, reason: "already-complete" };
523
+ if (existing.pid === process.pid) {
524
+ return { run: true, reason: "running-own-pid" };
681
525
  }
682
- /**
683
- * Embedding-cache diagnostics entries, hit/miss counters, oldest entry.
684
- * Useful for the WebUI's vector-memory panel and for diagnosing the
685
- * "why is search slow?" question.
686
- */
687
- cacheStats() {
688
- this.assertOpen();
689
- const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
690
- const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
691
- const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
692
- const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
693
- return {
694
- entries,
695
- providers,
696
- totalUseCount,
697
- oldestLastUsedAt: oldest?.t ?? null
698
- };
526
+ const startedAt = existing.startedAt ? Date.parse(existing.startedAt) : Number.NaN;
527
+ const ageMs = Number.isNaN(startedAt) ? Number.POSITIVE_INFINITY : now.getTime() - startedAt;
528
+ const stale = ageMs > staleAfterMs;
529
+ if (existing.pid === void 0) {
530
+ return Number.isNaN(startedAt) || stale ? {
531
+ run: true,
532
+ reason: Number.isNaN(startedAt) ? "running-marker-undated" : "running-marker-stale"
533
+ } : { run: false, reason: "running-unknown-pid" };
699
534
  }
700
- /**
701
- * LRU-evict the embedding cache down to `keepMostRecent` rows. The
702
- * `embedding_cache` table is independent of `entries`, so a sweep here
703
- * only removes cached vectors — never stored entries. Called by hosts
704
- * that want to bound cache growth on long-lived processes.
705
- */
706
- async evictCache(keepMostRecent) {
707
- this.assertOpen();
708
- if (keepMostRecent < 0) {
709
- throw new Error("evictCache: keepMostRecent must be >= 0");
535
+ try {
536
+ if (pidAlive(existing.pid)) {
537
+ return {
538
+ run: false,
539
+ reason: `running-pid-${existing.pid}${stale ? "-stale-but-alive" : ""}`
540
+ };
710
541
  }
711
- return withFileLock(
712
- this.lockPath,
713
- async () => {
714
- const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
715
- if (total <= keepMostRecent) return { removed: 0 };
716
- const toRemove = total - keepMostRecent;
717
- const stmt = this.db.prepare(
718
- `DELETE FROM embedding_cache
719
- WHERE content_hash IN (
720
- SELECT content_hash FROM embedding_cache
721
- ORDER BY last_used_at ASC
722
- LIMIT ?
723
- )`
724
- );
725
- const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
726
- return { removed: Number(info.changes) };
727
- },
728
- { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
729
- );
542
+ return { run: true, reason: `running-pid-${existing.pid}-dead` };
543
+ } catch {
544
+ return { run: false, reason: `running-pid-${existing.pid}-probe-failed` };
730
545
  }
731
- close() {
732
- if (this.closed) return;
733
- this.closed = true;
734
- this.db.close();
546
+ }
547
+ function defaultPidAlive(pid) {
548
+ try {
549
+ process.kill(pid, 0);
550
+ return true;
551
+ } catch (error) {
552
+ return error.code === "EPERM";
735
553
  }
736
- assertOpen() {
737
- if (this.closed) throw new Error("VectorMemoryStore is closed");
554
+ }
555
+ function readMarker(store) {
556
+ try {
557
+ const raw = fs2.readFileSync(markerPath(store), "utf8");
558
+ const parsed = JSON.parse(raw);
559
+ return parsed && (parsed.phase === "running" || parsed.phase === "complete") ? parsed : void 0;
560
+ } catch {
561
+ return void 0;
738
562
  }
739
- rowToEntry(row, vectorRow) {
740
- const summaryValue = row.summary;
741
- const entry = {
742
- id: row.id,
743
- text: row.text,
744
- summary: summaryValue ?? void 0,
745
- metadata: safeParseJson(row.metadata, {}),
746
- tags: safeParseJson(row.tags, []),
747
- scope: row.scope,
748
- kind: row.kind,
749
- contentHash: row.content_hash,
750
- createdAt: row.created_at,
751
- updatedAt: row.updated_at,
752
- providerId: vectorRow?.provider_id ?? "",
753
- dimensions: vectorRow?.dimensions ?? 0
754
- };
755
- if (vectorRow?.vector) {
756
- entry.vector = decodeVector(vectorRow.vector);
757
- }
758
- return entry;
563
+ }
564
+ function writeMarker(store, marker) {
565
+ try {
566
+ fs2.writeFileSync(markerPath(store), `${JSON.stringify(marker, null, 2)}
567
+ `, "utf8");
568
+ } catch {
569
+ }
570
+ }
571
+ function markerPath(store) {
572
+ const dir = store.directory;
573
+ if (typeof dir !== "string") {
574
+ throw new Error("VectorMemoryStore.directory is required to place the sage sync marker");
759
575
  }
760
- async syncFromSage(sage) {
761
- this.assertOpen();
762
- const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
763
- let indexed = 0;
764
- let skipped = 0;
765
- let failed = 0;
766
- const errors = [];
767
- for (const memory of memories) {
768
- try {
769
- const hash = _VectorMemoryStore.contentHash(memory.text);
770
- const existing = this.findByContentHash(hash);
771
- if (existing) {
772
- skipped++;
773
- continue;
774
- }
775
- await this.rememberUnlocked({
776
- text: memory.text,
777
- summary: memory.summary ?? void 0,
778
- metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
779
- tags: memory.tags ?? [],
780
- scope: "project",
781
- kind: "note"
782
- });
783
- indexed++;
784
- } catch (err) {
785
- failed++;
786
- errors.push({ memoryId: memory.id, message: errMsg(err) });
787
- }
788
- }
789
- return { scanned: memories.length, indexed, skipped, failed, errors };
576
+ return path2.join(dir, SAGE_SYNC_MARKER_FILENAME);
577
+ }
578
+ function storeProvider(store) {
579
+ const provider = store.provider;
580
+ if (provider && typeof provider.id === "string") {
581
+ return provider;
790
582
  }
791
- };
792
- function fallbackHashingProvider(dimensions) {
793
- return new HashingEmbeddingProvider({ dimensions });
583
+ return void 0;
794
584
  }
795
- function safeParseJson(value, fallback) {
796
- if (typeof value !== "string") return fallback;
585
+ function counts(report) {
586
+ return {
587
+ scanned: report.scanned,
588
+ indexed: report.indexed,
589
+ skipped: report.skipped,
590
+ failed: report.failed
591
+ };
592
+ }
593
+
594
+ // src/schema.ts
595
+ var VECTOR_SCHEMA_VERSION = 2;
596
+ var VECTOR_PROVIDER_KEY = "active_provider_id";
597
+ var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
598
+ function initVectorSchema(db) {
599
+ db.exec("PRAGMA journal_mode = WAL");
600
+ db.exec("PRAGMA synchronous = NORMAL");
601
+ db.exec("PRAGMA busy_timeout = 30000");
602
+ db.exec("PRAGMA temp_store = MEMORY");
603
+ db.exec("PRAGMA foreign_keys = ON");
604
+ db.exec(`
605
+ CREATE TABLE IF NOT EXISTS schema_meta (
606
+ key TEXT PRIMARY KEY,
607
+ value TEXT NOT NULL
608
+ );
609
+ `);
610
+ db.exec(`
611
+ CREATE TABLE IF NOT EXISTS entries (
612
+ id TEXT PRIMARY KEY,
613
+ text TEXT NOT NULL,
614
+ summary TEXT,
615
+ metadata TEXT NOT NULL DEFAULT '{}',
616
+ tags TEXT NOT NULL DEFAULT '[]',
617
+ scope TEXT NOT NULL DEFAULT 'project',
618
+ kind TEXT NOT NULL DEFAULT 'note',
619
+ content_hash TEXT NOT NULL,
620
+ created_at TEXT NOT NULL,
621
+ updated_at TEXT NOT NULL
622
+ );
623
+ `);
624
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
625
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
626
+ db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
627
+ db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
628
+ db.exec(`
629
+ CREATE TABLE IF NOT EXISTS vectors (
630
+ entry_id TEXT NOT NULL,
631
+ provider_id TEXT NOT NULL,
632
+ dimensions INTEGER NOT NULL,
633
+ vector BLOB NOT NULL,
634
+ created_at TEXT NOT NULL,
635
+ PRIMARY KEY (entry_id, provider_id),
636
+ FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
637
+ );
638
+ `);
639
+ db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
640
+ db.exec(`
641
+ CREATE TABLE IF NOT EXISTS embedding_cache (
642
+ content_hash TEXT NOT NULL,
643
+ provider_id TEXT NOT NULL,
644
+ dimensions INTEGER NOT NULL,
645
+ vector BLOB NOT NULL,
646
+ text TEXT NOT NULL,
647
+ created_at TEXT NOT NULL,
648
+ last_used_at TEXT NOT NULL,
649
+ use_count INTEGER NOT NULL DEFAULT 0,
650
+ PRIMARY KEY (content_hash, provider_id, dimensions)
651
+ );
652
+ `);
653
+ db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
654
+ db.exec(`
655
+ CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
656
+ id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
657
+ );
658
+ `);
659
+ db.exec(`
660
+ CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
661
+ INSERT INTO entries_fts(rowid, text, tags)
662
+ VALUES (new.rowid, new.text, new.tags);
663
+ END;
664
+ `);
665
+ db.exec(`
666
+ CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
667
+ INSERT INTO entries_fts(entries_fts, rowid, text, tags)
668
+ VALUES('delete', old.rowid, old.text, old.tags);
669
+ END;
670
+ `);
671
+ db.exec(`
672
+ CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
673
+ INSERT INTO entries_fts(entries_fts, rowid, text, tags)
674
+ VALUES('delete', old.rowid, old.text, old.tags);
675
+ INSERT INTO entries_fts(rowid, text, tags)
676
+ VALUES (new.rowid, new.text, new.tags);
677
+ END;
678
+ `);
679
+ }
680
+ function upsertEmbeddingCache(db, row) {
681
+ db.prepare(
682
+ `INSERT INTO embedding_cache
683
+ (content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
684
+ VALUES (?, ?, ?, ?, ?, ?, ?, 1)
685
+ ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
686
+ last_used_at = excluded.last_used_at,
687
+ use_count = embedding_cache.use_count + 1,
688
+ vector = excluded.vector`
689
+ ).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
690
+ }
691
+ function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
692
+ const row = db.prepare(
693
+ `SELECT vector FROM embedding_cache
694
+ WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
695
+ LIMIT 1`
696
+ ).get(contentHash, providerId, dimensions);
697
+ if (!row) return void 0;
797
698
  try {
798
- return JSON.parse(value);
699
+ db.prepare(
700
+ `UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
701
+ WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
702
+ ).run(now, contentHash, providerId, dimensions);
799
703
  } catch {
800
- return fallback;
801
704
  }
705
+ return decodeVector(row.vector);
802
706
  }
803
- function errMsg(err) {
804
- return err instanceof Error ? err.message : String(err);
707
+ function encodeVector(vec) {
708
+ return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
709
+ }
710
+ function decodeVector(buf) {
711
+ if (buf.byteLength % 4 !== 0) {
712
+ throw new Error(
713
+ `decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
714
+ );
715
+ }
716
+ const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
717
+ const copy = new Float32Array(buf.byteLength / 4);
718
+ for (let i = 0; i < copy.length; i++) {
719
+ copy[i] = view.getFloat32(i * 4, true);
720
+ }
721
+ return copy;
805
722
  }
806
723
 
807
- // src/sage-sync-source.ts
808
- var DEFAULT_PAGE_SIZE = 500;
809
- var HARD_FLOOR_PAGE = 1;
810
- var HARD_CEILING_PAGE = 500;
811
- var HARD_CEILING_TOTAL = 1e6;
812
- function createSageSurfaceSyncSource(sage, opts = {}) {
813
- const pageSize = clamp(opts.pageSize ?? DEFAULT_PAGE_SIZE, HARD_FLOOR_PAGE, HARD_CEILING_PAGE);
814
- const maxTotal = opts.maxTotal === void 0 ? Number.POSITIVE_INFINITY : clamp(opts.maxTotal, 1, HARD_CEILING_TOTAL);
815
- return {
816
- async listActiveMemories({ limit }) {
817
- const requested = limit === void 0 ? maxTotal : clamp(limit, 1, maxTotal === Number.POSITIVE_INFINITY ? limit : maxTotal);
818
- const memories = [];
819
- let cursor;
820
- let noProgressPages = 0;
821
- while (memories.length < requested) {
822
- const remaining = requested - memories.length;
823
- const page = await sage.listSagePage({
824
- statuses: ["active"],
825
- limit: Math.min(pageSize, Math.max(1, remaining)),
826
- ...cursor ? { cursor } : {}
827
- });
828
- const rows = page.memories ?? [];
829
- for (const m of rows) {
830
- memories.push({
831
- id: m.id,
832
- text: m.text,
833
- ...m.summary ? { summary: m.summary } : {},
834
- ...m.tags && m.tags.length > 0 ? { tags: m.tags } : {},
835
- metadata: {
836
- sageKind: m.kind,
837
- sageScope: m.scope,
838
- importance: m.importance,
839
- confidence: m.confidence
840
- }
841
- });
842
- }
843
- if (!page.nextCursor || rows.length === 0) break;
844
- if (rows.length === 0) {
845
- noProgressPages++;
846
- if (noProgressPages > 3) break;
847
- } else {
848
- noProgressPages = 0;
849
- }
850
- cursor = page.nextCursor;
724
+ // src/search-race.ts
725
+ function previewText(text, maxLen) {
726
+ if (text.length <= maxLen) return text;
727
+ return text.slice(0, maxLen - 1) + "\u2026";
728
+ }
729
+ async function runSearchRace(query, lexical, vectorStore, options = {}) {
730
+ const limit = options.limit ?? 20;
731
+ const threshold = options.threshold ?? 0;
732
+ let vectorHits = [];
733
+ try {
734
+ vectorHits = await vectorStore.search(query, {
735
+ limit,
736
+ ...threshold > 0 ? { threshold } : {}
737
+ });
738
+ } catch {
739
+ vectorHits = [];
740
+ }
741
+ const lexicalOnly = [];
742
+ const vectorOnly = [];
743
+ const overlap = [];
744
+ const seenIds = /* @__PURE__ */ new Set();
745
+ const lexicalCapped = lexical.slice(0, limit);
746
+ for (let i = 0; i < lexicalCapped.length; i++) {
747
+ const mem = lexicalCapped[i];
748
+ const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
749
+ const id = mem.id;
750
+ seenIds.add(id);
751
+ overlap.push({
752
+ id,
753
+ lexicalScore: score,
754
+ vectorScore: null,
755
+ // patched below when a vector hit carries this id
756
+ preview: previewText(mem.text, 140)
757
+ });
758
+ }
759
+ const overlapById = /* @__PURE__ */ new Map();
760
+ for (const row of overlap) overlapById.set(row.id, row);
761
+ for (const hit of vectorHits) {
762
+ const sageId = hit.entry.metadata?.["sageId"];
763
+ if (typeof sageId !== "string") continue;
764
+ const existing = overlapById.get(sageId);
765
+ if (existing) {
766
+ if (existing.vectorScore === null) {
767
+ existing.vectorScore = hit.score;
851
768
  }
852
- return memories.slice(0, requested);
769
+ continue;
770
+ }
771
+ if (seenIds.has(sageId)) continue;
772
+ seenIds.add(sageId);
773
+ vectorOnly.push({
774
+ id: sageId,
775
+ lexicalScore: null,
776
+ vectorScore: hit.score,
777
+ preview: previewText(hit.entry.text, 140)
778
+ });
779
+ }
780
+ const finalOverlap = [];
781
+ for (const row of overlap) {
782
+ if (row.vectorScore === null) {
783
+ lexicalOnly.push({
784
+ id: row.id,
785
+ lexicalScore: row.lexicalScore,
786
+ vectorScore: null,
787
+ preview: row.preview
788
+ });
789
+ } else {
790
+ finalOverlap.push(row);
791
+ }
792
+ }
793
+ const lexicalCount = lexicalOnly.length + finalOverlap.length;
794
+ const vectorCount = vectorOnly.length + finalOverlap.length;
795
+ const denom = Math.max(lexicalCount, vectorCount, 1);
796
+ return {
797
+ query,
798
+ lexicalOnly,
799
+ vectorOnly,
800
+ overlap: finalOverlap,
801
+ metrics: {
802
+ lexicalCount,
803
+ vectorCount,
804
+ overlapCount: finalOverlap.length,
805
+ lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
806
+ vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
807
+ agreementRatio: finalOverlap.length / denom
853
808
  }
854
809
  };
855
810
  }
856
- function clamp(value, min, max) {
857
- return Math.max(min, Math.min(max, value));
858
- }
859
811
 
860
- // src/sage-sync.ts
861
- import { getSageSurface } from "@wrongstack/sage";
862
- import * as fs2 from "node:fs";
863
- import * as path2 from "node:path";
864
- var SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
865
- var RUNNING_STALE_MS = 10 * 60 * 1e3;
866
- async function startFirstBootSageSync(opts) {
867
- const { store, memoryStore } = opts;
868
- const log = opts.logger;
869
- try {
870
- if (opts.force) {
871
- try {
872
- fs2.unlinkSync(markerPath(store));
873
- } catch {
874
- }
875
- }
876
- const decision = decideWhetherToSync(
877
- store,
878
- opts.staleAfterMs ?? RUNNING_STALE_MS,
879
- void 0,
880
- opts.pidAlive
881
- );
882
- if (!decision.run) {
883
- log?.debug?.(`vector-memory sage sync skipped: ${decision.reason}`);
884
- return { synced: false, reason: decision.reason };
885
- }
886
- const provider = storeProvider(store);
887
- if (provider && typeof provider.isAvailable === "function" && !await provider.isAvailable()) {
888
- log?.debug?.(
889
- "vector-memory sage sync deferred: embedding provider unavailable (optional dependency not installed?)"
890
- );
891
- return { synced: false, reason: "provider-unavailable" };
892
- }
893
- if (provider && typeof provider.embed === "function") {
894
- try {
895
- const probe = await provider.embed(["wrongstack vector memory warmup probe"]);
896
- if (!probe[0] || probe[0].length === 0) throw new Error("empty embedding");
897
- } catch {
898
- log?.debug?.(
899
- "vector-memory sage sync deferred: embedding probe failed (model not cached / backend error)"
900
- );
901
- return { synced: false, reason: "provider-unavailable" };
902
- }
903
- }
904
- const surface = getSageSurface(memoryStore);
905
- if (!surface) {
906
- log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
907
- return { synced: false, reason: "no-sage-surface" };
908
- }
909
- writeMarker(store, {
910
- phase: "running",
911
- pid: process.pid,
912
- startedAt: (/* @__PURE__ */ new Date()).toISOString()
913
- });
914
- const report = await store.syncFromSage(createSageSurfaceSyncSource(surface));
915
- if (report.failed > 0) {
916
- log?.warn?.(
917
- `vector-memory sage sync finished with ${report.failed} failure(s) \u2014 will retry on next boot`
918
- );
919
- return {
920
- synced: false,
921
- reason: "partial-failure",
922
- marker: { phase: "running", ...counts(report) }
923
- };
924
- }
925
- let stats = store.stats();
926
- if (stats.vectors !== stats.entries) {
927
- log?.warn?.(
928
- `vector-memory sage sync healed ${stats.entries - stats.vectors} vector-less entr(ies) via reindexAll()`
929
- );
930
- await store.reindexAll();
931
- stats = store.stats();
812
+ // src/store.ts
813
+ import { createHash, randomUUID } from "node:crypto";
814
+ import * as fs3 from "node:fs";
815
+ import * as path3 from "node:path";
816
+ import { withFileLock } from "@wrongstack/core/utils";
817
+ import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
818
+ import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
819
+ var DEFAULT_SEARCH_LIMIT = 10;
820
+ function normalizeLimit(limit) {
821
+ if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
822
+ const floored = Math.floor(limit);
823
+ return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
824
+ }
825
+ var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
826
+ var DEFAULT_FILENAME = "vector-memory.db";
827
+ var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
828
+ var CACHE_EVICT_BATCH = 256;
829
+ var VectorMemoryStore = class _VectorMemoryStore {
830
+ db;
831
+ dbPath;
832
+ rootDir;
833
+ provider;
834
+ closed = false;
835
+ constructor(opts) {
836
+ if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
837
+ if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
838
+ this.provider = opts.provider;
839
+ const dir = opts.directory ?? DEFAULT_DIRECTORY;
840
+ const filename = opts.filename ?? DEFAULT_FILENAME;
841
+ if (path3.isAbsolute(dir)) {
842
+ throw new Error("Vector memory directory must be project-relative.");
932
843
  }
933
- if (stats.vectors !== stats.entries) {
934
- log?.warn?.(
935
- `vector-memory sage sync incomplete: ${stats.entries - stats.vectors} entr(ies) still vector-less \u2014 will retry on next boot`
936
- );
937
- return {
938
- synced: false,
939
- reason: "vector-incomplete",
940
- marker: { phase: "running", ...counts(report) }
941
- };
844
+ const rootDir = path3.resolve(opts.projectRoot, dir);
845
+ const rel = path3.relative(path3.resolve(opts.projectRoot), rootDir);
846
+ if (rel.startsWith("..") || path3.isAbsolute(rel)) {
847
+ throw new Error("Vector memory directory must stay inside the project root.");
942
848
  }
943
- const marker = {
944
- phase: "complete",
945
- completedAt: (/* @__PURE__ */ new Date()).toISOString(),
946
- ...provider ? { providerId: provider.id } : {},
947
- ...counts(report)
948
- };
949
- writeMarker(store, marker);
950
- log?.info?.(
951
- `vector-memory sage sync complete: ${report.indexed} indexed, ${report.skipped} skipped (already present)`
952
- );
953
- return { synced: true, reason: "synced", marker };
954
- } catch (error) {
955
- log?.warn?.(
956
- `vector-memory sage sync failed: ${error instanceof Error ? error.message : String(error)} \u2014 will retry on next boot`
957
- );
958
- return { synced: false, reason: "error" };
849
+ fs3.mkdirSync(rootDir, { recursive: true });
850
+ this.rootDir = rootDir;
851
+ this.dbPath = path3.join(rootDir, filename);
852
+ const Database = loadRuntimeDatabaseSync();
853
+ this.db = new Database(this.dbPath);
854
+ initVectorSchema(this.db);
855
+ this.recordActiveProvider();
959
856
  }
960
- }
961
- function decideWhetherToSync(store, staleAfterMs, now = /* @__PURE__ */ new Date(), pidAlive = defaultPidAlive) {
962
- const existing = readMarker(store);
963
- if (!existing) return { run: true, reason: "no-marker" };
964
- if (existing.phase === "complete") return { run: false, reason: "already-complete" };
965
- if (existing.pid === process.pid) {
966
- return { run: true, reason: "running-own-pid" };
857
+ /**
858
+ * Absolute path of the store's data directory (the resolved
859
+ * `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
860
+ * to place sidecar state (e.g. the first-boot SAGE sync marker) next to
861
+ * the db instead of re-deriving the path and drifting from it.
862
+ */
863
+ get directory() {
864
+ return this.rootDir;
967
865
  }
968
- const startedAt = existing.startedAt ? Date.parse(existing.startedAt) : Number.NaN;
969
- const ageMs = Number.isNaN(startedAt) ? Number.POSITIVE_INFINITY : now.getTime() - startedAt;
970
- const stale = ageMs > staleAfterMs;
971
- if (existing.pid === void 0) {
972
- return Number.isNaN(startedAt) || stale ? {
973
- run: true,
974
- reason: Number.isNaN(startedAt) ? "running-marker-undated" : "running-marker-stale"
975
- } : { run: false, reason: "running-unknown-pid" };
866
+ /**
867
+ * Absolute path of the SQLite database file. Hosts use this to take a
868
+ * file-level lock that covers all mutating operations (see
869
+ * `withFileLock(this.dbPath + '.lock', )`).
870
+ */
871
+ get databasePath() {
872
+ return this.dbPath;
976
873
  }
977
- try {
978
- if (pidAlive(existing.pid)) {
979
- return {
980
- run: false,
981
- reason: `running-pid-${existing.pid}${stale ? "-stale-but-alive" : ""}`
982
- };
874
+ /** The lockfile path used to serialize mutating operations. */
875
+ get lockPath() {
876
+ return `${this.dbPath}.lock`;
877
+ }
878
+ get activeProviderId() {
879
+ const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
880
+ return row?.value ?? this.provider.id;
881
+ }
882
+ recordActiveProvider() {
883
+ this.db.exec("BEGIN");
884
+ try {
885
+ this.db.prepare(
886
+ `INSERT INTO schema_meta (key, value) VALUES (?, ?)
887
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value`
888
+ ).run(VECTOR_PROVIDER_KEY, this.provider.id);
889
+ this.db.prepare(
890
+ `INSERT INTO schema_meta (key, value) VALUES (?, ?)
891
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value`
892
+ ).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
893
+ this.db.exec("COMMIT");
894
+ } catch (e) {
895
+ this.db.exec("ROLLBACK");
896
+ throw e;
983
897
  }
984
- return { run: true, reason: `running-pid-${existing.pid}-dead` };
985
- } catch {
986
- return { run: false, reason: `running-pid-${existing.pid}-probe-failed` };
987
898
  }
988
- }
989
- function defaultPidAlive(pid) {
990
- try {
991
- process.kill(pid, 0);
992
- return true;
993
- } catch (error) {
994
- return error.code === "EPERM";
899
+ static contentHash(text) {
900
+ return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
995
901
  }
996
- }
997
- function readMarker(store) {
998
- try {
999
- const raw = fs2.readFileSync(markerPath(store), "utf8");
1000
- const parsed = JSON.parse(raw);
1001
- return parsed && (parsed.phase === "running" || parsed.phase === "complete") ? parsed : void 0;
1002
- } catch {
1003
- return void 0;
902
+ /**
903
+ * Look up a vector for `text` in the provider-level embedding cache.
904
+ * Cache hit returns the cached vector (no ONNX pass). Miss returns
905
+ * `undefined`.
906
+ */
907
+ cachedVector(text, now) {
908
+ return lookupEmbeddingCache(
909
+ this.db,
910
+ _VectorMemoryStore.contentHash(text),
911
+ this.provider.id,
912
+ this.provider.dimensions,
913
+ now
914
+ );
1004
915
  }
1005
- }
1006
- function writeMarker(store, marker) {
1007
- try {
1008
- fs2.writeFileSync(markerPath(store), `${JSON.stringify(marker, null, 2)}
1009
- `, "utf8");
1010
- } catch {
916
+ /** Persist `vec` for `text` to the embedding cache. */
917
+ cacheVector(text, vec, now) {
918
+ upsertEmbeddingCache(this.db, {
919
+ contentHash: _VectorMemoryStore.contentHash(text),
920
+ providerId: this.provider.id,
921
+ dimensions: vec.length,
922
+ vector: encodeVector(vec),
923
+ text,
924
+ now
925
+ });
1011
926
  }
1012
- }
1013
- function markerPath(store) {
1014
- const dir = store.directory;
1015
- if (typeof dir !== "string") {
1016
- throw new Error("VectorMemoryStore.directory is required to place the sage sync marker");
927
+ /**
928
+ * Embed `text`, hitting the provider-level cache first. Cache miss falls
929
+ * through to the configured provider and writes the result back. Returns
930
+ * `undefined` when the provider fails — the caller can persist the entry
931
+ * without a vector (fail-open).
932
+ */
933
+ async embedWithCache(text) {
934
+ const now = (/* @__PURE__ */ new Date()).toISOString();
935
+ const cached = this.cachedVector(text, now);
936
+ if (cached) return cached;
937
+ try {
938
+ const result = await this.provider.embed([text]);
939
+ const vec = result[0];
940
+ if (vec) this.cacheVector(text, vec, now);
941
+ return vec;
942
+ } catch {
943
+ return void 0;
944
+ }
1017
945
  }
1018
- return path2.join(dir, SAGE_SYNC_MARKER_FILENAME);
1019
- }
1020
- function storeProvider(store) {
1021
- const provider = store.provider;
1022
- if (provider && typeof provider.id === "string") {
1023
- return provider;
946
+ /**
947
+ * Look up an existing entry by `content_hash`. Returns `undefined` when
948
+ * the entry is not present. Used by `remember()` to make writes idempotent
949
+ * and by `syncFromSage()` to skip already-indexed SAGE memories.
950
+ */
951
+ findByContentHash(contentHash) {
952
+ this.assertOpen();
953
+ const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
954
+ if (!row) return void 0;
955
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
956
+ return this.rowToEntry(row, vectorRow);
1024
957
  }
1025
- return void 0;
1026
- }
1027
- function counts(report) {
1028
- return {
1029
- scanned: report.scanned,
1030
- indexed: report.indexed,
1031
- skipped: report.skipped,
1032
- failed: report.failed
1033
- };
1034
- }
1035
-
1036
- // src/tools.ts
1037
- function createVectorMemoryTools(store) {
1038
- return [
1039
- vectorMemoryRememberTool(store),
1040
- vectorMemorySearchTool(store),
1041
- vectorMemoryStatsTool(store),
1042
- vectorMemoryForgetTool(store)
1043
- ];
1044
- }
1045
- function vectorMemoryRememberTool(store) {
1046
- return {
1047
- name: "vector_memory_remember",
1048
- category: "Memory",
1049
- description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
1050
- usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
1051
- permission: "confirm",
1052
- mutating: true,
1053
- riskTier: "standard",
1054
- timeoutMs: 5e3,
1055
- capabilities: ["memory.write"],
1056
- icon: "settings",
1057
- inputSchema: {
1058
- type: "object",
1059
- properties: {
1060
- text: { type: "string", minLength: 1, description: "The text to embed and store." },
1061
- summary: { type: "string", description: "Optional short label." },
1062
- tags: {
1063
- type: "array",
1064
- items: { type: "string" },
1065
- description: "Optional tags for later filtering."
1066
- },
1067
- scope: {
1068
- type: "string",
1069
- enum: ["project", "user", "session"],
1070
- description: "Visibility scope. Defaults to `project`."
1071
- },
1072
- kind: {
1073
- type: "string",
1074
- enum: ["note", "fact", "summary", "snippet", "link"],
1075
- description: "Entry kind. Defaults to `note`."
1076
- },
1077
- metadata: {
1078
- type: "object",
1079
- additionalProperties: true,
1080
- description: "Free-form metadata persisted as JSON."
1081
- }
1082
- },
1083
- required: ["text"],
1084
- additionalProperties: false
1085
- },
1086
- execute: async (input) => {
1087
- const entry = await store.remember(input);
1088
- return { id: entry.id, hasVector: entry.vector !== void 0 };
958
+ /**
959
+ * Look up the entry mirroring a given SAGE memory id (i.e. the entry
960
+ * whose `metadata.sageId` equals `sageId`). Returns `undefined` when
961
+ * no such entry exists. Used by the event-driven mirror to delete
962
+ * vector entries on SAGE delete events (the emitter knows the SAGE
963
+ * id, not the vector entry id).
964
+ *
965
+ * Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
966
+ * column is the JSON blob `syncFromSage` writes, so this avoids a
967
+ * full table scan.
968
+ */
969
+ findBySageId(sageId) {
970
+ this.assertOpen();
971
+ const row = this.db.prepare(
972
+ `SELECT * FROM entries
973
+ WHERE json_extract(metadata, '$.sageId') = ?
974
+ ORDER BY updated_at DESC
975
+ LIMIT 1`
976
+ ).get(sageId);
977
+ if (!row) return void 0;
978
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
979
+ return this.rowToEntry(row, vectorRow);
980
+ }
981
+ /**
982
+ * Persist a new entry. Idempotent: if an entry with the same
983
+ * `content_hash` already exists, that entry is returned unchanged
984
+ * instead of inserting a duplicate. Mutating ops are wrapped in
985
+ * `withFileLock` so two processes cannot race the dedup check.
986
+ */
987
+ async remember(input) {
988
+ this.assertOpen();
989
+ if (!input.text || input.text.trim().length === 0) {
990
+ throw new Error("VectorMemoryStore.remember: text must be non-empty");
1089
991
  }
1090
- };
1091
- }
1092
- function vectorMemorySearchTool(store) {
1093
- return {
1094
- name: "vector_memory_search",
1095
- category: "Memory",
1096
- description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
1097
- usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
1098
- permission: "auto",
1099
- mutating: false,
1100
- riskTier: "safe",
1101
- timeoutMs: 5e3,
1102
- capabilities: ["memory.read"],
1103
- icon: "search",
1104
- inputSchema: {
1105
- type: "object",
1106
- properties: {
1107
- query: { type: "string", minLength: 1, description: "The natural-language query." },
1108
- limit: {
1109
- type: "number",
1110
- minimum: 1,
1111
- maximum: 100,
1112
- description: "Max results (default 10)."
1113
- },
1114
- threshold: {
1115
- type: "number",
1116
- minimum: 0,
1117
- maximum: 1,
1118
- description: "Minimum cosine similarity. Results below the floor are dropped."
1119
- },
1120
- scope: {
1121
- type: "string",
1122
- enum: ["project", "user", "session"],
1123
- description: "Restrict to a scope."
1124
- },
1125
- kind: {
1126
- type: "string",
1127
- enum: ["note", "fact", "summary", "snippet", "link"],
1128
- description: "Restrict to a kind."
1129
- }
1130
- },
1131
- required: ["query"],
1132
- additionalProperties: false
1133
- },
1134
- execute: async (input) => {
1135
- const hits = await store.search(input.query, {
1136
- limit: input.limit !== void 0 ? input.limit : void 0,
1137
- threshold: input.threshold !== void 0 ? input.threshold : void 0,
1138
- scope: input.scope,
1139
- kind: input.kind
1140
- });
1141
- return {
1142
- hits: hits.map((h) => ({
1143
- id: h.entry.id,
1144
- score: h.score,
1145
- text: h.entry.text,
1146
- summary: h.entry.summary ?? void 0,
1147
- tags: h.entry.tags
1148
- }))
1149
- };
992
+ return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
993
+ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
994
+ });
995
+ }
996
+ async rememberUnlocked(input) {
997
+ const now = (/* @__PURE__ */ new Date()).toISOString();
998
+ const contentHash = _VectorMemoryStore.contentHash(input.text);
999
+ const existing = this.findByContentHash(contentHash);
1000
+ if (existing) return existing;
1001
+ const metadata = input.metadata ?? {};
1002
+ const tags = input.tags ?? [];
1003
+ const scope = input.scope ?? "project";
1004
+ const kind = input.kind ?? "note";
1005
+ const id = randomUUID();
1006
+ const vector = await this.embedWithCache(input.text);
1007
+ const providerId = vector ? this.provider.id : void 0;
1008
+ this.db.exec("BEGIN");
1009
+ try {
1010
+ this.db.prepare(
1011
+ `INSERT INTO entries
1012
+ (id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
1013
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
1014
+ ).run(
1015
+ id,
1016
+ input.text,
1017
+ input.summary ?? null,
1018
+ JSON.stringify(metadata),
1019
+ JSON.stringify(tags),
1020
+ scope,
1021
+ kind,
1022
+ contentHash,
1023
+ now,
1024
+ now
1025
+ );
1026
+ if (vector && providerId) {
1027
+ this.db.prepare(
1028
+ `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
1029
+ VALUES (?, ?, ?, ?, ?)
1030
+ ON CONFLICT(entry_id, provider_id) DO UPDATE SET
1031
+ vector = excluded.vector,
1032
+ dimensions = excluded.dimensions,
1033
+ created_at = excluded.created_at`
1034
+ ).run(id, providerId, vector.length, encodeVector(vector), now);
1035
+ }
1036
+ this.db.exec("COMMIT");
1037
+ } catch (e) {
1038
+ this.db.exec("ROLLBACK");
1039
+ throw e;
1150
1040
  }
1151
- };
1152
- }
1153
- function vectorMemoryStatsTool(store) {
1154
- return {
1155
- name: "vector_memory_stats",
1156
- category: "Memory",
1157
- description: "Return counts, providers, and dimensions for the local vector memory store.",
1158
- usageHint: "Cheap diagnostic \u2014 safe to call any time.",
1159
- permission: "auto",
1160
- mutating: false,
1161
- riskTier: "safe",
1162
- timeoutMs: 1e3,
1163
- capabilities: ["memory.read"],
1164
- icon: "search",
1165
- inputSchema: {
1166
- type: "object",
1167
- properties: {},
1168
- additionalProperties: false
1169
- },
1170
- execute: async () => store.stats()
1171
- };
1172
- }
1173
- function vectorMemoryForgetTool(store) {
1174
- return {
1175
- name: "vector_memory_forget",
1176
- category: "Memory",
1177
- description: "Remove an entry (and its vector) from the vector memory store.",
1178
- usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
1179
- permission: "confirm",
1180
- mutating: true,
1181
- riskTier: "standard",
1182
- timeoutMs: 1e3,
1183
- capabilities: ["memory.write"],
1184
- icon: "settings",
1185
- inputSchema: {
1186
- type: "object",
1187
- properties: {
1188
- id: {
1189
- type: "string",
1190
- minLength: 1,
1191
- description: "Entry id returned by `vector_memory_remember`."
1041
+ const result = {
1042
+ id,
1043
+ text: input.text,
1044
+ summary: input.summary ?? void 0,
1045
+ metadata,
1046
+ tags,
1047
+ scope,
1048
+ kind,
1049
+ contentHash,
1050
+ createdAt: now,
1051
+ updatedAt: now,
1052
+ providerId: providerId ?? "",
1053
+ dimensions: vector?.length ?? 0
1054
+ };
1055
+ if (vector) result.vector = vector;
1056
+ return result;
1057
+ }
1058
+ get(id) {
1059
+ this.assertOpen();
1060
+ const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
1061
+ if (!row) return void 0;
1062
+ const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
1063
+ return this.rowToEntry(row, vectorRow);
1064
+ }
1065
+ /** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
1066
+ async forget(id) {
1067
+ this.assertOpen();
1068
+ return withFileLock(
1069
+ this.lockPath,
1070
+ async () => {
1071
+ this.db.exec("BEGIN");
1072
+ try {
1073
+ const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
1074
+ this.db.exec("COMMIT");
1075
+ return info.changes > 0;
1076
+ } catch (e) {
1077
+ this.db.exec("ROLLBACK");
1078
+ throw e;
1192
1079
  }
1193
1080
  },
1194
- required: ["id"],
1195
- additionalProperties: false
1196
- },
1197
- execute: async (input) => ({ removed: await store.forget(input.id) })
1198
- };
1199
- }
1200
-
1201
- // src/sage-fusion.ts
1202
- var DEFAULT_RRF_K = 60;
1203
- var DEFAULT_VECTOR_WEIGHT = 0.3;
1204
- function lexicalRankScore(index, total) {
1205
- if (total <= 1) return 1;
1206
- return 1 - index / Math.max(1, total - 1);
1207
- }
1208
- async function fuseWithVectorMemory(query, lexical, options = {}) {
1209
- const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
1210
- const k = options.rrfK ?? DEFAULT_RRF_K;
1211
- const limit = options.limit ?? 25;
1212
- const vectorOnlyThreshold = options.vectorOnlyThreshold ?? 0;
1213
- let vectorHits = [];
1214
- if (options.vectorHits) {
1215
- vectorHits = [...options.vectorHits];
1216
- } else if (options.store) {
1217
- try {
1218
- vectorHits = await options.store.search(query, {
1219
- ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
1220
- limit: Math.max(limit * 2, 50)
1221
- });
1222
- } catch {
1223
- vectorHits = [];
1081
+ { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
1082
+ );
1083
+ }
1084
+ async search(query, opts = {}) {
1085
+ this.assertOpen();
1086
+ const limit = normalizeLimit(opts.limit);
1087
+ const threshold = opts.threshold ?? 0;
1088
+ const includeVectors = opts.includeVectors === true;
1089
+ if (typeof query !== "string" || query.trim().length === 0) return [];
1090
+ const queryVec = await this.embedWithCache(query);
1091
+ if (!queryVec || queryVec.length === 0) return [];
1092
+ const providerId = this.provider.id;
1093
+ const dimensions = this.provider.dimensions;
1094
+ const filters = ["v.provider_id = ?", "v.dimensions = ?"];
1095
+ const params = [providerId, dimensions];
1096
+ if (opts.scope !== void 0) {
1097
+ filters.push("e.scope = ?");
1098
+ params.push(opts.scope);
1099
+ }
1100
+ if (opts.kind !== void 0) {
1101
+ filters.push("e.kind = ?");
1102
+ params.push(opts.kind);
1103
+ }
1104
+ const scanRows = this.db.prepare(
1105
+ `SELECT e.id AS id, v.vector AS vec_blob
1106
+ FROM entries e
1107
+ JOIN vectors v ON v.entry_id = e.id
1108
+ WHERE ${filters.join(" AND ")}`
1109
+ ).all(...params);
1110
+ const top = [];
1111
+ for (const row of scanRows) {
1112
+ const vec = decodeVector(row.vec_blob);
1113
+ const raw = cosineSimilarity(queryVec, vec);
1114
+ const score = Math.max(0, Math.min(1, raw));
1115
+ if (!Number.isFinite(score) || score < threshold) continue;
1116
+ if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
1117
+ let at = top.length;
1118
+ while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
1119
+ top.splice(at, 0, { id: row.id, score, vector: vec });
1120
+ if (top.length > limit) top.length = limit;
1121
+ }
1122
+ if (top.length === 0) return [];
1123
+ const placeholders = top.map(() => "?").join(",");
1124
+ const hydrated = this.db.prepare(
1125
+ `SELECT id, text, summary, metadata, tags, scope, kind,
1126
+ content_hash, created_at, updated_at
1127
+ FROM entries WHERE id IN (${placeholders})`
1128
+ ).all(...top.map((t) => t.id));
1129
+ const entryById = /* @__PURE__ */ new Map();
1130
+ for (const row of hydrated) {
1131
+ entryById.set(row.id, this.rowToEntry(row));
1132
+ }
1133
+ const scored = [];
1134
+ for (const candidate of top) {
1135
+ const entry = entryById.get(candidate.id);
1136
+ if (!entry) continue;
1137
+ const hit = { entry, score: candidate.score, providerId };
1138
+ if (includeVectors) hit.vector = candidate.vector;
1139
+ scored.push(hit);
1224
1140
  }
1141
+ return scored;
1225
1142
  }
1226
- const sageById = /* @__PURE__ */ new Map();
1227
- for (const memory of lexical) sageById.set(memory.id, memory);
1228
- const lexicalRanked = lexical.map((memory, index) => ({
1229
- memory,
1230
- rankScore: lexicalRankScore(index, lexical.length),
1231
- lexicalScore: lexicalRankScore(index, lexical.length)
1232
- }));
1233
- const vectorRanked = [];
1234
- const seenVectorSageIds = /* @__PURE__ */ new Set();
1235
- for (let i = 0; i < vectorHits.length; i++) {
1236
- const hit = vectorHits[i];
1237
- const sageId = hit.entry.metadata?.["sageId"];
1238
- if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
1239
- const memory = sageById.get(sageId);
1240
- if (!memory) continue;
1241
- seenVectorSageIds.add(sageId);
1242
- vectorRanked.push({ memory, vectorScore: hit.score });
1143
+ /**
1144
+ * Page through entries, newest first.
1145
+ *
1146
+ * Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
1147
+ * without the id tiebreak two entries written in the same millisecond can
1148
+ * swap places between calls and a paging caller silently skips one.
1149
+ *
1150
+ * Pagination is keyset (`after`), not offset, because the only caller that
1151
+ * pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
1152
+ * `OFFSET` every deletion shifts the remaining rows left and the next page
1153
+ * skips exactly as many entries as were removed. Keyset is immune: it
1154
+ * resumes from a position, and the rows a deletion removes are ones the
1155
+ * sweep has already passed.
1156
+ */
1157
+ list(opts = {}) {
1158
+ this.assertOpen();
1159
+ const where = [];
1160
+ const params = [];
1161
+ if (opts.scope !== void 0) {
1162
+ where.push("scope = ?");
1163
+ params.push(opts.scope);
1164
+ }
1165
+ if (opts.kind !== void 0) {
1166
+ where.push("kind = ?");
1167
+ params.push(opts.kind);
1168
+ }
1169
+ if (opts.after) {
1170
+ where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
1171
+ params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
1172
+ }
1173
+ const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
1174
+ ORDER BY updated_at DESC, id DESC LIMIT ?`;
1175
+ params.push(opts.limit ?? 100);
1176
+ const rows = this.db.prepare(sql).all(...params);
1177
+ return rows.map((r) => this.rowToEntry(r));
1178
+ }
1179
+ async reindexAll() {
1180
+ this.assertOpen();
1181
+ return withFileLock(
1182
+ this.lockPath,
1183
+ async () => {
1184
+ const rows = this.db.prepare("SELECT id, text FROM entries").all();
1185
+ let processed = 0;
1186
+ let errors = 0;
1187
+ for (const row of rows) {
1188
+ try {
1189
+ const result = await this.provider.embed([row.text]);
1190
+ const v = result[0];
1191
+ if (!v) {
1192
+ errors++;
1193
+ continue;
1194
+ }
1195
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1196
+ this.db.prepare(
1197
+ `INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
1198
+ VALUES (?, ?, ?, ?, ?)
1199
+ ON CONFLICT(entry_id, provider_id) DO UPDATE SET
1200
+ vector = excluded.vector,
1201
+ dimensions = excluded.dimensions,
1202
+ created_at = excluded.created_at`
1203
+ ).run(row.id, this.provider.id, v.length, encodeVector(v), now);
1204
+ this.cacheVector(row.text, v, now);
1205
+ processed++;
1206
+ } catch {
1207
+ errors++;
1208
+ }
1209
+ }
1210
+ return { processed, errors };
1211
+ },
1212
+ { timeoutMs: 6e4 }
1213
+ );
1243
1214
  }
1244
- const fused = /* @__PURE__ */ new Map();
1245
- for (let i = 0; i < lexicalRanked.length; i++) {
1246
- const c = lexicalRanked[i];
1247
- const rrf = (1 - weight) * (1 / (k + i + 1));
1248
- fused.set(c.memory.id, {
1249
- memory: c.memory,
1250
- vectorScore: null,
1251
- lexicalScore: c.lexicalScore,
1252
- finalScore: rrf,
1253
- source: "lexical"
1254
- });
1215
+ stats() {
1216
+ this.assertOpen();
1217
+ const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
1218
+ const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
1219
+ const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
1220
+ return {
1221
+ entries: entryCount,
1222
+ vectors: vectorCount,
1223
+ providers: providerRows.map((r) => r.provider_id),
1224
+ modelAvailable: true,
1225
+ modelId: this.provider.id,
1226
+ dimensions: this.provider.dimensions
1227
+ };
1255
1228
  }
1256
- for (let i = 0; i < vectorRanked.length; i++) {
1257
- const v = vectorRanked[i];
1258
- const rrf = weight * (1 / (k + i + 1));
1259
- const existing = fused.get(v.memory.id);
1260
- if (existing) {
1261
- existing.finalScore += rrf;
1262
- existing.vectorScore = v.vectorScore;
1263
- existing.source = "both";
1264
- } else if (v.vectorScore >= vectorOnlyThreshold) {
1265
- fused.set(v.memory.id, {
1266
- memory: v.memory,
1267
- vectorScore: v.vectorScore,
1268
- lexicalScore: null,
1269
- finalScore: rrf,
1270
- source: "vector"
1271
- });
1272
- }
1229
+ /**
1230
+ * Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
1231
+ * Useful for the WebUI's vector-memory panel and for diagnosing the
1232
+ * "why is search slow?" question.
1233
+ */
1234
+ cacheStats() {
1235
+ this.assertOpen();
1236
+ const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
1237
+ const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
1238
+ const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
1239
+ const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
1240
+ return {
1241
+ entries,
1242
+ providers,
1243
+ totalUseCount,
1244
+ oldestLastUsedAt: oldest?.t ?? null
1245
+ };
1273
1246
  }
1274
- const out = Array.from(fused.values());
1275
- out.sort((a, b) => b.finalScore - a.finalScore);
1276
- return out.slice(0, limit);
1277
- }
1278
- function asVectorRecallProvider(store) {
1279
- return {
1280
- async search(query, opts) {
1281
- const hits = await store.search(query, {
1282
- limit: opts.limit,
1283
- ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
1284
- });
1285
- return hits.map((h) => ({
1286
- id: h.entry.id,
1287
- score: h.score,
1288
- text: h.entry.text,
1289
- ...h.entry.summary ? { summary: h.entry.summary } : {},
1290
- tags: h.entry.tags,
1291
- ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
1292
- }));
1293
- }
1294
- };
1295
- }
1296
- function clamp01(value) {
1297
- if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
1298
- if (value < 0) return 0;
1299
- if (value > 1) return 1;
1300
- return value;
1301
- }
1302
-
1303
- // src/sage-port-wrapper.ts
1304
- import {
1305
- augmentLexicalWithVectorRecall,
1306
- isSageVisibleForSearch,
1307
- SAGE_RETRIEVAL_CAPABILITY,
1308
- SAGE_SURFACE_CAPABILITY
1309
- } from "@wrongstack/sage";
1310
- function asVectorRecallProviderAdapter(store) {
1311
- return {
1312
- async search(query, opts) {
1313
- const hits = await store.search(query, {
1314
- limit: opts.limit,
1315
- ...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
1316
- });
1317
- return hits.map((h) => ({
1318
- id: h.entry.id,
1319
- score: h.score,
1320
- text: h.entry.text,
1321
- ...h.entry.summary ? { summary: h.entry.summary } : {},
1322
- tags: h.entry.tags,
1323
- ...h.entry.metadata ? { metadata: h.entry.metadata } : {}
1324
- }));
1247
+ /**
1248
+ * LRU-evict the embedding cache down to `keepMostRecent` rows. The
1249
+ * `embedding_cache` table is independent of `entries`, so a sweep here
1250
+ * only removes cached vectors — never stored entries. Called by hosts
1251
+ * that want to bound cache growth on long-lived processes.
1252
+ */
1253
+ async evictCache(keepMostRecent) {
1254
+ this.assertOpen();
1255
+ if (keepMostRecent < 0) {
1256
+ throw new Error("evictCache: keepMostRecent must be >= 0");
1325
1257
  }
1326
- };
1327
- }
1328
- function wrapMemoryPortWithVectorRecall(port, options) {
1329
- const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
1330
- const materializeFor = (searchOpts) => async (sageId) => {
1331
- const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
1332
- if (!surface?.getSage) return void 0;
1333
- const memory = await surface.getSage(sageId);
1334
- if (!memory) return void 0;
1335
- return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
1336
- };
1337
- const fusionOptions = (searchOpts) => ({
1338
- vectorRecall: recall,
1339
- materializeVectorOnly: materializeFor(searchOpts),
1340
- ...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
1341
- ...options.threshold !== void 0 ? { threshold: options.threshold } : {},
1342
- ...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
1343
- ...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
1344
- ...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
1345
- });
1346
- const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
1347
- const wrapSearchSage = (original) => async (query, searchOpts) => {
1348
- const opts = searchOpts;
1349
- const lexical = await original(query, searchOpts);
1350
- if (callerOwnsFusion(opts)) return lexical;
1351
- const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
1352
- return fused.map((hit) => hit.memory);
1353
- };
1354
- const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
1355
- const opts = searchOpts;
1356
- const lexicalHits = await original(query, searchOpts);
1357
- if (callerOwnsFusion(opts)) return lexicalHits;
1358
- return augmentLexicalWithVectorRecall(
1359
- query,
1360
- lexicalHits.map((hit) => hit.memory),
1361
- fusionOptions(opts)
1258
+ return withFileLock(
1259
+ this.lockPath,
1260
+ async () => {
1261
+ const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
1262
+ if (total <= keepMostRecent) return { removed: 0 };
1263
+ const toRemove = total - keepMostRecent;
1264
+ const stmt = this.db.prepare(
1265
+ `DELETE FROM embedding_cache
1266
+ WHERE content_hash IN (
1267
+ SELECT content_hash FROM embedding_cache
1268
+ ORDER BY last_used_at ASC
1269
+ LIMIT ?
1270
+ )`
1271
+ );
1272
+ const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
1273
+ return { removed: Number(info.changes) };
1274
+ },
1275
+ { timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
1362
1276
  );
1363
- };
1364
- const wrapped = Object.create(
1365
- Object.getPrototypeOf(port),
1366
- Object.getOwnPropertyDescriptors(port)
1367
- );
1368
- wrapped.getCapability = (capability) => {
1369
- if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
1370
- const original = port.getCapability(capability);
1371
- if (!original) return void 0;
1372
- return {
1373
- ...original,
1374
- searchSage: wrapSearchSage(original.searchSage),
1375
- ...original.searchSageWithBreakdown ? {
1376
- searchSageWithBreakdown: wrapSearchWithBreakdown(
1377
- original.searchSageWithBreakdown
1378
- )
1379
- } : {}
1380
- };
1381
- }
1382
- if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
1383
- const original = port.getCapability(capability);
1384
- if (!original) return void 0;
1385
- return {
1386
- ...original,
1387
- searchSage: wrapSearchSage(original.searchSage),
1388
- ...original.searchSageWithBreakdown ? {
1389
- searchSageWithBreakdown: wrapSearchWithBreakdown(
1390
- original.searchSageWithBreakdown
1391
- )
1392
- } : {}
1393
- };
1394
- }
1395
- return port.getCapability(capability);
1396
- };
1397
- return wrapped;
1398
- }
1399
-
1400
- // src/sage-event-mirror.ts
1401
- import * as fs3 from "node:fs";
1402
- import * as path3 from "node:path";
1403
- import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
1404
- function subscribeVectorMemoryToSage(opts) {
1405
- const { store, memoryStore } = opts;
1406
- const log = opts.logger;
1407
- if (opts.enabled === false) {
1408
- return { dispose: () => void 0 };
1409
1277
  }
1410
- const surface = getSageSurface2(memoryStore);
1411
- if (!surface) {
1412
- log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
1413
- return { dispose: () => void 0 };
1278
+ close() {
1279
+ if (this.closed) return;
1280
+ this.closed = true;
1281
+ this.db.close();
1282
+ }
1283
+ assertOpen() {
1284
+ if (this.closed) throw new Error("VectorMemoryStore is closed");
1285
+ }
1286
+ rowToEntry(row, vectorRow) {
1287
+ const summaryValue = row.summary;
1288
+ const entry = {
1289
+ id: row.id,
1290
+ text: row.text,
1291
+ summary: summaryValue ?? void 0,
1292
+ metadata: safeParseJson(row.metadata, {}),
1293
+ tags: safeParseJson(row.tags, []),
1294
+ scope: row.scope,
1295
+ kind: row.kind,
1296
+ contentHash: row.content_hash,
1297
+ createdAt: row.created_at,
1298
+ updatedAt: row.updated_at,
1299
+ providerId: vectorRow?.provider_id ?? "",
1300
+ dimensions: vectorRow?.dimensions ?? 0
1301
+ };
1302
+ if (vectorRow?.vector) {
1303
+ entry.vector = decodeVector(vectorRow.vector);
1304
+ }
1305
+ return entry;
1306
+ }
1307
+ async syncFromSage(sage) {
1308
+ this.assertOpen();
1309
+ const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
1310
+ let indexed = 0;
1311
+ let skipped = 0;
1312
+ let failed = 0;
1313
+ const errors = [];
1314
+ for (const memory of memories) {
1315
+ try {
1316
+ const hash = _VectorMemoryStore.contentHash(memory.text);
1317
+ const existing = this.findByContentHash(hash);
1318
+ if (existing) {
1319
+ skipped++;
1320
+ continue;
1321
+ }
1322
+ await this.rememberUnlocked({
1323
+ text: memory.text,
1324
+ summary: memory.summary ?? void 0,
1325
+ metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
1326
+ tags: memory.tags ?? [],
1327
+ scope: "project",
1328
+ kind: "note"
1329
+ });
1330
+ indexed++;
1331
+ } catch (err) {
1332
+ failed++;
1333
+ errors.push({ memoryId: memory.id, message: errMsg2(err) });
1334
+ }
1335
+ }
1336
+ return { scanned: memories.length, indexed, skipped, failed, errors };
1414
1337
  }
1415
- const events = memoryStore.events;
1416
- if (!events) {
1417
- log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
1418
- return { dispose: () => void 0 };
1338
+ };
1339
+ function fallbackHashingProvider(dimensions) {
1340
+ return new HashingEmbeddingProvider({ dimensions });
1341
+ }
1342
+ function safeParseJson(value, fallback) {
1343
+ if (typeof value !== "string") return fallback;
1344
+ try {
1345
+ return JSON.parse(value);
1346
+ } catch {
1347
+ return fallback;
1419
1348
  }
1420
- const fetch = async (memoryId) => {
1421
- try {
1422
- return await surface.getSage(memoryId);
1423
- } catch (err) {
1424
- log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg2(err)}`);
1425
- return null;
1349
+ }
1350
+ function errMsg2(err) {
1351
+ return err instanceof Error ? err.message : String(err);
1352
+ }
1353
+
1354
+ // src/tools.ts
1355
+ function createVectorMemoryTools(store) {
1356
+ return [
1357
+ vectorMemoryRememberTool(store),
1358
+ vectorMemorySearchTool(store),
1359
+ vectorMemoryStatsTool(store),
1360
+ vectorMemoryForgetTool(store)
1361
+ ];
1362
+ }
1363
+ function vectorMemoryRememberTool(store) {
1364
+ return {
1365
+ name: "vector_memory_remember",
1366
+ category: "Memory",
1367
+ description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
1368
+ usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
1369
+ permission: "confirm",
1370
+ mutating: true,
1371
+ riskTier: "standard",
1372
+ timeoutMs: 5e3,
1373
+ capabilities: ["memory.write"],
1374
+ icon: "settings",
1375
+ inputSchema: {
1376
+ type: "object",
1377
+ properties: {
1378
+ text: { type: "string", minLength: 1, description: "The text to embed and store." },
1379
+ summary: { type: "string", description: "Optional short label." },
1380
+ tags: {
1381
+ type: "array",
1382
+ items: { type: "string" },
1383
+ description: "Optional tags for later filtering."
1384
+ },
1385
+ scope: {
1386
+ type: "string",
1387
+ enum: ["project", "user", "session"],
1388
+ description: "Visibility scope. Defaults to `project`."
1389
+ },
1390
+ kind: {
1391
+ type: "string",
1392
+ enum: ["note", "fact", "summary", "snippet", "link"],
1393
+ description: "Entry kind. Defaults to `note`."
1394
+ },
1395
+ metadata: {
1396
+ type: "object",
1397
+ additionalProperties: true,
1398
+ description: "Free-form metadata persisted as JSON."
1399
+ }
1400
+ },
1401
+ required: ["text"],
1402
+ additionalProperties: false
1403
+ },
1404
+ execute: async (input) => {
1405
+ const entry = await store.remember(input);
1406
+ return { id: entry.id, hasVector: entry.vector !== void 0 };
1426
1407
  }
1427
1408
  };
1428
- const mirror = async (memoryId) => {
1429
- const memory = await fetch(memoryId);
1430
- if (!memory) return;
1431
- if (memory.scope === "session") return;
1432
- try {
1433
- const existing = store.findBySageId(memoryId);
1434
- if (existing) await store.forget(existing.id);
1435
- await store.remember({
1436
- text: memory.text,
1437
- ...memory.summary ? { summary: memory.summary } : {},
1438
- tags: memory.tags ?? [],
1439
- scope: "project",
1440
- kind: "note",
1441
- metadata: {
1442
- source: "sage",
1443
- sageId: memory.id,
1444
- sageKind: memory.kind,
1445
- sageScope: memory.scope,
1446
- importance: memory.importance,
1447
- confidence: memory.confidence
1409
+ }
1410
+ function vectorMemorySearchTool(store) {
1411
+ return {
1412
+ name: "vector_memory_search",
1413
+ category: "Memory",
1414
+ description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
1415
+ usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
1416
+ permission: "auto",
1417
+ mutating: false,
1418
+ riskTier: "safe",
1419
+ timeoutMs: 5e3,
1420
+ capabilities: ["memory.read"],
1421
+ icon: "search",
1422
+ inputSchema: {
1423
+ type: "object",
1424
+ properties: {
1425
+ query: { type: "string", minLength: 1, description: "The natural-language query." },
1426
+ limit: {
1427
+ type: "number",
1428
+ minimum: 1,
1429
+ maximum: 100,
1430
+ description: "Max results (default 10)."
1431
+ },
1432
+ threshold: {
1433
+ type: "number",
1434
+ minimum: 0,
1435
+ maximum: 1,
1436
+ description: "Minimum cosine similarity. Results below the floor are dropped."
1437
+ },
1438
+ scope: {
1439
+ type: "string",
1440
+ enum: ["project", "user", "session"],
1441
+ description: "Restrict to a scope."
1442
+ },
1443
+ kind: {
1444
+ type: "string",
1445
+ enum: ["note", "fact", "summary", "snippet", "link"],
1446
+ description: "Restrict to a kind."
1448
1447
  }
1448
+ },
1449
+ required: ["query"],
1450
+ additionalProperties: false
1451
+ },
1452
+ execute: async (input) => {
1453
+ const hits = await store.search(input.query, {
1454
+ limit: input.limit !== void 0 ? input.limit : void 0,
1455
+ threshold: input.threshold !== void 0 ? input.threshold : void 0,
1456
+ scope: input.scope,
1457
+ kind: input.kind
1449
1458
  });
1450
- } catch (err) {
1451
- log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg2(err)}`);
1459
+ return {
1460
+ hits: hits.map((h) => ({
1461
+ id: h.entry.id,
1462
+ score: h.score,
1463
+ text: h.entry.text,
1464
+ summary: h.entry.summary ?? void 0,
1465
+ tags: h.entry.tags
1466
+ }))
1467
+ };
1452
1468
  }
1453
1469
  };
1454
- const forgetMirror = async (memoryId) => {
1455
- try {
1456
- const existing = store.findBySageId(memoryId);
1457
- if (existing) await store.forget(existing.id);
1458
- } catch (err) {
1459
- log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg2(err)}`);
1460
- }
1470
+ }
1471
+ function vectorMemoryStatsTool(store) {
1472
+ return {
1473
+ name: "vector_memory_stats",
1474
+ category: "Memory",
1475
+ description: "Return counts, providers, and dimensions for the local vector memory store.",
1476
+ usageHint: "Cheap diagnostic \u2014 safe to call any time.",
1477
+ permission: "auto",
1478
+ mutating: false,
1479
+ riskTier: "safe",
1480
+ timeoutMs: 1e3,
1481
+ capabilities: ["memory.read"],
1482
+ icon: "search",
1483
+ inputSchema: {
1484
+ type: "object",
1485
+ properties: {},
1486
+ additionalProperties: false
1487
+ },
1488
+ execute: async () => store.stats()
1461
1489
  };
1462
- const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
1463
- const memoryId = payload?.memoryId;
1464
- if (typeof memoryId !== "string") return;
1465
- void mirror(memoryId);
1466
- });
1467
- const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
1468
- const memoryId = payload?.memoryId;
1469
- if (typeof memoryId !== "string") return;
1470
- void mirror(memoryId);
1471
- });
1472
- const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
1473
- const memoryId = payload?.memoryId;
1474
- if (typeof memoryId !== "string") return;
1475
- const status = payload?.status;
1476
- if (status === "deleted") return;
1477
- void mirror(memoryId);
1478
- });
1479
- const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
1480
- const memoryId = payload?.memoryId;
1481
- if (typeof memoryId !== "string") return;
1482
- void forgetMirror(memoryId);
1483
- });
1490
+ }
1491
+ function vectorMemoryForgetTool(store) {
1484
1492
  return {
1485
- dispose: () => {
1486
- offAccepted();
1487
- offRecovered();
1488
- offUpdated();
1489
- offDeleted();
1490
- }
1493
+ name: "vector_memory_forget",
1494
+ category: "Memory",
1495
+ description: "Remove an entry (and its vector) from the vector memory store.",
1496
+ usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
1497
+ permission: "confirm",
1498
+ mutating: true,
1499
+ riskTier: "standard",
1500
+ timeoutMs: 1e3,
1501
+ capabilities: ["memory.write"],
1502
+ icon: "settings",
1503
+ inputSchema: {
1504
+ type: "object",
1505
+ properties: {
1506
+ id: {
1507
+ type: "string",
1508
+ minLength: 1,
1509
+ description: "Entry id returned by `vector_memory_remember`."
1510
+ }
1511
+ },
1512
+ required: ["id"],
1513
+ additionalProperties: false
1514
+ },
1515
+ execute: async (input) => ({ removed: await store.forget(input.id) })
1491
1516
  };
1492
1517
  }
1493
- async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
1494
- const surface = getSageSurface2(memoryStore);
1495
- if (!surface) return { scanned: 0, removed: 0 };
1496
- let scanned = 0;
1497
- let removed = 0;
1498
- const PAGE = Math.max(1, options?.pageSize ?? 500);
1499
- let after;
1500
- for (; ; ) {
1501
- const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
1502
- if (page.length === 0) break;
1503
- const last = page[page.length - 1];
1504
- after = { updatedAt: last.updatedAt, id: last.id };
1505
- for (const entry of page) {
1506
- scanned++;
1507
- const sageId = entry.metadata?.sageId;
1508
- if (typeof sageId !== "string") continue;
1509
- try {
1510
- const memory = await surface.getSage(sageId);
1511
- if (memory !== null && memory.status !== "deleted") continue;
1512
- await store.forget(entry.id);
1513
- removed++;
1514
- } catch (err) {
1515
- logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg2(err)}`);
1516
- }
1517
- }
1518
- if (page.length < PAGE) break;
1518
+
1519
+ // src/transformers-provider.ts
1520
+ var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
1521
+ var DEFAULT_VECTOR_DIMENSIONS = 384;
1522
+ var DEFAULT_VECTOR_DTYPE = "q8";
1523
+ var TransformersEmbeddingProvider = class {
1524
+ id;
1525
+ dimensions;
1526
+ modelId;
1527
+ cacheDir;
1528
+ dtype;
1529
+ device;
1530
+ batchSize;
1531
+ maxChars;
1532
+ allowRemote;
1533
+ extractor;
1534
+ loadPromise;
1535
+ constructor(opts = {}) {
1536
+ this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
1537
+ this.cacheDir = opts.cacheDir;
1538
+ this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
1539
+ this.device = opts.device ?? "cpu";
1540
+ this.batchSize = opts.batchSize ?? 16;
1541
+ this.maxChars = opts.maxChars ?? 2048;
1542
+ this.allowRemote = opts.allowRemoteModels ?? true;
1543
+ this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
1544
+ this.id = `transformers-js:${this.modelId}:${this.dtype}`;
1519
1545
  }
1520
- return { scanned, removed };
1521
- }
1522
- function errMsg2(err) {
1523
- return err instanceof Error ? err.message : String(err);
1524
- }
1525
- var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
1526
- var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
1527
- async function sweepStaleSageMirrors(opts) {
1528
- const markerPath2 = path3.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
1529
- const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
1530
- if (!opts.force) {
1546
+ /**
1547
+ * Synchronous capability check. Returns false when the optional
1548
+ * `@huggingface/transformers` dependency is not installed.
1549
+ *
1550
+ * NOTE: this probes via dynamic import and caches the result, but does
1551
+ * NOT load the model itself — model loading is deferred to `embed()`.
1552
+ */
1553
+ async isAvailable() {
1531
1554
  try {
1532
- const raw = JSON.parse(fs3.readFileSync(markerPath2, "utf8"));
1533
- const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
1534
- if (Number.isFinite(at) && Date.now() - at < interval) {
1535
- return { swept: false, reason: "throttled" };
1536
- }
1555
+ await this.loadModule();
1556
+ return true;
1537
1557
  } catch {
1558
+ return false;
1538
1559
  }
1539
1560
  }
1540
- try {
1541
- fs3.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
1542
- } catch {
1543
- }
1544
- try {
1545
- const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
1546
- opts.logger?.debug?.(
1547
- `vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
1548
- );
1549
- return { swept: true, ...result };
1550
- } catch (err) {
1551
- opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg2(err)}`);
1552
- return { swept: false, reason: errMsg2(err) };
1553
- }
1554
- }
1555
-
1556
- // src/search-race.ts
1557
- function previewText(text, maxLen) {
1558
- if (text.length <= maxLen) return text;
1559
- return text.slice(0, maxLen - 1) + "\u2026";
1560
- }
1561
- async function runSearchRace(query, lexical, vectorStore, options = {}) {
1562
- const limit = options.limit ?? 20;
1563
- const threshold = options.threshold ?? 0;
1564
- let vectorHits = [];
1565
- try {
1566
- vectorHits = await vectorStore.search(query, {
1567
- limit,
1568
- ...threshold > 0 ? { threshold } : {}
1569
- });
1570
- } catch {
1571
- vectorHits = [];
1561
+ async embed(texts) {
1562
+ if (texts.length === 0) return [];
1563
+ const extractor = await this.getExtractor();
1564
+ const prepared = texts.map((t) => this.prepare(t));
1565
+ const batches = [];
1566
+ for (let i = 0; i < prepared.length; i += this.batchSize) {
1567
+ batches.push(prepared.slice(i, i + this.batchSize));
1568
+ }
1569
+ const results = [];
1570
+ for (const batch of batches) {
1571
+ const out = await extractor(batch, { pooling: "mean", normalize: true });
1572
+ results.push(...this.tensorToVectors(out, batch.length));
1573
+ }
1574
+ return results;
1572
1575
  }
1573
- const lexicalOnly = [];
1574
- const vectorOnly = [];
1575
- const overlap = [];
1576
- const seenIds = /* @__PURE__ */ new Set();
1577
- const lexicalCapped = lexical.slice(0, limit);
1578
- for (let i = 0; i < lexicalCapped.length; i++) {
1579
- const mem = lexicalCapped[i];
1580
- const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
1581
- const id = mem.id;
1582
- seenIds.add(id);
1583
- overlap.push({
1584
- id,
1585
- lexicalScore: score,
1586
- vectorScore: null,
1587
- // patched below when a vector hit carries this id
1588
- preview: previewText(mem.text, 140)
1589
- });
1576
+ /** Truncate + normalize text before embedding. */
1577
+ prepare(text) {
1578
+ if (!text) return "";
1579
+ const normalized = text.normalize("NFKC").trim();
1580
+ return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
1590
1581
  }
1591
- const overlapById = /* @__PURE__ */ new Map();
1592
- for (const row of overlap) overlapById.set(row.id, row);
1593
- for (const hit of vectorHits) {
1594
- const sageId = hit.entry.metadata?.["sageId"];
1595
- if (typeof sageId !== "string") continue;
1596
- const existing = overlapById.get(sageId);
1597
- if (existing) {
1598
- if (existing.vectorScore === null) {
1599
- existing.vectorScore = hit.score;
1582
+ tensorToVectors(out, batchSize) {
1583
+ if (typeof out.tolist === "function") {
1584
+ const nested = out.tolist();
1585
+ if (Array.isArray(nested) && Array.isArray(nested[0])) {
1586
+ return nested.map((row) => Float32Array.from(row));
1600
1587
  }
1601
- continue;
1588
+ return [Float32Array.from(nested)];
1602
1589
  }
1603
- if (seenIds.has(sageId)) continue;
1604
- seenIds.add(sageId);
1605
- vectorOnly.push({
1606
- id: sageId,
1607
- lexicalScore: null,
1608
- vectorScore: hit.score,
1609
- preview: previewText(hit.entry.text, 140)
1590
+ const flat = out.data;
1591
+ if (flat instanceof Float32Array) {
1592
+ if (batchSize === 1) return [flat];
1593
+ const dim = flat.length / batchSize;
1594
+ const vectors = [];
1595
+ for (let i = 0; i < batchSize; i++) {
1596
+ vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
1597
+ }
1598
+ return vectors;
1599
+ }
1600
+ if (Array.isArray(flat) && Array.isArray(flat[0])) {
1601
+ return flat.map((row) => Float32Array.from(row));
1602
+ }
1603
+ if (Array.isArray(flat)) {
1604
+ return [Float32Array.from(flat)];
1605
+ }
1606
+ throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
1607
+ }
1608
+ async getExtractor() {
1609
+ if (this.extractor) return this.extractor;
1610
+ if (!this.loadPromise) this.loadPromise = this.loadExtractor();
1611
+ this.extractor = await this.loadPromise;
1612
+ return this.extractor;
1613
+ }
1614
+ async loadExtractor() {
1615
+ const mod = await this.loadModule();
1616
+ if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
1617
+ mod.env.allowRemoteModels = this.allowRemote;
1618
+ if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
1619
+ const pipe = await mod.pipeline("feature-extraction", this.modelId, {
1620
+ dtype: this.dtype,
1621
+ device: this.device
1610
1622
  });
1623
+ return pipe;
1611
1624
  }
1612
- const finalOverlap = [];
1613
- for (const row of overlap) {
1614
- if (row.vectorScore === null) {
1615
- lexicalOnly.push({
1616
- id: row.id,
1617
- lexicalScore: row.lexicalScore,
1618
- vectorScore: null,
1619
- preview: row.preview
1620
- });
1621
- } else {
1622
- finalOverlap.push(row);
1625
+ async loadModule() {
1626
+ try {
1627
+ return await import("@huggingface/transformers");
1628
+ } catch (err) {
1629
+ throw new VectorMemoryProviderUnavailableError(
1630
+ "@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
1631
+ err
1632
+ );
1623
1633
  }
1624
1634
  }
1625
- const lexicalCount = lexicalOnly.length + finalOverlap.length;
1626
- const vectorCount = vectorOnly.length + finalOverlap.length;
1627
- const denom = Math.max(lexicalCount, vectorCount, 1);
1628
- return {
1629
- query,
1630
- lexicalOnly,
1631
- vectorOnly,
1632
- overlap: finalOverlap,
1633
- metrics: {
1634
- lexicalCount,
1635
- vectorCount,
1636
- overlapCount: finalOverlap.length,
1637
- lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
1638
- vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
1639
- agreementRatio: finalOverlap.length / denom
1640
- }
1641
- };
1642
- }
1635
+ };
1643
1636
  export {
1644
1637
  DEFAULT_SWEEP_INTERVAL_MS,
1645
1638
  DEFAULT_VECTOR_DIMENSIONS,