@wrongstack/vector-memory 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/index.d.ts +9 -9
- package/dist/index.js +1323 -1329
- package/dist/sage-sync.d.ts +30 -0
- package/dist/transformers-provider.d.ts +2 -2
- package/package.json +5 -8
package/dist/index.js
CHANGED
|
@@ -14,796 +14,357 @@ var VectorMemoryProviderUnavailableError = class extends VectorMemoryError {
|
|
|
14
14
|
}
|
|
15
15
|
};
|
|
16
16
|
|
|
17
|
-
// src/
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
dtype;
|
|
27
|
-
device;
|
|
28
|
-
batchSize;
|
|
29
|
-
maxChars;
|
|
30
|
-
allowRemote;
|
|
31
|
-
extractor;
|
|
32
|
-
loadPromise;
|
|
33
|
-
constructor(opts = {}) {
|
|
34
|
-
this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
|
|
35
|
-
this.cacheDir = opts.cacheDir;
|
|
36
|
-
this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
|
|
37
|
-
this.device = opts.device ?? "cpu";
|
|
38
|
-
this.batchSize = opts.batchSize ?? 16;
|
|
39
|
-
this.maxChars = opts.maxChars ?? 2048;
|
|
40
|
-
this.allowRemote = opts.allowRemoteModels ?? true;
|
|
41
|
-
this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
|
|
42
|
-
this.id = `transformers-js:${this.modelId}:${this.dtype}`;
|
|
17
|
+
// src/sage-event-mirror.ts
|
|
18
|
+
import * as fs from "node:fs";
|
|
19
|
+
import * as path from "node:path";
|
|
20
|
+
import { getSageSurface } from "@wrongstack/sage";
|
|
21
|
+
function subscribeVectorMemoryToSage(opts) {
|
|
22
|
+
const { store, memoryStore } = opts;
|
|
23
|
+
const log = opts.logger;
|
|
24
|
+
if (opts.enabled === false) {
|
|
25
|
+
return { dispose: () => void 0 };
|
|
43
26
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
27
|
+
const surface = getSageSurface(memoryStore);
|
|
28
|
+
if (!surface) {
|
|
29
|
+
log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
|
|
30
|
+
return { dispose: () => void 0 };
|
|
31
|
+
}
|
|
32
|
+
const events = memoryStore.events;
|
|
33
|
+
if (!events) {
|
|
34
|
+
log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
|
|
35
|
+
return { dispose: () => void 0 };
|
|
36
|
+
}
|
|
37
|
+
const fetch = async (memoryId) => {
|
|
52
38
|
try {
|
|
53
|
-
await
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
return
|
|
39
|
+
return await surface.getSage(memoryId);
|
|
40
|
+
} catch (err) {
|
|
41
|
+
log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg(err)}`);
|
|
42
|
+
return null;
|
|
57
43
|
}
|
|
58
|
-
}
|
|
59
|
-
async
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
44
|
+
};
|
|
45
|
+
const mirror = async (memoryId) => {
|
|
46
|
+
const memory = await fetch(memoryId);
|
|
47
|
+
if (!memory) return;
|
|
48
|
+
if (memory.scope === "session") return;
|
|
49
|
+
try {
|
|
50
|
+
const existing = store.findBySageId(memoryId);
|
|
51
|
+
if (existing) await store.forget(existing.id);
|
|
52
|
+
await store.remember({
|
|
53
|
+
text: memory.text,
|
|
54
|
+
...memory.summary ? { summary: memory.summary } : {},
|
|
55
|
+
tags: memory.tags ?? [],
|
|
56
|
+
scope: "project",
|
|
57
|
+
kind: "note",
|
|
58
|
+
metadata: {
|
|
59
|
+
source: "sage",
|
|
60
|
+
sageId: memory.id,
|
|
61
|
+
sageKind: memory.kind,
|
|
62
|
+
sageScope: memory.scope,
|
|
63
|
+
importance: memory.importance,
|
|
64
|
+
confidence: memory.confidence
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
} catch (err) {
|
|
68
|
+
log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg(err)}`);
|
|
66
69
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
};
|
|
71
|
+
const forgetMirror = async (memoryId) => {
|
|
72
|
+
try {
|
|
73
|
+
const existing = store.findBySageId(memoryId);
|
|
74
|
+
if (existing) await store.forget(existing.id);
|
|
75
|
+
} catch (err) {
|
|
76
|
+
log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg(err)}`);
|
|
71
77
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
78
|
+
};
|
|
79
|
+
const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
|
|
80
|
+
const memoryId = payload?.memoryId;
|
|
81
|
+
if (typeof memoryId !== "string") return;
|
|
82
|
+
void mirror(memoryId);
|
|
83
|
+
});
|
|
84
|
+
const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
|
|
85
|
+
const memoryId = payload?.memoryId;
|
|
86
|
+
if (typeof memoryId !== "string") return;
|
|
87
|
+
void mirror(memoryId);
|
|
88
|
+
});
|
|
89
|
+
const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
|
|
90
|
+
const memoryId = payload?.memoryId;
|
|
91
|
+
if (typeof memoryId !== "string") return;
|
|
92
|
+
const status = payload?.status;
|
|
93
|
+
if (status === "deleted") return;
|
|
94
|
+
void mirror(memoryId);
|
|
95
|
+
});
|
|
96
|
+
const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
|
|
97
|
+
const memoryId = payload?.memoryId;
|
|
98
|
+
if (typeof memoryId !== "string") return;
|
|
99
|
+
void forgetMirror(memoryId);
|
|
100
|
+
});
|
|
101
|
+
return {
|
|
102
|
+
dispose: () => {
|
|
103
|
+
offAccepted();
|
|
104
|
+
offRecovered();
|
|
105
|
+
offUpdated();
|
|
106
|
+
offDeleted();
|
|
87
107
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
|
|
111
|
+
const surface = getSageSurface(memoryStore);
|
|
112
|
+
if (!surface) return { scanned: 0, removed: 0 };
|
|
113
|
+
let scanned = 0;
|
|
114
|
+
let removed = 0;
|
|
115
|
+
const PAGE = Math.max(1, options?.pageSize ?? 500);
|
|
116
|
+
let after;
|
|
117
|
+
for (; ; ) {
|
|
118
|
+
const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
|
|
119
|
+
if (page.length === 0) break;
|
|
120
|
+
const last = page[page.length - 1];
|
|
121
|
+
after = { updatedAt: last.updatedAt, id: last.id };
|
|
122
|
+
for (const entry of page) {
|
|
123
|
+
scanned++;
|
|
124
|
+
const sageId = entry.metadata?.sageId;
|
|
125
|
+
if (typeof sageId !== "string") continue;
|
|
126
|
+
try {
|
|
127
|
+
const memory = await surface.getSage(sageId);
|
|
128
|
+
if (memory !== null && memory.status !== "deleted") continue;
|
|
129
|
+
await store.forget(entry.id);
|
|
130
|
+
removed++;
|
|
131
|
+
} catch (err) {
|
|
132
|
+
logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg(err)}`);
|
|
95
133
|
}
|
|
96
|
-
return vectors;
|
|
97
|
-
}
|
|
98
|
-
if (Array.isArray(flat) && Array.isArray(flat[0])) {
|
|
99
|
-
return flat.map((row) => Float32Array.from(row));
|
|
100
134
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
135
|
+
if (page.length < PAGE) break;
|
|
136
|
+
}
|
|
137
|
+
return { scanned, removed };
|
|
138
|
+
}
|
|
139
|
+
function errMsg(err) {
|
|
140
|
+
return err instanceof Error ? err.message : String(err);
|
|
141
|
+
}
|
|
142
|
+
var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
|
|
143
|
+
var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
|
|
144
|
+
async function sweepStaleSageMirrors(opts) {
|
|
145
|
+
const markerPath2 = path.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
|
|
146
|
+
const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
147
|
+
if (!opts.force) {
|
|
148
|
+
try {
|
|
149
|
+
const raw = JSON.parse(fs.readFileSync(markerPath2, "utf8"));
|
|
150
|
+
const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
|
|
151
|
+
if (Number.isFinite(at) && Date.now() - at < interval) {
|
|
152
|
+
return { swept: false, reason: "throttled" };
|
|
153
|
+
}
|
|
154
|
+
} catch {
|
|
103
155
|
}
|
|
104
|
-
throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
|
|
105
156
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
this.extractor = await this.loadPromise;
|
|
110
|
-
return this.extractor;
|
|
157
|
+
try {
|
|
158
|
+
fs.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
|
|
159
|
+
} catch {
|
|
111
160
|
}
|
|
112
|
-
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
return pipe;
|
|
161
|
+
try {
|
|
162
|
+
const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
|
|
163
|
+
opts.logger?.debug?.(
|
|
164
|
+
`vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
|
|
165
|
+
);
|
|
166
|
+
return { swept: true, ...result };
|
|
167
|
+
} catch (err) {
|
|
168
|
+
opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg(err)}`);
|
|
169
|
+
return { swept: false, reason: errMsg(err) };
|
|
122
170
|
}
|
|
123
|
-
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// src/sage-fusion.ts
|
|
174
|
+
var DEFAULT_RRF_K = 60;
|
|
175
|
+
var DEFAULT_VECTOR_WEIGHT = 0.3;
|
|
176
|
+
function lexicalRankScore(index, total) {
|
|
177
|
+
if (total <= 1) return 1;
|
|
178
|
+
return 1 - index / Math.max(1, total - 1);
|
|
179
|
+
}
|
|
180
|
+
async function fuseWithVectorMemory(query, lexical, options = {}) {
|
|
181
|
+
const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
|
|
182
|
+
const k = options.rrfK ?? DEFAULT_RRF_K;
|
|
183
|
+
const limit = options.limit ?? 25;
|
|
184
|
+
let vectorHits = [];
|
|
185
|
+
if (options.vectorHits) {
|
|
186
|
+
vectorHits = [...options.vectorHits];
|
|
187
|
+
} else if (options.store) {
|
|
124
188
|
try {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
189
|
+
vectorHits = await options.store.search(query, {
|
|
190
|
+
...options.threshold !== void 0 ? { threshold: options.threshold } : {},
|
|
191
|
+
limit: Math.max(limit * 2, 50)
|
|
192
|
+
});
|
|
193
|
+
} catch {
|
|
194
|
+
vectorHits = [];
|
|
131
195
|
}
|
|
132
196
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
);
|
|
150
|
-
`);
|
|
151
|
-
db.exec(`
|
|
152
|
-
CREATE TABLE IF NOT EXISTS entries (
|
|
153
|
-
id TEXT PRIMARY KEY,
|
|
154
|
-
text TEXT NOT NULL,
|
|
155
|
-
summary TEXT,
|
|
156
|
-
metadata TEXT NOT NULL DEFAULT '{}',
|
|
157
|
-
tags TEXT NOT NULL DEFAULT '[]',
|
|
158
|
-
scope TEXT NOT NULL DEFAULT 'project',
|
|
159
|
-
kind TEXT NOT NULL DEFAULT 'note',
|
|
160
|
-
content_hash TEXT NOT NULL,
|
|
161
|
-
created_at TEXT NOT NULL,
|
|
162
|
-
updated_at TEXT NOT NULL
|
|
163
|
-
);
|
|
164
|
-
`);
|
|
165
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
|
|
166
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
|
|
167
|
-
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
|
|
168
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
|
|
169
|
-
db.exec(`
|
|
170
|
-
CREATE TABLE IF NOT EXISTS vectors (
|
|
171
|
-
entry_id TEXT NOT NULL,
|
|
172
|
-
provider_id TEXT NOT NULL,
|
|
173
|
-
dimensions INTEGER NOT NULL,
|
|
174
|
-
vector BLOB NOT NULL,
|
|
175
|
-
created_at TEXT NOT NULL,
|
|
176
|
-
PRIMARY KEY (entry_id, provider_id),
|
|
177
|
-
FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
|
|
178
|
-
);
|
|
179
|
-
`);
|
|
180
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
|
|
181
|
-
db.exec(`
|
|
182
|
-
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
183
|
-
content_hash TEXT NOT NULL,
|
|
184
|
-
provider_id TEXT NOT NULL,
|
|
185
|
-
dimensions INTEGER NOT NULL,
|
|
186
|
-
vector BLOB NOT NULL,
|
|
187
|
-
text TEXT NOT NULL,
|
|
188
|
-
created_at TEXT NOT NULL,
|
|
189
|
-
last_used_at TEXT NOT NULL,
|
|
190
|
-
use_count INTEGER NOT NULL DEFAULT 0,
|
|
191
|
-
PRIMARY KEY (content_hash, provider_id, dimensions)
|
|
192
|
-
);
|
|
193
|
-
`);
|
|
194
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
|
|
195
|
-
db.exec(`
|
|
196
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
|
|
197
|
-
id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
|
|
198
|
-
);
|
|
199
|
-
`);
|
|
200
|
-
db.exec(`
|
|
201
|
-
CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
|
|
202
|
-
INSERT INTO entries_fts(rowid, text, tags)
|
|
203
|
-
VALUES (new.rowid, new.text, new.tags);
|
|
204
|
-
END;
|
|
205
|
-
`);
|
|
206
|
-
db.exec(`
|
|
207
|
-
CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
|
|
208
|
-
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
209
|
-
VALUES('delete', old.rowid, old.text, old.tags);
|
|
210
|
-
END;
|
|
211
|
-
`);
|
|
212
|
-
db.exec(`
|
|
213
|
-
CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
|
|
214
|
-
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
215
|
-
VALUES('delete', old.rowid, old.text, old.tags);
|
|
216
|
-
INSERT INTO entries_fts(rowid, text, tags)
|
|
217
|
-
VALUES (new.rowid, new.text, new.tags);
|
|
218
|
-
END;
|
|
219
|
-
`);
|
|
220
|
-
}
|
|
221
|
-
function upsertEmbeddingCache(db, row) {
|
|
222
|
-
db.prepare(
|
|
223
|
-
`INSERT INTO embedding_cache
|
|
224
|
-
(content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
|
|
225
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, 1)
|
|
226
|
-
ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
|
|
227
|
-
last_used_at = excluded.last_used_at,
|
|
228
|
-
use_count = embedding_cache.use_count + 1,
|
|
229
|
-
vector = excluded.vector`
|
|
230
|
-
).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
|
|
231
|
-
}
|
|
232
|
-
function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
|
|
233
|
-
const row = db.prepare(
|
|
234
|
-
`SELECT vector FROM embedding_cache
|
|
235
|
-
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
|
|
236
|
-
LIMIT 1`
|
|
237
|
-
).get(contentHash, providerId, dimensions);
|
|
238
|
-
if (!row) return void 0;
|
|
239
|
-
try {
|
|
240
|
-
db.prepare(
|
|
241
|
-
`UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
|
|
242
|
-
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
|
|
243
|
-
).run(now, contentHash, providerId, dimensions);
|
|
244
|
-
} catch {
|
|
197
|
+
const sageById = /* @__PURE__ */ new Map();
|
|
198
|
+
for (const memory of lexical) sageById.set(memory.id, memory);
|
|
199
|
+
const lexicalRanked = lexical.map((memory, index) => ({
|
|
200
|
+
memory,
|
|
201
|
+
rankScore: lexicalRankScore(index, lexical.length),
|
|
202
|
+
lexicalScore: lexicalRankScore(index, lexical.length)
|
|
203
|
+
}));
|
|
204
|
+
const vectorRanked = [];
|
|
205
|
+
const seenVectorSageIds = /* @__PURE__ */ new Set();
|
|
206
|
+
for (let i = 0; i < vectorHits.length; i++) {
|
|
207
|
+
const hit = vectorHits[i];
|
|
208
|
+
const sageId = hit.entry.metadata?.["sageId"];
|
|
209
|
+
if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
|
|
210
|
+
const memory = sageById.get(sageId);
|
|
211
|
+
if (!memory) continue;
|
|
212
|
+
seenVectorSageIds.add(sageId);
|
|
213
|
+
vectorRanked.push({ memory, vectorScore: hit.score });
|
|
245
214
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
215
|
+
const fused = /* @__PURE__ */ new Map();
|
|
216
|
+
for (let i = 0; i < lexicalRanked.length; i++) {
|
|
217
|
+
const c = lexicalRanked[i];
|
|
218
|
+
const rrf = (1 - weight) * (1 / (k + i + 1));
|
|
219
|
+
fused.set(c.memory.id, {
|
|
220
|
+
memory: c.memory,
|
|
221
|
+
vectorScore: null,
|
|
222
|
+
lexicalScore: c.lexicalScore,
|
|
223
|
+
finalScore: rrf,
|
|
224
|
+
source: "lexical"
|
|
225
|
+
});
|
|
256
226
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
227
|
+
for (let i = 0; i < vectorRanked.length; i++) {
|
|
228
|
+
const v = vectorRanked[i];
|
|
229
|
+
const rrf = weight * (1 / (k + i + 1));
|
|
230
|
+
const existing = fused.get(v.memory.id);
|
|
231
|
+
if (existing) {
|
|
232
|
+
existing.finalScore += rrf;
|
|
233
|
+
existing.vectorScore = v.vectorScore;
|
|
234
|
+
existing.source = "both";
|
|
235
|
+
}
|
|
261
236
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
// src/store.ts
|
|
266
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
267
|
-
import * as fs from "node:fs";
|
|
268
|
-
import * as path from "node:path";
|
|
269
|
-
import { withFileLock } from "@wrongstack/core/utils";
|
|
270
|
-
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
271
|
-
import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
|
|
272
|
-
var DEFAULT_SEARCH_LIMIT = 10;
|
|
273
|
-
function normalizeLimit(limit) {
|
|
274
|
-
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
|
|
275
|
-
const floored = Math.floor(limit);
|
|
276
|
-
return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
|
|
237
|
+
const out = Array.from(fused.values());
|
|
238
|
+
out.sort((a, b) => b.finalScore - a.finalScore);
|
|
239
|
+
return out.slice(0, limit);
|
|
277
240
|
}
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
const filename = opts.filename ?? DEFAULT_FILENAME;
|
|
294
|
-
if (path.isAbsolute(dir)) {
|
|
295
|
-
throw new Error("Vector memory directory must be project-relative.");
|
|
241
|
+
function asVectorRecallProvider(store) {
|
|
242
|
+
return {
|
|
243
|
+
async search(query, opts) {
|
|
244
|
+
const hits = await store.search(query, {
|
|
245
|
+
limit: opts.limit,
|
|
246
|
+
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
247
|
+
});
|
|
248
|
+
return hits.map((h) => ({
|
|
249
|
+
id: h.entry.id,
|
|
250
|
+
score: h.score,
|
|
251
|
+
text: h.entry.text,
|
|
252
|
+
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
253
|
+
tags: h.entry.tags,
|
|
254
|
+
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
255
|
+
}));
|
|
296
256
|
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
function clamp01(value) {
|
|
260
|
+
if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
|
|
261
|
+
if (value < 0) return 0;
|
|
262
|
+
if (value > 1) return 1;
|
|
263
|
+
return value;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// src/sage-port-wrapper.ts
|
|
267
|
+
import {
|
|
268
|
+
augmentLexicalWithVectorRecall,
|
|
269
|
+
isSageVisibleForSearch,
|
|
270
|
+
SAGE_RETRIEVAL_CAPABILITY,
|
|
271
|
+
SAGE_SURFACE_CAPABILITY
|
|
272
|
+
} from "@wrongstack/sage";
|
|
273
|
+
function asVectorRecallProviderAdapter(store) {
|
|
274
|
+
return {
|
|
275
|
+
async search(query, opts) {
|
|
276
|
+
const hits = await store.search(query, {
|
|
277
|
+
limit: opts.limit,
|
|
278
|
+
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
279
|
+
});
|
|
280
|
+
return hits.map((h) => ({
|
|
281
|
+
id: h.entry.id,
|
|
282
|
+
score: h.score,
|
|
283
|
+
text: h.entry.text,
|
|
284
|
+
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
285
|
+
tags: h.entry.tags,
|
|
286
|
+
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
287
|
+
}));
|
|
301
288
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
this.db.prepare(
|
|
339
|
-
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
340
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
341
|
-
).run(VECTOR_PROVIDER_KEY, this.provider.id);
|
|
342
|
-
this.db.prepare(
|
|
343
|
-
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
344
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
345
|
-
).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
|
|
346
|
-
this.db.exec("COMMIT");
|
|
347
|
-
} catch (e) {
|
|
348
|
-
this.db.exec("ROLLBACK");
|
|
349
|
-
throw e;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
static contentHash(text) {
|
|
353
|
-
return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
|
|
354
|
-
}
|
|
355
|
-
/**
|
|
356
|
-
* Look up a vector for `text` in the provider-level embedding cache.
|
|
357
|
-
* Cache hit returns the cached vector (no ONNX pass). Miss returns
|
|
358
|
-
* `undefined`.
|
|
359
|
-
*/
|
|
360
|
-
cachedVector(text, now) {
|
|
361
|
-
return lookupEmbeddingCache(
|
|
362
|
-
this.db,
|
|
363
|
-
_VectorMemoryStore.contentHash(text),
|
|
364
|
-
this.provider.id,
|
|
365
|
-
this.provider.dimensions,
|
|
366
|
-
now
|
|
367
|
-
);
|
|
368
|
-
}
|
|
369
|
-
/** Persist `vec` for `text` to the embedding cache. */
|
|
370
|
-
cacheVector(text, vec, now) {
|
|
371
|
-
upsertEmbeddingCache(this.db, {
|
|
372
|
-
contentHash: _VectorMemoryStore.contentHash(text),
|
|
373
|
-
providerId: this.provider.id,
|
|
374
|
-
dimensions: vec.length,
|
|
375
|
-
vector: encodeVector(vec),
|
|
376
|
-
text,
|
|
377
|
-
now
|
|
378
|
-
});
|
|
379
|
-
}
|
|
380
|
-
/**
|
|
381
|
-
* Embed `text`, hitting the provider-level cache first. Cache miss falls
|
|
382
|
-
* through to the configured provider and writes the result back. Returns
|
|
383
|
-
* `undefined` when the provider fails — the caller can persist the entry
|
|
384
|
-
* without a vector (fail-open).
|
|
385
|
-
*/
|
|
386
|
-
async embedWithCache(text) {
|
|
387
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
388
|
-
const cached = this.cachedVector(text, now);
|
|
389
|
-
if (cached) return cached;
|
|
390
|
-
try {
|
|
391
|
-
const result = await this.provider.embed([text]);
|
|
392
|
-
const vec = result[0];
|
|
393
|
-
if (vec) this.cacheVector(text, vec, now);
|
|
394
|
-
return vec;
|
|
395
|
-
} catch {
|
|
396
|
-
return void 0;
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
/**
|
|
400
|
-
* Look up an existing entry by `content_hash`. Returns `undefined` when
|
|
401
|
-
* the entry is not present. Used by `remember()` to make writes idempotent
|
|
402
|
-
* and by `syncFromSage()` to skip already-indexed SAGE memories.
|
|
403
|
-
*/
|
|
404
|
-
findByContentHash(contentHash) {
|
|
405
|
-
this.assertOpen();
|
|
406
|
-
const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
|
|
407
|
-
if (!row) return void 0;
|
|
408
|
-
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
409
|
-
return this.rowToEntry(row, vectorRow);
|
|
410
|
-
}
|
|
411
|
-
/**
|
|
412
|
-
* Look up the entry mirroring a given SAGE memory id (i.e. the entry
|
|
413
|
-
* whose `metadata.sageId` equals `sageId`). Returns `undefined` when
|
|
414
|
-
* no such entry exists. Used by the event-driven mirror to delete
|
|
415
|
-
* vector entries on SAGE delete events (the emitter knows the SAGE
|
|
416
|
-
* id, not the vector entry id).
|
|
417
|
-
*
|
|
418
|
-
* Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
|
|
419
|
-
* column is the JSON blob `syncFromSage` writes, so this avoids a
|
|
420
|
-
* full table scan.
|
|
421
|
-
*/
|
|
422
|
-
findBySageId(sageId) {
|
|
423
|
-
this.assertOpen();
|
|
424
|
-
const row = this.db.prepare(
|
|
425
|
-
`SELECT * FROM entries
|
|
426
|
-
WHERE json_extract(metadata, '$.sageId') = ?
|
|
427
|
-
ORDER BY updated_at DESC
|
|
428
|
-
LIMIT 1`
|
|
429
|
-
).get(sageId);
|
|
430
|
-
if (!row) return void 0;
|
|
431
|
-
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
432
|
-
return this.rowToEntry(row, vectorRow);
|
|
433
|
-
}
|
|
434
|
-
/**
|
|
435
|
-
* Persist a new entry. Idempotent: if an entry with the same
|
|
436
|
-
* `content_hash` already exists, that entry is returned unchanged
|
|
437
|
-
* instead of inserting a duplicate. Mutating ops are wrapped in
|
|
438
|
-
* `withFileLock` so two processes cannot race the dedup check.
|
|
439
|
-
*/
|
|
440
|
-
async remember(input) {
|
|
441
|
-
this.assertOpen();
|
|
442
|
-
if (!input.text || input.text.trim().length === 0) {
|
|
443
|
-
throw new Error("VectorMemoryStore.remember: text must be non-empty");
|
|
444
|
-
}
|
|
445
|
-
return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
|
|
446
|
-
timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
|
-
async rememberUnlocked(input) {
|
|
450
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
451
|
-
const contentHash = _VectorMemoryStore.contentHash(input.text);
|
|
452
|
-
const existing = this.findByContentHash(contentHash);
|
|
453
|
-
if (existing) return existing;
|
|
454
|
-
const metadata = input.metadata ?? {};
|
|
455
|
-
const tags = input.tags ?? [];
|
|
456
|
-
const scope = input.scope ?? "project";
|
|
457
|
-
const kind = input.kind ?? "note";
|
|
458
|
-
const id = randomUUID();
|
|
459
|
-
const vector = await this.embedWithCache(input.text);
|
|
460
|
-
const providerId = vector ? this.provider.id : void 0;
|
|
461
|
-
this.db.exec("BEGIN");
|
|
462
|
-
try {
|
|
463
|
-
this.db.prepare(
|
|
464
|
-
`INSERT INTO entries
|
|
465
|
-
(id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
|
|
466
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
467
|
-
).run(
|
|
468
|
-
id,
|
|
469
|
-
input.text,
|
|
470
|
-
input.summary ?? null,
|
|
471
|
-
JSON.stringify(metadata),
|
|
472
|
-
JSON.stringify(tags),
|
|
473
|
-
scope,
|
|
474
|
-
kind,
|
|
475
|
-
contentHash,
|
|
476
|
-
now,
|
|
477
|
-
now
|
|
478
|
-
);
|
|
479
|
-
if (vector && providerId) {
|
|
480
|
-
this.db.prepare(
|
|
481
|
-
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
482
|
-
VALUES (?, ?, ?, ?, ?)
|
|
483
|
-
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
484
|
-
vector = excluded.vector,
|
|
485
|
-
dimensions = excluded.dimensions,
|
|
486
|
-
created_at = excluded.created_at`
|
|
487
|
-
).run(id, providerId, vector.length, encodeVector(vector), now);
|
|
488
|
-
}
|
|
489
|
-
this.db.exec("COMMIT");
|
|
490
|
-
} catch (e) {
|
|
491
|
-
this.db.exec("ROLLBACK");
|
|
492
|
-
throw e;
|
|
493
|
-
}
|
|
494
|
-
const result = {
|
|
495
|
-
id,
|
|
496
|
-
text: input.text,
|
|
497
|
-
summary: input.summary ?? void 0,
|
|
498
|
-
metadata,
|
|
499
|
-
tags,
|
|
500
|
-
scope,
|
|
501
|
-
kind,
|
|
502
|
-
contentHash,
|
|
503
|
-
createdAt: now,
|
|
504
|
-
updatedAt: now,
|
|
505
|
-
providerId: providerId ?? "",
|
|
506
|
-
dimensions: vector?.length ?? 0
|
|
507
|
-
};
|
|
508
|
-
if (vector) result.vector = vector;
|
|
509
|
-
return result;
|
|
510
|
-
}
|
|
511
|
-
get(id) {
|
|
512
|
-
this.assertOpen();
|
|
513
|
-
const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
|
|
514
|
-
if (!row) return void 0;
|
|
515
|
-
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
|
|
516
|
-
return this.rowToEntry(row, vectorRow);
|
|
517
|
-
}
|
|
518
|
-
/** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
|
|
519
|
-
async forget(id) {
|
|
520
|
-
this.assertOpen();
|
|
521
|
-
return withFileLock(
|
|
522
|
-
this.lockPath,
|
|
523
|
-
async () => {
|
|
524
|
-
this.db.exec("BEGIN");
|
|
525
|
-
try {
|
|
526
|
-
const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
|
|
527
|
-
this.db.exec("COMMIT");
|
|
528
|
-
return info.changes > 0;
|
|
529
|
-
} catch (e) {
|
|
530
|
-
this.db.exec("ROLLBACK");
|
|
531
|
-
throw e;
|
|
532
|
-
}
|
|
533
|
-
},
|
|
534
|
-
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
535
|
-
);
|
|
536
|
-
}
|
|
537
|
-
async search(query, opts = {}) {
|
|
538
|
-
this.assertOpen();
|
|
539
|
-
const limit = normalizeLimit(opts.limit);
|
|
540
|
-
const threshold = opts.threshold ?? 0;
|
|
541
|
-
const includeVectors = opts.includeVectors === true;
|
|
542
|
-
if (typeof query !== "string" || query.trim().length === 0) return [];
|
|
543
|
-
const queryVec = await this.embedWithCache(query);
|
|
544
|
-
if (!queryVec || queryVec.length === 0) return [];
|
|
545
|
-
const providerId = this.provider.id;
|
|
546
|
-
const dimensions = this.provider.dimensions;
|
|
547
|
-
const filters = ["v.provider_id = ?", "v.dimensions = ?"];
|
|
548
|
-
const params = [providerId, dimensions];
|
|
549
|
-
if (opts.scope !== void 0) {
|
|
550
|
-
filters.push("e.scope = ?");
|
|
551
|
-
params.push(opts.scope);
|
|
552
|
-
}
|
|
553
|
-
if (opts.kind !== void 0) {
|
|
554
|
-
filters.push("e.kind = ?");
|
|
555
|
-
params.push(opts.kind);
|
|
556
|
-
}
|
|
557
|
-
const scanRows = this.db.prepare(
|
|
558
|
-
`SELECT e.id AS id, v.vector AS vec_blob
|
|
559
|
-
FROM entries e
|
|
560
|
-
JOIN vectors v ON v.entry_id = e.id
|
|
561
|
-
WHERE ${filters.join(" AND ")}`
|
|
562
|
-
).all(...params);
|
|
563
|
-
const top = [];
|
|
564
|
-
for (const row of scanRows) {
|
|
565
|
-
const vec = decodeVector(row.vec_blob);
|
|
566
|
-
const raw = cosineSimilarity(queryVec, vec);
|
|
567
|
-
const score = Math.max(0, Math.min(1, raw));
|
|
568
|
-
if (!Number.isFinite(score) || score < threshold) continue;
|
|
569
|
-
if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
|
|
570
|
-
let at = top.length;
|
|
571
|
-
while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
|
|
572
|
-
top.splice(at, 0, { id: row.id, score, vector: vec });
|
|
573
|
-
if (top.length > limit) top.length = limit;
|
|
574
|
-
}
|
|
575
|
-
if (top.length === 0) return [];
|
|
576
|
-
const placeholders = top.map(() => "?").join(",");
|
|
577
|
-
const hydrated = this.db.prepare(
|
|
578
|
-
`SELECT id, text, summary, metadata, tags, scope, kind,
|
|
579
|
-
content_hash, created_at, updated_at
|
|
580
|
-
FROM entries WHERE id IN (${placeholders})`
|
|
581
|
-
).all(...top.map((t) => t.id));
|
|
582
|
-
const entryById = /* @__PURE__ */ new Map();
|
|
583
|
-
for (const row of hydrated) {
|
|
584
|
-
entryById.set(row.id, this.rowToEntry(row));
|
|
585
|
-
}
|
|
586
|
-
const scored = [];
|
|
587
|
-
for (const candidate of top) {
|
|
588
|
-
const entry = entryById.get(candidate.id);
|
|
589
|
-
if (!entry) continue;
|
|
590
|
-
const hit = { entry, score: candidate.score, providerId };
|
|
591
|
-
if (includeVectors) hit.vector = candidate.vector;
|
|
592
|
-
scored.push(hit);
|
|
593
|
-
}
|
|
594
|
-
return scored;
|
|
595
|
-
}
|
|
596
|
-
/**
|
|
597
|
-
* Page through entries, newest first.
|
|
598
|
-
*
|
|
599
|
-
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
600
|
-
* without the id tiebreak two entries written in the same millisecond can
|
|
601
|
-
* swap places between calls and a paging caller silently skips one.
|
|
602
|
-
*
|
|
603
|
-
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
604
|
-
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
605
|
-
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
606
|
-
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
607
|
-
* resumes from a position, and the rows a deletion removes are ones the
|
|
608
|
-
* sweep has already passed.
|
|
609
|
-
*/
|
|
610
|
-
list(opts = {}) {
|
|
611
|
-
this.assertOpen();
|
|
612
|
-
const where = [];
|
|
613
|
-
const params = [];
|
|
614
|
-
if (opts.scope !== void 0) {
|
|
615
|
-
where.push("scope = ?");
|
|
616
|
-
params.push(opts.scope);
|
|
617
|
-
}
|
|
618
|
-
if (opts.kind !== void 0) {
|
|
619
|
-
where.push("kind = ?");
|
|
620
|
-
params.push(opts.kind);
|
|
621
|
-
}
|
|
622
|
-
if (opts.after) {
|
|
623
|
-
where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
|
|
624
|
-
params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
|
|
625
|
-
}
|
|
626
|
-
const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
|
|
627
|
-
ORDER BY updated_at DESC, id DESC LIMIT ?`;
|
|
628
|
-
params.push(opts.limit ?? 100);
|
|
629
|
-
const rows = this.db.prepare(sql).all(...params);
|
|
630
|
-
return rows.map((r) => this.rowToEntry(r));
|
|
631
|
-
}
|
|
632
|
-
async reindexAll() {
|
|
633
|
-
this.assertOpen();
|
|
634
|
-
return withFileLock(
|
|
635
|
-
this.lockPath,
|
|
636
|
-
async () => {
|
|
637
|
-
const rows = this.db.prepare("SELECT id, text FROM entries").all();
|
|
638
|
-
let processed = 0;
|
|
639
|
-
let errors = 0;
|
|
640
|
-
for (const row of rows) {
|
|
641
|
-
try {
|
|
642
|
-
const result = await this.provider.embed([row.text]);
|
|
643
|
-
const v = result[0];
|
|
644
|
-
if (!v) {
|
|
645
|
-
errors++;
|
|
646
|
-
continue;
|
|
647
|
-
}
|
|
648
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
649
|
-
this.db.prepare(
|
|
650
|
-
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
651
|
-
VALUES (?, ?, ?, ?, ?)
|
|
652
|
-
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
653
|
-
vector = excluded.vector,
|
|
654
|
-
dimensions = excluded.dimensions,
|
|
655
|
-
created_at = excluded.created_at`
|
|
656
|
-
).run(row.id, this.provider.id, v.length, encodeVector(v), now);
|
|
657
|
-
this.cacheVector(row.text, v, now);
|
|
658
|
-
processed++;
|
|
659
|
-
} catch {
|
|
660
|
-
errors++;
|
|
661
|
-
}
|
|
662
|
-
}
|
|
663
|
-
return { processed, errors };
|
|
664
|
-
},
|
|
665
|
-
{ timeoutMs: 6e4 }
|
|
666
|
-
);
|
|
667
|
-
}
|
|
668
|
-
stats() {
|
|
669
|
-
this.assertOpen();
|
|
670
|
-
const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
|
|
671
|
-
const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
|
|
672
|
-
const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
|
|
673
|
-
return {
|
|
674
|
-
entries: entryCount,
|
|
675
|
-
vectors: vectorCount,
|
|
676
|
-
providers: providerRows.map((r) => r.provider_id),
|
|
677
|
-
modelAvailable: true,
|
|
678
|
-
modelId: this.provider.id,
|
|
679
|
-
dimensions: this.provider.dimensions
|
|
680
|
-
};
|
|
681
|
-
}
|
|
682
|
-
/**
|
|
683
|
-
* Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
|
|
684
|
-
* Useful for the WebUI's vector-memory panel and for diagnosing the
|
|
685
|
-
* "why is search slow?" question.
|
|
686
|
-
*/
|
|
687
|
-
cacheStats() {
|
|
688
|
-
this.assertOpen();
|
|
689
|
-
const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
690
|
-
const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
|
|
691
|
-
const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
|
|
692
|
-
const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
|
|
693
|
-
return {
|
|
694
|
-
entries,
|
|
695
|
-
providers,
|
|
696
|
-
totalUseCount,
|
|
697
|
-
oldestLastUsedAt: oldest?.t ?? null
|
|
698
|
-
};
|
|
699
|
-
}
|
|
700
|
-
/**
|
|
701
|
-
* LRU-evict the embedding cache down to `keepMostRecent` rows. The
|
|
702
|
-
* `embedding_cache` table is independent of `entries`, so a sweep here
|
|
703
|
-
* only removes cached vectors — never stored entries. Called by hosts
|
|
704
|
-
* that want to bound cache growth on long-lived processes.
|
|
705
|
-
*/
|
|
706
|
-
async evictCache(keepMostRecent) {
|
|
707
|
-
this.assertOpen();
|
|
708
|
-
if (keepMostRecent < 0) {
|
|
709
|
-
throw new Error("evictCache: keepMostRecent must be >= 0");
|
|
710
|
-
}
|
|
711
|
-
return withFileLock(
|
|
712
|
-
this.lockPath,
|
|
713
|
-
async () => {
|
|
714
|
-
const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
715
|
-
if (total <= keepMostRecent) return { removed: 0 };
|
|
716
|
-
const toRemove = total - keepMostRecent;
|
|
717
|
-
const stmt = this.db.prepare(
|
|
718
|
-
`DELETE FROM embedding_cache
|
|
719
|
-
WHERE content_hash IN (
|
|
720
|
-
SELECT content_hash FROM embedding_cache
|
|
721
|
-
ORDER BY last_used_at ASC
|
|
722
|
-
LIMIT ?
|
|
723
|
-
)`
|
|
724
|
-
);
|
|
725
|
-
const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
|
|
726
|
-
return { removed: Number(info.changes) };
|
|
727
|
-
},
|
|
728
|
-
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function wrapMemoryPortWithVectorRecall(port, options) {
|
|
292
|
+
const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
|
|
293
|
+
const materializeFor = (searchOpts) => async (sageId) => {
|
|
294
|
+
const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
|
|
295
|
+
if (!surface?.getSage) return void 0;
|
|
296
|
+
const memory = await surface.getSage(sageId);
|
|
297
|
+
if (!memory) return void 0;
|
|
298
|
+
return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
|
|
299
|
+
};
|
|
300
|
+
const fusionOptions = (searchOpts) => ({
|
|
301
|
+
vectorRecall: recall,
|
|
302
|
+
materializeVectorOnly: materializeFor(searchOpts),
|
|
303
|
+
...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
|
|
304
|
+
...options.threshold !== void 0 ? { threshold: options.threshold } : {},
|
|
305
|
+
...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
|
|
306
|
+
...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
|
|
307
|
+
...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
|
|
308
|
+
});
|
|
309
|
+
const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
|
|
310
|
+
const wrapSearchSage = (original) => async (query, searchOpts) => {
|
|
311
|
+
const opts = searchOpts;
|
|
312
|
+
const lexical = await original(query, searchOpts);
|
|
313
|
+
if (callerOwnsFusion(opts)) return lexical;
|
|
314
|
+
const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
|
|
315
|
+
return fused.map((hit) => hit.memory);
|
|
316
|
+
};
|
|
317
|
+
const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
|
|
318
|
+
const opts = searchOpts;
|
|
319
|
+
const lexicalHits = await original(query, searchOpts);
|
|
320
|
+
if (callerOwnsFusion(opts)) return lexicalHits;
|
|
321
|
+
return augmentLexicalWithVectorRecall(
|
|
322
|
+
query,
|
|
323
|
+
lexicalHits.map((hit) => hit.memory),
|
|
324
|
+
fusionOptions(opts)
|
|
729
325
|
);
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
kind: row.kind,
|
|
749
|
-
contentHash: row.content_hash,
|
|
750
|
-
createdAt: row.created_at,
|
|
751
|
-
updatedAt: row.updated_at,
|
|
752
|
-
providerId: vectorRow?.provider_id ?? "",
|
|
753
|
-
dimensions: vectorRow?.dimensions ?? 0
|
|
754
|
-
};
|
|
755
|
-
if (vectorRow?.vector) {
|
|
756
|
-
entry.vector = decodeVector(vectorRow.vector);
|
|
326
|
+
};
|
|
327
|
+
const wrapped = Object.create(
|
|
328
|
+
Object.getPrototypeOf(port),
|
|
329
|
+
Object.getOwnPropertyDescriptors(port)
|
|
330
|
+
);
|
|
331
|
+
wrapped.getCapability = (capability) => {
|
|
332
|
+
if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
|
|
333
|
+
const original = port.getCapability(capability);
|
|
334
|
+
if (!original) return void 0;
|
|
335
|
+
return {
|
|
336
|
+
...original,
|
|
337
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
338
|
+
...original.searchSageWithBreakdown ? {
|
|
339
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
340
|
+
original.searchSageWithBreakdown
|
|
341
|
+
)
|
|
342
|
+
} : {}
|
|
343
|
+
};
|
|
757
344
|
}
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
const existing = this.findByContentHash(hash);
|
|
771
|
-
if (existing) {
|
|
772
|
-
skipped++;
|
|
773
|
-
continue;
|
|
774
|
-
}
|
|
775
|
-
await this.rememberUnlocked({
|
|
776
|
-
text: memory.text,
|
|
777
|
-
summary: memory.summary ?? void 0,
|
|
778
|
-
metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
|
|
779
|
-
tags: memory.tags ?? [],
|
|
780
|
-
scope: "project",
|
|
781
|
-
kind: "note"
|
|
782
|
-
});
|
|
783
|
-
indexed++;
|
|
784
|
-
} catch (err) {
|
|
785
|
-
failed++;
|
|
786
|
-
errors.push({ memoryId: memory.id, message: errMsg(err) });
|
|
787
|
-
}
|
|
345
|
+
if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
|
|
346
|
+
const original = port.getCapability(capability);
|
|
347
|
+
if (!original) return void 0;
|
|
348
|
+
return {
|
|
349
|
+
...original,
|
|
350
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
351
|
+
...original.searchSageWithBreakdown ? {
|
|
352
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
353
|
+
original.searchSageWithBreakdown
|
|
354
|
+
)
|
|
355
|
+
} : {}
|
|
356
|
+
};
|
|
788
357
|
}
|
|
789
|
-
return
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
function fallbackHashingProvider(dimensions) {
|
|
793
|
-
return new HashingEmbeddingProvider({ dimensions });
|
|
794
|
-
}
|
|
795
|
-
function safeParseJson(value, fallback) {
|
|
796
|
-
if (typeof value !== "string") return fallback;
|
|
797
|
-
try {
|
|
798
|
-
return JSON.parse(value);
|
|
799
|
-
} catch {
|
|
800
|
-
return fallback;
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
function errMsg(err) {
|
|
804
|
-
return err instanceof Error ? err.message : String(err);
|
|
358
|
+
return port.getCapability(capability);
|
|
359
|
+
};
|
|
360
|
+
return wrapped;
|
|
805
361
|
}
|
|
806
362
|
|
|
363
|
+
// src/sage-sync.ts
|
|
364
|
+
import * as fs2 from "node:fs";
|
|
365
|
+
import * as path2 from "node:path";
|
|
366
|
+
import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
|
|
367
|
+
|
|
807
368
|
// src/sage-sync-source.ts
|
|
808
369
|
var DEFAULT_PAGE_SIZE = 500;
|
|
809
370
|
var HARD_FLOOR_PAGE = 1;
|
|
@@ -840,7 +401,7 @@ function createSageSurfaceSyncSource(sage, opts = {}) {
|
|
|
840
401
|
}
|
|
841
402
|
});
|
|
842
403
|
}
|
|
843
|
-
if (!page.nextCursor
|
|
404
|
+
if (!page.nextCursor) break;
|
|
844
405
|
if (rows.length === 0) {
|
|
845
406
|
noProgressPages++;
|
|
846
407
|
if (noProgressPages > 3) break;
|
|
@@ -858,9 +419,6 @@ function clamp(value, min, max) {
|
|
|
858
419
|
}
|
|
859
420
|
|
|
860
421
|
// src/sage-sync.ts
|
|
861
|
-
import { getSageSurface } from "@wrongstack/sage";
|
|
862
|
-
import * as fs2 from "node:fs";
|
|
863
|
-
import * as path2 from "node:path";
|
|
864
422
|
var SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
|
|
865
423
|
var RUNNING_STALE_MS = 10 * 60 * 1e3;
|
|
866
424
|
async function startFirstBootSageSync(opts) {
|
|
@@ -901,7 +459,7 @@ async function startFirstBootSageSync(opts) {
|
|
|
901
459
|
return { synced: false, reason: "provider-unavailable" };
|
|
902
460
|
}
|
|
903
461
|
}
|
|
904
|
-
const surface =
|
|
462
|
+
const surface = getSageSurface2(memoryStore);
|
|
905
463
|
if (!surface) {
|
|
906
464
|
log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
|
|
907
465
|
return { synced: false, reason: "no-sage-surface" };
|
|
@@ -1033,613 +591,1049 @@ function counts(report) {
|
|
|
1033
591
|
};
|
|
1034
592
|
}
|
|
1035
593
|
|
|
1036
|
-
// src/
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
594
|
+
// src/schema.ts
|
|
595
|
+
var VECTOR_SCHEMA_VERSION = 2;
|
|
596
|
+
var VECTOR_PROVIDER_KEY = "active_provider_id";
|
|
597
|
+
var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
|
|
598
|
+
function initVectorSchema(db) {
|
|
599
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
600
|
+
db.exec("PRAGMA synchronous = NORMAL");
|
|
601
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
602
|
+
db.exec("PRAGMA temp_store = MEMORY");
|
|
603
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
604
|
+
db.exec(`
|
|
605
|
+
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
606
|
+
key TEXT PRIMARY KEY,
|
|
607
|
+
value TEXT NOT NULL
|
|
608
|
+
);
|
|
609
|
+
`);
|
|
610
|
+
db.exec(`
|
|
611
|
+
CREATE TABLE IF NOT EXISTS entries (
|
|
612
|
+
id TEXT PRIMARY KEY,
|
|
613
|
+
text TEXT NOT NULL,
|
|
614
|
+
summary TEXT,
|
|
615
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
616
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
617
|
+
scope TEXT NOT NULL DEFAULT 'project',
|
|
618
|
+
kind TEXT NOT NULL DEFAULT 'note',
|
|
619
|
+
content_hash TEXT NOT NULL,
|
|
620
|
+
created_at TEXT NOT NULL,
|
|
621
|
+
updated_at TEXT NOT NULL
|
|
622
|
+
);
|
|
623
|
+
`);
|
|
624
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
|
|
625
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
|
|
626
|
+
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
|
|
627
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
|
|
628
|
+
db.exec(`
|
|
629
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
630
|
+
entry_id TEXT NOT NULL,
|
|
631
|
+
provider_id TEXT NOT NULL,
|
|
632
|
+
dimensions INTEGER NOT NULL,
|
|
633
|
+
vector BLOB NOT NULL,
|
|
634
|
+
created_at TEXT NOT NULL,
|
|
635
|
+
PRIMARY KEY (entry_id, provider_id),
|
|
636
|
+
FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
|
|
637
|
+
);
|
|
638
|
+
`);
|
|
639
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
|
|
640
|
+
db.exec(`
|
|
641
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
642
|
+
content_hash TEXT NOT NULL,
|
|
643
|
+
provider_id TEXT NOT NULL,
|
|
644
|
+
dimensions INTEGER NOT NULL,
|
|
645
|
+
vector BLOB NOT NULL,
|
|
646
|
+
text TEXT NOT NULL,
|
|
647
|
+
created_at TEXT NOT NULL,
|
|
648
|
+
last_used_at TEXT NOT NULL,
|
|
649
|
+
use_count INTEGER NOT NULL DEFAULT 0,
|
|
650
|
+
PRIMARY KEY (content_hash, provider_id, dimensions)
|
|
651
|
+
);
|
|
652
|
+
`);
|
|
653
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
|
|
654
|
+
db.exec(`
|
|
655
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
|
|
656
|
+
id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
|
|
657
|
+
);
|
|
658
|
+
`);
|
|
659
|
+
db.exec(`
|
|
660
|
+
CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
|
|
661
|
+
INSERT INTO entries_fts(rowid, text, tags)
|
|
662
|
+
VALUES (new.rowid, new.text, new.tags);
|
|
663
|
+
END;
|
|
664
|
+
`);
|
|
665
|
+
db.exec(`
|
|
666
|
+
CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
|
|
667
|
+
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
668
|
+
VALUES('delete', old.rowid, old.text, old.tags);
|
|
669
|
+
END;
|
|
670
|
+
`);
|
|
671
|
+
db.exec(`
|
|
672
|
+
CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
|
|
673
|
+
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
674
|
+
VALUES('delete', old.rowid, old.text, old.tags);
|
|
675
|
+
INSERT INTO entries_fts(rowid, text, tags)
|
|
676
|
+
VALUES (new.rowid, new.text, new.tags);
|
|
677
|
+
END;
|
|
678
|
+
`);
|
|
1044
679
|
}
|
|
1045
|
-
function
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
capabilities: ["memory.write"],
|
|
1056
|
-
icon: "settings",
|
|
1057
|
-
inputSchema: {
|
|
1058
|
-
type: "object",
|
|
1059
|
-
properties: {
|
|
1060
|
-
text: { type: "string", minLength: 1, description: "The text to embed and store." },
|
|
1061
|
-
summary: { type: "string", description: "Optional short label." },
|
|
1062
|
-
tags: {
|
|
1063
|
-
type: "array",
|
|
1064
|
-
items: { type: "string" },
|
|
1065
|
-
description: "Optional tags for later filtering."
|
|
1066
|
-
},
|
|
1067
|
-
scope: {
|
|
1068
|
-
type: "string",
|
|
1069
|
-
enum: ["project", "user", "session"],
|
|
1070
|
-
description: "Visibility scope. Defaults to `project`."
|
|
1071
|
-
},
|
|
1072
|
-
kind: {
|
|
1073
|
-
type: "string",
|
|
1074
|
-
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1075
|
-
description: "Entry kind. Defaults to `note`."
|
|
1076
|
-
},
|
|
1077
|
-
metadata: {
|
|
1078
|
-
type: "object",
|
|
1079
|
-
additionalProperties: true,
|
|
1080
|
-
description: "Free-form metadata persisted as JSON."
|
|
1081
|
-
}
|
|
1082
|
-
},
|
|
1083
|
-
required: ["text"],
|
|
1084
|
-
additionalProperties: false
|
|
1085
|
-
},
|
|
1086
|
-
execute: async (input) => {
|
|
1087
|
-
const entry = await store.remember(input);
|
|
1088
|
-
return { id: entry.id, hasVector: entry.vector !== void 0 };
|
|
1089
|
-
}
|
|
1090
|
-
};
|
|
680
|
+
function upsertEmbeddingCache(db, row) {
|
|
681
|
+
db.prepare(
|
|
682
|
+
`INSERT INTO embedding_cache
|
|
683
|
+
(content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
|
|
684
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 1)
|
|
685
|
+
ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
|
|
686
|
+
last_used_at = excluded.last_used_at,
|
|
687
|
+
use_count = embedding_cache.use_count + 1,
|
|
688
|
+
vector = excluded.vector`
|
|
689
|
+
).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
|
|
1091
690
|
}
|
|
1092
|
-
function
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
691
|
+
function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
|
|
692
|
+
const row = db.prepare(
|
|
693
|
+
`SELECT vector FROM embedding_cache
|
|
694
|
+
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
|
|
695
|
+
LIMIT 1`
|
|
696
|
+
).get(contentHash, providerId, dimensions);
|
|
697
|
+
if (!row) return void 0;
|
|
698
|
+
try {
|
|
699
|
+
db.prepare(
|
|
700
|
+
`UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
|
|
701
|
+
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
|
|
702
|
+
).run(now, contentHash, providerId, dimensions);
|
|
703
|
+
} catch {
|
|
704
|
+
}
|
|
705
|
+
return decodeVector(row.vector);
|
|
706
|
+
}
|
|
707
|
+
function encodeVector(vec) {
|
|
708
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
709
|
+
}
|
|
710
|
+
function decodeVector(buf) {
|
|
711
|
+
if (buf.byteLength % 4 !== 0) {
|
|
712
|
+
throw new Error(
|
|
713
|
+
`decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
717
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
718
|
+
for (let i = 0; i < copy.length; i++) {
|
|
719
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
720
|
+
}
|
|
721
|
+
return copy;
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// src/search-race.ts
|
|
725
|
+
function previewText(text, maxLen) {
|
|
726
|
+
if (text.length <= maxLen) return text;
|
|
727
|
+
return text.slice(0, maxLen - 1) + "\u2026";
|
|
728
|
+
}
|
|
729
|
+
async function runSearchRace(query, lexical, vectorStore, options = {}) {
|
|
730
|
+
const limit = options.limit ?? 20;
|
|
731
|
+
const threshold = options.threshold ?? 0;
|
|
732
|
+
let vectorHits = [];
|
|
733
|
+
try {
|
|
734
|
+
vectorHits = await vectorStore.search(query, {
|
|
735
|
+
limit,
|
|
736
|
+
...threshold > 0 ? { threshold } : {}
|
|
737
|
+
});
|
|
738
|
+
} catch {
|
|
739
|
+
vectorHits = [];
|
|
740
|
+
}
|
|
741
|
+
const lexicalOnly = [];
|
|
742
|
+
const vectorOnly = [];
|
|
743
|
+
const overlap = [];
|
|
744
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
745
|
+
const lexicalCapped = lexical.slice(0, limit);
|
|
746
|
+
for (let i = 0; i < lexicalCapped.length; i++) {
|
|
747
|
+
const mem = lexicalCapped[i];
|
|
748
|
+
const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
|
|
749
|
+
const id = mem.id;
|
|
750
|
+
seenIds.add(id);
|
|
751
|
+
overlap.push({
|
|
752
|
+
id,
|
|
753
|
+
lexicalScore: score,
|
|
754
|
+
vectorScore: null,
|
|
755
|
+
// patched below when a vector hit carries this id
|
|
756
|
+
preview: previewText(mem.text, 140)
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
const overlapById = /* @__PURE__ */ new Map();
|
|
760
|
+
for (const row of overlap) overlapById.set(row.id, row);
|
|
761
|
+
for (const hit of vectorHits) {
|
|
762
|
+
const sageId = hit.entry.metadata?.["sageId"];
|
|
763
|
+
if (typeof sageId !== "string") continue;
|
|
764
|
+
const existing = overlapById.get(sageId);
|
|
765
|
+
if (existing) {
|
|
766
|
+
if (existing.vectorScore === null) {
|
|
767
|
+
existing.vectorScore = hit.score;
|
|
768
|
+
}
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
if (seenIds.has(sageId)) continue;
|
|
772
|
+
seenIds.add(sageId);
|
|
773
|
+
vectorOnly.push({
|
|
774
|
+
id: sageId,
|
|
775
|
+
lexicalScore: null,
|
|
776
|
+
vectorScore: hit.score,
|
|
777
|
+
preview: previewText(hit.entry.text, 140)
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
const finalOverlap = [];
|
|
781
|
+
for (const row of overlap) {
|
|
782
|
+
if (row.vectorScore === null) {
|
|
783
|
+
lexicalOnly.push({
|
|
784
|
+
id: row.id,
|
|
785
|
+
lexicalScore: row.lexicalScore,
|
|
786
|
+
vectorScore: null,
|
|
787
|
+
preview: row.preview
|
|
1140
788
|
});
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
id: h.entry.id,
|
|
1144
|
-
score: h.score,
|
|
1145
|
-
text: h.entry.text,
|
|
1146
|
-
summary: h.entry.summary ?? void 0,
|
|
1147
|
-
tags: h.entry.tags
|
|
1148
|
-
}))
|
|
1149
|
-
};
|
|
789
|
+
} else {
|
|
790
|
+
finalOverlap.push(row);
|
|
1150
791
|
}
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
name: "vector_memory_stats",
|
|
1156
|
-
category: "Memory",
|
|
1157
|
-
description: "Return counts, providers, and dimensions for the local vector memory store.",
|
|
1158
|
-
usageHint: "Cheap diagnostic \u2014 safe to call any time.",
|
|
1159
|
-
permission: "auto",
|
|
1160
|
-
mutating: false,
|
|
1161
|
-
riskTier: "safe",
|
|
1162
|
-
timeoutMs: 1e3,
|
|
1163
|
-
capabilities: ["memory.read"],
|
|
1164
|
-
icon: "search",
|
|
1165
|
-
inputSchema: {
|
|
1166
|
-
type: "object",
|
|
1167
|
-
properties: {},
|
|
1168
|
-
additionalProperties: false
|
|
1169
|
-
},
|
|
1170
|
-
execute: async () => store.stats()
|
|
1171
|
-
};
|
|
1172
|
-
}
|
|
1173
|
-
function vectorMemoryForgetTool(store) {
|
|
792
|
+
}
|
|
793
|
+
const lexicalCount = lexicalOnly.length + finalOverlap.length;
|
|
794
|
+
const vectorCount = vectorOnly.length + finalOverlap.length;
|
|
795
|
+
const denom = Math.max(lexicalCount, vectorCount, 1);
|
|
1174
796
|
return {
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
properties: {
|
|
1188
|
-
id: {
|
|
1189
|
-
type: "string",
|
|
1190
|
-
minLength: 1,
|
|
1191
|
-
description: "Entry id returned by `vector_memory_remember`."
|
|
1192
|
-
}
|
|
1193
|
-
},
|
|
1194
|
-
required: ["id"],
|
|
1195
|
-
additionalProperties: false
|
|
1196
|
-
},
|
|
1197
|
-
execute: async (input) => ({ removed: await store.forget(input.id) })
|
|
797
|
+
query,
|
|
798
|
+
lexicalOnly,
|
|
799
|
+
vectorOnly,
|
|
800
|
+
overlap: finalOverlap,
|
|
801
|
+
metrics: {
|
|
802
|
+
lexicalCount,
|
|
803
|
+
vectorCount,
|
|
804
|
+
overlapCount: finalOverlap.length,
|
|
805
|
+
lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
|
|
806
|
+
vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
|
|
807
|
+
agreementRatio: finalOverlap.length / denom
|
|
808
|
+
}
|
|
1198
809
|
};
|
|
1199
810
|
}
|
|
1200
811
|
|
|
1201
|
-
// src/
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
812
|
+
// src/store.ts
|
|
813
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
814
|
+
import * as fs3 from "node:fs";
|
|
815
|
+
import * as path3 from "node:path";
|
|
816
|
+
import { withFileLock } from "@wrongstack/core/utils";
|
|
817
|
+
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
818
|
+
import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
|
|
819
|
+
var DEFAULT_SEARCH_LIMIT = 10;
|
|
820
|
+
function normalizeLimit(limit) {
|
|
821
|
+
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
|
|
822
|
+
const floored = Math.floor(limit);
|
|
823
|
+
return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
|
|
1207
824
|
}
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
825
|
+
var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
|
|
826
|
+
var DEFAULT_FILENAME = "vector-memory.db";
|
|
827
|
+
var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
|
|
828
|
+
var CACHE_EVICT_BATCH = 256;
|
|
829
|
+
var VectorMemoryStore = class _VectorMemoryStore {
|
|
830
|
+
db;
|
|
831
|
+
dbPath;
|
|
832
|
+
rootDir;
|
|
833
|
+
provider;
|
|
834
|
+
closed = false;
|
|
835
|
+
constructor(opts) {
|
|
836
|
+
if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
|
|
837
|
+
if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
|
|
838
|
+
this.provider = opts.provider;
|
|
839
|
+
const dir = opts.directory ?? DEFAULT_DIRECTORY;
|
|
840
|
+
const filename = opts.filename ?? DEFAULT_FILENAME;
|
|
841
|
+
if (path3.isAbsolute(dir)) {
|
|
842
|
+
throw new Error("Vector memory directory must be project-relative.");
|
|
843
|
+
}
|
|
844
|
+
const rootDir = path3.resolve(opts.projectRoot, dir);
|
|
845
|
+
const rel = path3.relative(path3.resolve(opts.projectRoot), rootDir);
|
|
846
|
+
if (rel.startsWith("..") || path3.isAbsolute(rel)) {
|
|
847
|
+
throw new Error("Vector memory directory must stay inside the project root.");
|
|
848
|
+
}
|
|
849
|
+
fs3.mkdirSync(rootDir, { recursive: true });
|
|
850
|
+
this.rootDir = rootDir;
|
|
851
|
+
this.dbPath = path3.join(rootDir, filename);
|
|
852
|
+
const Database = loadRuntimeDatabaseSync();
|
|
853
|
+
this.db = new Database(this.dbPath);
|
|
854
|
+
initVectorSchema(this.db);
|
|
855
|
+
this.recordActiveProvider();
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Absolute path of the store's data directory (the resolved
|
|
859
|
+
* `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
|
|
860
|
+
* to place sidecar state (e.g. the first-boot SAGE sync marker) next to
|
|
861
|
+
* the db instead of re-deriving the path and drifting from it.
|
|
862
|
+
*/
|
|
863
|
+
get directory() {
|
|
864
|
+
return this.rootDir;
|
|
865
|
+
}
|
|
866
|
+
/**
|
|
867
|
+
* Absolute path of the SQLite database file. Hosts use this to take a
|
|
868
|
+
* file-level lock that covers all mutating operations (see
|
|
869
|
+
* `withFileLock(this.dbPath + '.lock', …)`).
|
|
870
|
+
*/
|
|
871
|
+
get databasePath() {
|
|
872
|
+
return this.dbPath;
|
|
873
|
+
}
|
|
874
|
+
/** The lockfile path used to serialize mutating operations. */
|
|
875
|
+
get lockPath() {
|
|
876
|
+
return `${this.dbPath}.lock`;
|
|
877
|
+
}
|
|
878
|
+
get activeProviderId() {
|
|
879
|
+
const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
|
|
880
|
+
return row?.value ?? this.provider.id;
|
|
881
|
+
}
|
|
882
|
+
recordActiveProvider() {
|
|
883
|
+
this.db.exec("BEGIN");
|
|
1217
884
|
try {
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
885
|
+
this.db.prepare(
|
|
886
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
887
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
888
|
+
).run(VECTOR_PROVIDER_KEY, this.provider.id);
|
|
889
|
+
this.db.prepare(
|
|
890
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
891
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
892
|
+
).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
|
|
893
|
+
this.db.exec("COMMIT");
|
|
894
|
+
} catch (e) {
|
|
895
|
+
this.db.exec("ROLLBACK");
|
|
896
|
+
throw e;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
static contentHash(text) {
|
|
900
|
+
return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
|
|
901
|
+
}
|
|
902
|
+
/**
|
|
903
|
+
* Look up a vector for `text` in the provider-level embedding cache.
|
|
904
|
+
* Cache hit returns the cached vector (no ONNX pass). Miss returns
|
|
905
|
+
* `undefined`.
|
|
906
|
+
*/
|
|
907
|
+
cachedVector(text, now) {
|
|
908
|
+
return lookupEmbeddingCache(
|
|
909
|
+
this.db,
|
|
910
|
+
_VectorMemoryStore.contentHash(text),
|
|
911
|
+
this.provider.id,
|
|
912
|
+
this.provider.dimensions,
|
|
913
|
+
now
|
|
914
|
+
);
|
|
915
|
+
}
|
|
916
|
+
/** Persist `vec` for `text` to the embedding cache. */
|
|
917
|
+
cacheVector(text, vec, now) {
|
|
918
|
+
upsertEmbeddingCache(this.db, {
|
|
919
|
+
contentHash: _VectorMemoryStore.contentHash(text),
|
|
920
|
+
providerId: this.provider.id,
|
|
921
|
+
dimensions: vec.length,
|
|
922
|
+
vector: encodeVector(vec),
|
|
923
|
+
text,
|
|
924
|
+
now
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
/**
|
|
928
|
+
* Embed `text`, hitting the provider-level cache first. Cache miss falls
|
|
929
|
+
* through to the configured provider and writes the result back. Returns
|
|
930
|
+
* `undefined` when the provider fails — the caller can persist the entry
|
|
931
|
+
* without a vector (fail-open).
|
|
932
|
+
*/
|
|
933
|
+
async embedWithCache(text) {
|
|
934
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
935
|
+
const cached = this.cachedVector(text, now);
|
|
936
|
+
if (cached) return cached;
|
|
937
|
+
try {
|
|
938
|
+
const result = await this.provider.embed([text]);
|
|
939
|
+
const vec = result[0];
|
|
940
|
+
if (vec) this.cacheVector(text, vec, now);
|
|
941
|
+
return vec;
|
|
1222
942
|
} catch {
|
|
1223
|
-
|
|
943
|
+
return void 0;
|
|
1224
944
|
}
|
|
1225
945
|
}
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
const sageId = hit.entry.metadata?.["sageId"];
|
|
1238
|
-
if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
|
|
1239
|
-
const memory = sageById.get(sageId);
|
|
1240
|
-
if (!memory) continue;
|
|
1241
|
-
seenVectorSageIds.add(sageId);
|
|
1242
|
-
vectorRanked.push({ memory, vectorScore: hit.score });
|
|
946
|
+
/**
|
|
947
|
+
* Look up an existing entry by `content_hash`. Returns `undefined` when
|
|
948
|
+
* the entry is not present. Used by `remember()` to make writes idempotent
|
|
949
|
+
* and by `syncFromSage()` to skip already-indexed SAGE memories.
|
|
950
|
+
*/
|
|
951
|
+
findByContentHash(contentHash) {
|
|
952
|
+
this.assertOpen();
|
|
953
|
+
const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
|
|
954
|
+
if (!row) return void 0;
|
|
955
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
956
|
+
return this.rowToEntry(row, vectorRow);
|
|
1243
957
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
958
|
+
/**
|
|
959
|
+
* Look up the entry mirroring a given SAGE memory id (i.e. the entry
|
|
960
|
+
* whose `metadata.sageId` equals `sageId`). Returns `undefined` when
|
|
961
|
+
* no such entry exists. Used by the event-driven mirror to delete
|
|
962
|
+
* vector entries on SAGE delete events (the emitter knows the SAGE
|
|
963
|
+
* id, not the vector entry id).
|
|
964
|
+
*
|
|
965
|
+
* Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
|
|
966
|
+
* column is the JSON blob `syncFromSage` writes, so this avoids a
|
|
967
|
+
* full table scan.
|
|
968
|
+
*/
|
|
969
|
+
findBySageId(sageId) {
|
|
970
|
+
this.assertOpen();
|
|
971
|
+
const row = this.db.prepare(
|
|
972
|
+
`SELECT * FROM entries
|
|
973
|
+
WHERE json_extract(metadata, '$.sageId') = ?
|
|
974
|
+
ORDER BY updated_at DESC
|
|
975
|
+
LIMIT 1`
|
|
976
|
+
).get(sageId);
|
|
977
|
+
if (!row) return void 0;
|
|
978
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
979
|
+
return this.rowToEntry(row, vectorRow);
|
|
980
|
+
}
|
|
981
|
+
/**
|
|
982
|
+
* Persist a new entry. Idempotent: if an entry with the same
|
|
983
|
+
* `content_hash` already exists, that entry is returned unchanged
|
|
984
|
+
* instead of inserting a duplicate. Mutating ops are wrapped in
|
|
985
|
+
* `withFileLock` so two processes cannot race the dedup check.
|
|
986
|
+
*/
|
|
987
|
+
async remember(input) {
|
|
988
|
+
this.assertOpen();
|
|
989
|
+
if (!input.text || input.text.trim().length === 0) {
|
|
990
|
+
throw new Error("VectorMemoryStore.remember: text must be non-empty");
|
|
991
|
+
}
|
|
992
|
+
return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
|
|
993
|
+
timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
|
|
1254
994
|
});
|
|
1255
995
|
}
|
|
1256
|
-
|
|
1257
|
-
const
|
|
1258
|
-
const
|
|
1259
|
-
const existing =
|
|
1260
|
-
if (existing)
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
996
|
+
async rememberUnlocked(input) {
|
|
997
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
998
|
+
const contentHash = _VectorMemoryStore.contentHash(input.text);
|
|
999
|
+
const existing = this.findByContentHash(contentHash);
|
|
1000
|
+
if (existing) return existing;
|
|
1001
|
+
const metadata = input.metadata ?? {};
|
|
1002
|
+
const tags = input.tags ?? [];
|
|
1003
|
+
const scope = input.scope ?? "project";
|
|
1004
|
+
const kind = input.kind ?? "note";
|
|
1005
|
+
const id = randomUUID();
|
|
1006
|
+
const vector = await this.embedWithCache(input.text);
|
|
1007
|
+
const providerId = vector ? this.provider.id : void 0;
|
|
1008
|
+
this.db.exec("BEGIN");
|
|
1009
|
+
try {
|
|
1010
|
+
this.db.prepare(
|
|
1011
|
+
`INSERT INTO entries
|
|
1012
|
+
(id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
|
|
1013
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1014
|
+
).run(
|
|
1015
|
+
id,
|
|
1016
|
+
input.text,
|
|
1017
|
+
input.summary ?? null,
|
|
1018
|
+
JSON.stringify(metadata),
|
|
1019
|
+
JSON.stringify(tags),
|
|
1020
|
+
scope,
|
|
1021
|
+
kind,
|
|
1022
|
+
contentHash,
|
|
1023
|
+
now,
|
|
1024
|
+
now
|
|
1025
|
+
);
|
|
1026
|
+
if (vector && providerId) {
|
|
1027
|
+
this.db.prepare(
|
|
1028
|
+
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
1029
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1030
|
+
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
1031
|
+
vector = excluded.vector,
|
|
1032
|
+
dimensions = excluded.dimensions,
|
|
1033
|
+
created_at = excluded.created_at`
|
|
1034
|
+
).run(id, providerId, vector.length, encodeVector(vector), now);
|
|
1035
|
+
}
|
|
1036
|
+
this.db.exec("COMMIT");
|
|
1037
|
+
} catch (e) {
|
|
1038
|
+
this.db.exec("ROLLBACK");
|
|
1039
|
+
throw e;
|
|
1040
|
+
}
|
|
1041
|
+
const result = {
|
|
1042
|
+
id,
|
|
1043
|
+
text: input.text,
|
|
1044
|
+
summary: input.summary ?? void 0,
|
|
1045
|
+
metadata,
|
|
1046
|
+
tags,
|
|
1047
|
+
scope,
|
|
1048
|
+
kind,
|
|
1049
|
+
contentHash,
|
|
1050
|
+
createdAt: now,
|
|
1051
|
+
updatedAt: now,
|
|
1052
|
+
providerId: providerId ?? "",
|
|
1053
|
+
dimensions: vector?.length ?? 0
|
|
1054
|
+
};
|
|
1055
|
+
if (vector) result.vector = vector;
|
|
1056
|
+
return result;
|
|
1057
|
+
}
|
|
1058
|
+
get(id) {
|
|
1059
|
+
this.assertOpen();
|
|
1060
|
+
const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
|
|
1061
|
+
if (!row) return void 0;
|
|
1062
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
|
|
1063
|
+
return this.rowToEntry(row, vectorRow);
|
|
1064
|
+
}
|
|
1065
|
+
/** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
|
|
1066
|
+
async forget(id) {
|
|
1067
|
+
this.assertOpen();
|
|
1068
|
+
return withFileLock(
|
|
1069
|
+
this.lockPath,
|
|
1070
|
+
async () => {
|
|
1071
|
+
this.db.exec("BEGIN");
|
|
1072
|
+
try {
|
|
1073
|
+
const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
|
|
1074
|
+
this.db.exec("COMMIT");
|
|
1075
|
+
return info.changes > 0;
|
|
1076
|
+
} catch (e) {
|
|
1077
|
+
this.db.exec("ROLLBACK");
|
|
1078
|
+
throw e;
|
|
1079
|
+
}
|
|
1080
|
+
},
|
|
1081
|
+
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
async search(query, opts = {}) {
|
|
1085
|
+
this.assertOpen();
|
|
1086
|
+
const limit = normalizeLimit(opts.limit);
|
|
1087
|
+
const threshold = opts.threshold ?? 0;
|
|
1088
|
+
const includeVectors = opts.includeVectors === true;
|
|
1089
|
+
if (typeof query !== "string" || query.trim().length === 0) return [];
|
|
1090
|
+
const queryVec = await this.embedWithCache(query);
|
|
1091
|
+
if (!queryVec || queryVec.length === 0) return [];
|
|
1092
|
+
const providerId = this.provider.id;
|
|
1093
|
+
const dimensions = this.provider.dimensions;
|
|
1094
|
+
const filters = ["v.provider_id = ?", "v.dimensions = ?"];
|
|
1095
|
+
const params = [providerId, dimensions];
|
|
1096
|
+
if (opts.scope !== void 0) {
|
|
1097
|
+
filters.push("e.scope = ?");
|
|
1098
|
+
params.push(opts.scope);
|
|
1099
|
+
}
|
|
1100
|
+
if (opts.kind !== void 0) {
|
|
1101
|
+
filters.push("e.kind = ?");
|
|
1102
|
+
params.push(opts.kind);
|
|
1103
|
+
}
|
|
1104
|
+
const scanRows = this.db.prepare(
|
|
1105
|
+
`SELECT e.id AS id, v.vector AS vec_blob
|
|
1106
|
+
FROM entries e
|
|
1107
|
+
JOIN vectors v ON v.entry_id = e.id
|
|
1108
|
+
WHERE ${filters.join(" AND ")}`
|
|
1109
|
+
).all(...params);
|
|
1110
|
+
const top = [];
|
|
1111
|
+
for (const row of scanRows) {
|
|
1112
|
+
const vec = decodeVector(row.vec_blob);
|
|
1113
|
+
const raw = cosineSimilarity(queryVec, vec);
|
|
1114
|
+
const score = Math.max(0, Math.min(1, raw));
|
|
1115
|
+
if (!Number.isFinite(score) || score < threshold) continue;
|
|
1116
|
+
if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
|
|
1117
|
+
let at = top.length;
|
|
1118
|
+
while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
|
|
1119
|
+
top.splice(at, 0, { id: row.id, score, vector: vec });
|
|
1120
|
+
if (top.length > limit) top.length = limit;
|
|
1121
|
+
}
|
|
1122
|
+
if (top.length === 0) return [];
|
|
1123
|
+
const placeholders = top.map(() => "?").join(",");
|
|
1124
|
+
const hydrated = this.db.prepare(
|
|
1125
|
+
`SELECT id, text, summary, metadata, tags, scope, kind,
|
|
1126
|
+
content_hash, created_at, updated_at
|
|
1127
|
+
FROM entries WHERE id IN (${placeholders})`
|
|
1128
|
+
).all(...top.map((t) => t.id));
|
|
1129
|
+
const entryById = /* @__PURE__ */ new Map();
|
|
1130
|
+
for (const row of hydrated) {
|
|
1131
|
+
entryById.set(row.id, this.rowToEntry(row));
|
|
1132
|
+
}
|
|
1133
|
+
const scored = [];
|
|
1134
|
+
for (const candidate of top) {
|
|
1135
|
+
const entry = entryById.get(candidate.id);
|
|
1136
|
+
if (!entry) continue;
|
|
1137
|
+
const hit = { entry, score: candidate.score, providerId };
|
|
1138
|
+
if (includeVectors) hit.vector = candidate.vector;
|
|
1139
|
+
scored.push(hit);
|
|
1140
|
+
}
|
|
1141
|
+
return scored;
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Page through entries, newest first.
|
|
1145
|
+
*
|
|
1146
|
+
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
1147
|
+
* without the id tiebreak two entries written in the same millisecond can
|
|
1148
|
+
* swap places between calls and a paging caller silently skips one.
|
|
1149
|
+
*
|
|
1150
|
+
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
1151
|
+
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
1152
|
+
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
1153
|
+
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
1154
|
+
* resumes from a position, and the rows a deletion removes are ones the
|
|
1155
|
+
* sweep has already passed.
|
|
1156
|
+
*/
|
|
1157
|
+
list(opts = {}) {
|
|
1158
|
+
this.assertOpen();
|
|
1159
|
+
const where = [];
|
|
1160
|
+
const params = [];
|
|
1161
|
+
if (opts.scope !== void 0) {
|
|
1162
|
+
where.push("scope = ?");
|
|
1163
|
+
params.push(opts.scope);
|
|
1272
1164
|
}
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
return out.slice(0, limit);
|
|
1277
|
-
}
|
|
1278
|
-
function asVectorRecallProvider(store) {
|
|
1279
|
-
return {
|
|
1280
|
-
async search(query, opts) {
|
|
1281
|
-
const hits = await store.search(query, {
|
|
1282
|
-
limit: opts.limit,
|
|
1283
|
-
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
1284
|
-
});
|
|
1285
|
-
return hits.map((h) => ({
|
|
1286
|
-
id: h.entry.id,
|
|
1287
|
-
score: h.score,
|
|
1288
|
-
text: h.entry.text,
|
|
1289
|
-
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
1290
|
-
tags: h.entry.tags,
|
|
1291
|
-
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
1292
|
-
}));
|
|
1165
|
+
if (opts.kind !== void 0) {
|
|
1166
|
+
where.push("kind = ?");
|
|
1167
|
+
params.push(opts.kind);
|
|
1293
1168
|
}
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
|
|
1298
|
-
if (value < 0) return 0;
|
|
1299
|
-
if (value > 1) return 1;
|
|
1300
|
-
return value;
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
// src/sage-port-wrapper.ts
|
|
1304
|
-
import {
|
|
1305
|
-
augmentLexicalWithVectorRecall,
|
|
1306
|
-
isSageVisibleForSearch,
|
|
1307
|
-
SAGE_RETRIEVAL_CAPABILITY,
|
|
1308
|
-
SAGE_SURFACE_CAPABILITY
|
|
1309
|
-
} from "@wrongstack/sage";
|
|
1310
|
-
function asVectorRecallProviderAdapter(store) {
|
|
1311
|
-
return {
|
|
1312
|
-
async search(query, opts) {
|
|
1313
|
-
const hits = await store.search(query, {
|
|
1314
|
-
limit: opts.limit,
|
|
1315
|
-
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
1316
|
-
});
|
|
1317
|
-
return hits.map((h) => ({
|
|
1318
|
-
id: h.entry.id,
|
|
1319
|
-
score: h.score,
|
|
1320
|
-
text: h.entry.text,
|
|
1321
|
-
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
1322
|
-
tags: h.entry.tags,
|
|
1323
|
-
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
1324
|
-
}));
|
|
1169
|
+
if (opts.after) {
|
|
1170
|
+
where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
|
|
1171
|
+
params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
|
|
1325
1172
|
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1173
|
+
const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
|
|
1174
|
+
ORDER BY updated_at DESC, id DESC LIMIT ?`;
|
|
1175
|
+
params.push(opts.limit ?? 100);
|
|
1176
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
1177
|
+
return rows.map((r) => this.rowToEntry(r));
|
|
1178
|
+
}
|
|
1179
|
+
async reindexAll() {
|
|
1180
|
+
this.assertOpen();
|
|
1181
|
+
return withFileLock(
|
|
1182
|
+
this.lockPath,
|
|
1183
|
+
async () => {
|
|
1184
|
+
const rows = this.db.prepare("SELECT id, text FROM entries").all();
|
|
1185
|
+
let processed = 0;
|
|
1186
|
+
let errors = 0;
|
|
1187
|
+
for (const row of rows) {
|
|
1188
|
+
try {
|
|
1189
|
+
const result = await this.provider.embed([row.text]);
|
|
1190
|
+
const v = result[0];
|
|
1191
|
+
if (!v) {
|
|
1192
|
+
errors++;
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1196
|
+
this.db.prepare(
|
|
1197
|
+
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
1198
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1199
|
+
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
1200
|
+
vector = excluded.vector,
|
|
1201
|
+
dimensions = excluded.dimensions,
|
|
1202
|
+
created_at = excluded.created_at`
|
|
1203
|
+
).run(row.id, this.provider.id, v.length, encodeVector(v), now);
|
|
1204
|
+
this.cacheVector(row.text, v, now);
|
|
1205
|
+
processed++;
|
|
1206
|
+
} catch {
|
|
1207
|
+
errors++;
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
return { processed, errors };
|
|
1211
|
+
},
|
|
1212
|
+
{ timeoutMs: 6e4 }
|
|
1362
1213
|
);
|
|
1363
|
-
}
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1214
|
+
}
|
|
1215
|
+
stats() {
|
|
1216
|
+
this.assertOpen();
|
|
1217
|
+
const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
|
|
1218
|
+
const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
|
|
1219
|
+
const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
|
|
1220
|
+
return {
|
|
1221
|
+
entries: entryCount,
|
|
1222
|
+
vectors: vectorCount,
|
|
1223
|
+
providers: providerRows.map((r) => r.provider_id),
|
|
1224
|
+
modelAvailable: true,
|
|
1225
|
+
modelId: this.provider.id,
|
|
1226
|
+
dimensions: this.provider.dimensions
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
|
|
1231
|
+
* Useful for the WebUI's vector-memory panel and for diagnosing the
|
|
1232
|
+
* "why is search slow?" question.
|
|
1233
|
+
*/
|
|
1234
|
+
cacheStats() {
|
|
1235
|
+
this.assertOpen();
|
|
1236
|
+
const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
1237
|
+
const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
|
|
1238
|
+
const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
|
|
1239
|
+
const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
|
|
1240
|
+
return {
|
|
1241
|
+
entries,
|
|
1242
|
+
providers,
|
|
1243
|
+
totalUseCount,
|
|
1244
|
+
oldestLastUsedAt: oldest?.t ?? null
|
|
1245
|
+
};
|
|
1246
|
+
}
|
|
1247
|
+
/**
|
|
1248
|
+
* LRU-evict the embedding cache down to `keepMostRecent` rows. The
|
|
1249
|
+
* `embedding_cache` table is independent of `entries`, so a sweep here
|
|
1250
|
+
* only removes cached vectors — never stored entries. Called by hosts
|
|
1251
|
+
* that want to bound cache growth on long-lived processes.
|
|
1252
|
+
*/
|
|
1253
|
+
async evictCache(keepMostRecent) {
|
|
1254
|
+
this.assertOpen();
|
|
1255
|
+
if (keepMostRecent < 0) {
|
|
1256
|
+
throw new Error("evictCache: keepMostRecent must be >= 0");
|
|
1394
1257
|
}
|
|
1395
|
-
return
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1258
|
+
return withFileLock(
|
|
1259
|
+
this.lockPath,
|
|
1260
|
+
async () => {
|
|
1261
|
+
const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
1262
|
+
if (total <= keepMostRecent) return { removed: 0 };
|
|
1263
|
+
const toRemove = total - keepMostRecent;
|
|
1264
|
+
const stmt = this.db.prepare(
|
|
1265
|
+
`DELETE FROM embedding_cache
|
|
1266
|
+
WHERE content_hash IN (
|
|
1267
|
+
SELECT content_hash FROM embedding_cache
|
|
1268
|
+
ORDER BY last_used_at ASC
|
|
1269
|
+
LIMIT ?
|
|
1270
|
+
)`
|
|
1271
|
+
);
|
|
1272
|
+
const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
|
|
1273
|
+
return { removed: Number(info.changes) };
|
|
1274
|
+
},
|
|
1275
|
+
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
1276
|
+
);
|
|
1277
|
+
}
|
|
1278
|
+
close() {
|
|
1279
|
+
if (this.closed) return;
|
|
1280
|
+
this.closed = true;
|
|
1281
|
+
this.db.close();
|
|
1282
|
+
}
|
|
1283
|
+
assertOpen() {
|
|
1284
|
+
if (this.closed) throw new Error("VectorMemoryStore is closed");
|
|
1285
|
+
}
|
|
1286
|
+
rowToEntry(row, vectorRow) {
|
|
1287
|
+
const summaryValue = row.summary;
|
|
1288
|
+
const entry = {
|
|
1289
|
+
id: row.id,
|
|
1290
|
+
text: row.text,
|
|
1291
|
+
summary: summaryValue ?? void 0,
|
|
1292
|
+
metadata: safeParseJson(row.metadata, {}),
|
|
1293
|
+
tags: safeParseJson(row.tags, []),
|
|
1294
|
+
scope: row.scope,
|
|
1295
|
+
kind: row.kind,
|
|
1296
|
+
contentHash: row.content_hash,
|
|
1297
|
+
createdAt: row.created_at,
|
|
1298
|
+
updatedAt: row.updated_at,
|
|
1299
|
+
providerId: vectorRow?.provider_id ?? "",
|
|
1300
|
+
dimensions: vectorRow?.dimensions ?? 0
|
|
1301
|
+
};
|
|
1302
|
+
if (vectorRow?.vector) {
|
|
1303
|
+
entry.vector = decodeVector(vectorRow.vector);
|
|
1304
|
+
}
|
|
1305
|
+
return entry;
|
|
1409
1306
|
}
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1307
|
+
async syncFromSage(sage) {
|
|
1308
|
+
this.assertOpen();
|
|
1309
|
+
const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
|
|
1310
|
+
let indexed = 0;
|
|
1311
|
+
let skipped = 0;
|
|
1312
|
+
let failed = 0;
|
|
1313
|
+
const errors = [];
|
|
1314
|
+
for (const memory of memories) {
|
|
1315
|
+
try {
|
|
1316
|
+
const hash = _VectorMemoryStore.contentHash(memory.text);
|
|
1317
|
+
const existing = this.findByContentHash(hash);
|
|
1318
|
+
if (existing) {
|
|
1319
|
+
skipped++;
|
|
1320
|
+
continue;
|
|
1321
|
+
}
|
|
1322
|
+
await this.rememberUnlocked({
|
|
1323
|
+
text: memory.text,
|
|
1324
|
+
summary: memory.summary ?? void 0,
|
|
1325
|
+
metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
|
|
1326
|
+
tags: memory.tags ?? [],
|
|
1327
|
+
scope: "project",
|
|
1328
|
+
kind: "note"
|
|
1329
|
+
});
|
|
1330
|
+
indexed++;
|
|
1331
|
+
} catch (err) {
|
|
1332
|
+
failed++;
|
|
1333
|
+
errors.push({ memoryId: memory.id, message: errMsg2(err) });
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
return { scanned: memories.length, indexed, skipped, failed, errors };
|
|
1414
1337
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1338
|
+
};
|
|
1339
|
+
function fallbackHashingProvider(dimensions) {
|
|
1340
|
+
return new HashingEmbeddingProvider({ dimensions });
|
|
1341
|
+
}
|
|
1342
|
+
function safeParseJson(value, fallback) {
|
|
1343
|
+
if (typeof value !== "string") return fallback;
|
|
1344
|
+
try {
|
|
1345
|
+
return JSON.parse(value);
|
|
1346
|
+
} catch {
|
|
1347
|
+
return fallback;
|
|
1419
1348
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1349
|
+
}
|
|
1350
|
+
function errMsg2(err) {
|
|
1351
|
+
return err instanceof Error ? err.message : String(err);
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
// src/tools.ts
|
|
1355
|
+
function createVectorMemoryTools(store) {
|
|
1356
|
+
return [
|
|
1357
|
+
vectorMemoryRememberTool(store),
|
|
1358
|
+
vectorMemorySearchTool(store),
|
|
1359
|
+
vectorMemoryStatsTool(store),
|
|
1360
|
+
vectorMemoryForgetTool(store)
|
|
1361
|
+
];
|
|
1362
|
+
}
|
|
1363
|
+
function vectorMemoryRememberTool(store) {
|
|
1364
|
+
return {
|
|
1365
|
+
name: "vector_memory_remember",
|
|
1366
|
+
category: "Memory",
|
|
1367
|
+
description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
|
|
1368
|
+
usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
|
|
1369
|
+
permission: "confirm",
|
|
1370
|
+
mutating: true,
|
|
1371
|
+
riskTier: "standard",
|
|
1372
|
+
timeoutMs: 5e3,
|
|
1373
|
+
capabilities: ["memory.write"],
|
|
1374
|
+
icon: "settings",
|
|
1375
|
+
inputSchema: {
|
|
1376
|
+
type: "object",
|
|
1377
|
+
properties: {
|
|
1378
|
+
text: { type: "string", minLength: 1, description: "The text to embed and store." },
|
|
1379
|
+
summary: { type: "string", description: "Optional short label." },
|
|
1380
|
+
tags: {
|
|
1381
|
+
type: "array",
|
|
1382
|
+
items: { type: "string" },
|
|
1383
|
+
description: "Optional tags for later filtering."
|
|
1384
|
+
},
|
|
1385
|
+
scope: {
|
|
1386
|
+
type: "string",
|
|
1387
|
+
enum: ["project", "user", "session"],
|
|
1388
|
+
description: "Visibility scope. Defaults to `project`."
|
|
1389
|
+
},
|
|
1390
|
+
kind: {
|
|
1391
|
+
type: "string",
|
|
1392
|
+
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1393
|
+
description: "Entry kind. Defaults to `note`."
|
|
1394
|
+
},
|
|
1395
|
+
metadata: {
|
|
1396
|
+
type: "object",
|
|
1397
|
+
additionalProperties: true,
|
|
1398
|
+
description: "Free-form metadata persisted as JSON."
|
|
1399
|
+
}
|
|
1400
|
+
},
|
|
1401
|
+
required: ["text"],
|
|
1402
|
+
additionalProperties: false
|
|
1403
|
+
},
|
|
1404
|
+
execute: async (input) => {
|
|
1405
|
+
const entry = await store.remember(input);
|
|
1406
|
+
return { id: entry.id, hasVector: entry.vector !== void 0 };
|
|
1426
1407
|
}
|
|
1427
1408
|
};
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1409
|
+
}
|
|
1410
|
+
function vectorMemorySearchTool(store) {
|
|
1411
|
+
return {
|
|
1412
|
+
name: "vector_memory_search",
|
|
1413
|
+
category: "Memory",
|
|
1414
|
+
description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
|
|
1415
|
+
usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
|
|
1416
|
+
permission: "auto",
|
|
1417
|
+
mutating: false,
|
|
1418
|
+
riskTier: "safe",
|
|
1419
|
+
timeoutMs: 5e3,
|
|
1420
|
+
capabilities: ["memory.read"],
|
|
1421
|
+
icon: "search",
|
|
1422
|
+
inputSchema: {
|
|
1423
|
+
type: "object",
|
|
1424
|
+
properties: {
|
|
1425
|
+
query: { type: "string", minLength: 1, description: "The natural-language query." },
|
|
1426
|
+
limit: {
|
|
1427
|
+
type: "number",
|
|
1428
|
+
minimum: 1,
|
|
1429
|
+
maximum: 100,
|
|
1430
|
+
description: "Max results (default 10)."
|
|
1431
|
+
},
|
|
1432
|
+
threshold: {
|
|
1433
|
+
type: "number",
|
|
1434
|
+
minimum: 0,
|
|
1435
|
+
maximum: 1,
|
|
1436
|
+
description: "Minimum cosine similarity. Results below the floor are dropped."
|
|
1437
|
+
},
|
|
1438
|
+
scope: {
|
|
1439
|
+
type: "string",
|
|
1440
|
+
enum: ["project", "user", "session"],
|
|
1441
|
+
description: "Restrict to a scope."
|
|
1442
|
+
},
|
|
1443
|
+
kind: {
|
|
1444
|
+
type: "string",
|
|
1445
|
+
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1446
|
+
description: "Restrict to a kind."
|
|
1448
1447
|
}
|
|
1448
|
+
},
|
|
1449
|
+
required: ["query"],
|
|
1450
|
+
additionalProperties: false
|
|
1451
|
+
},
|
|
1452
|
+
execute: async (input) => {
|
|
1453
|
+
const hits = await store.search(input.query, {
|
|
1454
|
+
limit: input.limit !== void 0 ? input.limit : void 0,
|
|
1455
|
+
threshold: input.threshold !== void 0 ? input.threshold : void 0,
|
|
1456
|
+
scope: input.scope,
|
|
1457
|
+
kind: input.kind
|
|
1449
1458
|
});
|
|
1450
|
-
|
|
1451
|
-
|
|
1459
|
+
return {
|
|
1460
|
+
hits: hits.map((h) => ({
|
|
1461
|
+
id: h.entry.id,
|
|
1462
|
+
score: h.score,
|
|
1463
|
+
text: h.entry.text,
|
|
1464
|
+
summary: h.entry.summary ?? void 0,
|
|
1465
|
+
tags: h.entry.tags
|
|
1466
|
+
}))
|
|
1467
|
+
};
|
|
1452
1468
|
}
|
|
1453
1469
|
};
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1470
|
+
}
|
|
1471
|
+
function vectorMemoryStatsTool(store) {
|
|
1472
|
+
return {
|
|
1473
|
+
name: "vector_memory_stats",
|
|
1474
|
+
category: "Memory",
|
|
1475
|
+
description: "Return counts, providers, and dimensions for the local vector memory store.",
|
|
1476
|
+
usageHint: "Cheap diagnostic \u2014 safe to call any time.",
|
|
1477
|
+
permission: "auto",
|
|
1478
|
+
mutating: false,
|
|
1479
|
+
riskTier: "safe",
|
|
1480
|
+
timeoutMs: 1e3,
|
|
1481
|
+
capabilities: ["memory.read"],
|
|
1482
|
+
icon: "search",
|
|
1483
|
+
inputSchema: {
|
|
1484
|
+
type: "object",
|
|
1485
|
+
properties: {},
|
|
1486
|
+
additionalProperties: false
|
|
1487
|
+
},
|
|
1488
|
+
execute: async () => store.stats()
|
|
1461
1489
|
};
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
if (typeof memoryId !== "string") return;
|
|
1465
|
-
void mirror(memoryId);
|
|
1466
|
-
});
|
|
1467
|
-
const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
|
|
1468
|
-
const memoryId = payload?.memoryId;
|
|
1469
|
-
if (typeof memoryId !== "string") return;
|
|
1470
|
-
void mirror(memoryId);
|
|
1471
|
-
});
|
|
1472
|
-
const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
|
|
1473
|
-
const memoryId = payload?.memoryId;
|
|
1474
|
-
if (typeof memoryId !== "string") return;
|
|
1475
|
-
const status = payload?.status;
|
|
1476
|
-
if (status === "deleted") return;
|
|
1477
|
-
void mirror(memoryId);
|
|
1478
|
-
});
|
|
1479
|
-
const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
|
|
1480
|
-
const memoryId = payload?.memoryId;
|
|
1481
|
-
if (typeof memoryId !== "string") return;
|
|
1482
|
-
void forgetMirror(memoryId);
|
|
1483
|
-
});
|
|
1490
|
+
}
|
|
1491
|
+
function vectorMemoryForgetTool(store) {
|
|
1484
1492
|
return {
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1493
|
+
name: "vector_memory_forget",
|
|
1494
|
+
category: "Memory",
|
|
1495
|
+
description: "Remove an entry (and its vector) from the vector memory store.",
|
|
1496
|
+
usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
|
|
1497
|
+
permission: "confirm",
|
|
1498
|
+
mutating: true,
|
|
1499
|
+
riskTier: "standard",
|
|
1500
|
+
timeoutMs: 1e3,
|
|
1501
|
+
capabilities: ["memory.write"],
|
|
1502
|
+
icon: "settings",
|
|
1503
|
+
inputSchema: {
|
|
1504
|
+
type: "object",
|
|
1505
|
+
properties: {
|
|
1506
|
+
id: {
|
|
1507
|
+
type: "string",
|
|
1508
|
+
minLength: 1,
|
|
1509
|
+
description: "Entry id returned by `vector_memory_remember`."
|
|
1510
|
+
}
|
|
1511
|
+
},
|
|
1512
|
+
required: ["id"],
|
|
1513
|
+
additionalProperties: false
|
|
1514
|
+
},
|
|
1515
|
+
execute: async (input) => ({ removed: await store.forget(input.id) })
|
|
1491
1516
|
};
|
|
1492
1517
|
}
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1518
|
+
|
|
1519
|
+
// src/transformers-provider.ts
|
|
1520
|
+
var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
1521
|
+
var DEFAULT_VECTOR_DIMENSIONS = 384;
|
|
1522
|
+
var DEFAULT_VECTOR_DTYPE = "q8";
|
|
1523
|
+
var TransformersEmbeddingProvider = class {
|
|
1524
|
+
id;
|
|
1525
|
+
dimensions;
|
|
1526
|
+
modelId;
|
|
1527
|
+
cacheDir;
|
|
1528
|
+
dtype;
|
|
1529
|
+
device;
|
|
1530
|
+
batchSize;
|
|
1531
|
+
maxChars;
|
|
1532
|
+
allowRemote;
|
|
1533
|
+
extractor;
|
|
1534
|
+
loadPromise;
|
|
1535
|
+
constructor(opts = {}) {
|
|
1536
|
+
this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
|
|
1537
|
+
this.cacheDir = opts.cacheDir;
|
|
1538
|
+
this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
|
|
1539
|
+
this.device = opts.device ?? "cpu";
|
|
1540
|
+
this.batchSize = opts.batchSize ?? 16;
|
|
1541
|
+
this.maxChars = opts.maxChars ?? 2048;
|
|
1542
|
+
this.allowRemote = opts.allowRemoteModels ?? true;
|
|
1543
|
+
this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
|
|
1544
|
+
this.id = `transformers-js:${this.modelId}:${this.dtype}`;
|
|
1519
1545
|
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
async
|
|
1528
|
-
const markerPath2 = path3.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
|
|
1529
|
-
const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
1530
|
-
if (!opts.force) {
|
|
1546
|
+
/**
|
|
1547
|
+
* Synchronous capability check. Returns false when the optional
|
|
1548
|
+
* `@huggingface/transformers` dependency is not installed.
|
|
1549
|
+
*
|
|
1550
|
+
* NOTE: this probes via dynamic import and caches the result, but does
|
|
1551
|
+
* NOT load the model itself — model loading is deferred to `embed()`.
|
|
1552
|
+
*/
|
|
1553
|
+
async isAvailable() {
|
|
1531
1554
|
try {
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
if (Number.isFinite(at) && Date.now() - at < interval) {
|
|
1535
|
-
return { swept: false, reason: "throttled" };
|
|
1536
|
-
}
|
|
1555
|
+
await this.loadModule();
|
|
1556
|
+
return true;
|
|
1537
1557
|
} catch {
|
|
1558
|
+
return false;
|
|
1538
1559
|
}
|
|
1539
1560
|
}
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
// src/search-race.ts
|
|
1557
|
-
function previewText(text, maxLen) {
|
|
1558
|
-
if (text.length <= maxLen) return text;
|
|
1559
|
-
return text.slice(0, maxLen - 1) + "\u2026";
|
|
1560
|
-
}
|
|
1561
|
-
async function runSearchRace(query, lexical, vectorStore, options = {}) {
|
|
1562
|
-
const limit = options.limit ?? 20;
|
|
1563
|
-
const threshold = options.threshold ?? 0;
|
|
1564
|
-
let vectorHits = [];
|
|
1565
|
-
try {
|
|
1566
|
-
vectorHits = await vectorStore.search(query, {
|
|
1567
|
-
limit,
|
|
1568
|
-
...threshold > 0 ? { threshold } : {}
|
|
1569
|
-
});
|
|
1570
|
-
} catch {
|
|
1571
|
-
vectorHits = [];
|
|
1561
|
+
async embed(texts) {
|
|
1562
|
+
if (texts.length === 0) return [];
|
|
1563
|
+
const extractor = await this.getExtractor();
|
|
1564
|
+
const prepared = texts.map((t) => this.prepare(t));
|
|
1565
|
+
const batches = [];
|
|
1566
|
+
for (let i = 0; i < prepared.length; i += this.batchSize) {
|
|
1567
|
+
batches.push(prepared.slice(i, i + this.batchSize));
|
|
1568
|
+
}
|
|
1569
|
+
const results = [];
|
|
1570
|
+
for (const batch of batches) {
|
|
1571
|
+
const out = await extractor(batch, { pooling: "mean", normalize: true });
|
|
1572
|
+
results.push(...this.tensorToVectors(out, batch.length));
|
|
1573
|
+
}
|
|
1574
|
+
return results;
|
|
1572
1575
|
}
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
for (let i = 0; i < lexicalCapped.length; i++) {
|
|
1579
|
-
const mem = lexicalCapped[i];
|
|
1580
|
-
const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
|
|
1581
|
-
const id = mem.id;
|
|
1582
|
-
seenIds.add(id);
|
|
1583
|
-
overlap.push({
|
|
1584
|
-
id,
|
|
1585
|
-
lexicalScore: score,
|
|
1586
|
-
vectorScore: null,
|
|
1587
|
-
// patched below when a vector hit carries this id
|
|
1588
|
-
preview: previewText(mem.text, 140)
|
|
1589
|
-
});
|
|
1576
|
+
/** Truncate + normalize text before embedding. */
|
|
1577
|
+
prepare(text) {
|
|
1578
|
+
if (!text) return "";
|
|
1579
|
+
const normalized = text.normalize("NFKC").trim();
|
|
1580
|
+
return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
|
|
1590
1581
|
}
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
const existing = overlapById.get(sageId);
|
|
1597
|
-
if (existing) {
|
|
1598
|
-
if (existing.vectorScore === null) {
|
|
1599
|
-
existing.vectorScore = hit.score;
|
|
1582
|
+
tensorToVectors(out, batchSize) {
|
|
1583
|
+
if (typeof out.tolist === "function") {
|
|
1584
|
+
const nested = out.tolist();
|
|
1585
|
+
if (Array.isArray(nested) && Array.isArray(nested[0])) {
|
|
1586
|
+
return nested.map((row) => Float32Array.from(row));
|
|
1600
1587
|
}
|
|
1601
|
-
|
|
1588
|
+
return [Float32Array.from(nested)];
|
|
1602
1589
|
}
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1590
|
+
const flat = out.data;
|
|
1591
|
+
if (flat instanceof Float32Array) {
|
|
1592
|
+
if (batchSize === 1) return [flat];
|
|
1593
|
+
const dim = flat.length / batchSize;
|
|
1594
|
+
const vectors = [];
|
|
1595
|
+
for (let i = 0; i < batchSize; i++) {
|
|
1596
|
+
vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
|
|
1597
|
+
}
|
|
1598
|
+
return vectors;
|
|
1599
|
+
}
|
|
1600
|
+
if (Array.isArray(flat) && Array.isArray(flat[0])) {
|
|
1601
|
+
return flat.map((row) => Float32Array.from(row));
|
|
1602
|
+
}
|
|
1603
|
+
if (Array.isArray(flat)) {
|
|
1604
|
+
return [Float32Array.from(flat)];
|
|
1605
|
+
}
|
|
1606
|
+
throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
|
|
1607
|
+
}
|
|
1608
|
+
async getExtractor() {
|
|
1609
|
+
if (this.extractor) return this.extractor;
|
|
1610
|
+
if (!this.loadPromise) this.loadPromise = this.loadExtractor();
|
|
1611
|
+
this.extractor = await this.loadPromise;
|
|
1612
|
+
return this.extractor;
|
|
1613
|
+
}
|
|
1614
|
+
async loadExtractor() {
|
|
1615
|
+
const mod = await this.loadModule();
|
|
1616
|
+
if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
|
|
1617
|
+
mod.env.allowRemoteModels = this.allowRemote;
|
|
1618
|
+
if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
|
|
1619
|
+
const pipe = await mod.pipeline("feature-extraction", this.modelId, {
|
|
1620
|
+
dtype: this.dtype,
|
|
1621
|
+
device: this.device
|
|
1610
1622
|
});
|
|
1623
|
+
return pipe;
|
|
1611
1624
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
} else {
|
|
1622
|
-
finalOverlap.push(row);
|
|
1625
|
+
async loadModule() {
|
|
1626
|
+
try {
|
|
1627
|
+
const packageName = "@huggingface/transformers";
|
|
1628
|
+
return await import(packageName);
|
|
1629
|
+
} catch (err) {
|
|
1630
|
+
throw new VectorMemoryProviderUnavailableError(
|
|
1631
|
+
"@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
|
|
1632
|
+
err
|
|
1633
|
+
);
|
|
1623
1634
|
}
|
|
1624
1635
|
}
|
|
1625
|
-
|
|
1626
|
-
const vectorCount = vectorOnly.length + finalOverlap.length;
|
|
1627
|
-
const denom = Math.max(lexicalCount, vectorCount, 1);
|
|
1628
|
-
return {
|
|
1629
|
-
query,
|
|
1630
|
-
lexicalOnly,
|
|
1631
|
-
vectorOnly,
|
|
1632
|
-
overlap: finalOverlap,
|
|
1633
|
-
metrics: {
|
|
1634
|
-
lexicalCount,
|
|
1635
|
-
vectorCount,
|
|
1636
|
-
overlapCount: finalOverlap.length,
|
|
1637
|
-
lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
|
|
1638
|
-
vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
|
|
1639
|
-
agreementRatio: finalOverlap.length / denom
|
|
1640
|
-
}
|
|
1641
|
-
};
|
|
1642
|
-
}
|
|
1636
|
+
};
|
|
1643
1637
|
export {
|
|
1644
1638
|
DEFAULT_SWEEP_INTERVAL_MS,
|
|
1645
1639
|
DEFAULT_VECTOR_DIMENSIONS,
|