@wrongstack/vector-memory 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +9 -9
- package/dist/index.js +1515 -1522
- package/dist/sage-sync.d.ts +30 -0
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -14,1632 +14,1625 @@ var VectorMemoryProviderUnavailableError = class extends VectorMemoryError {
|
|
|
14
14
|
}
|
|
15
15
|
};
|
|
16
16
|
|
|
17
|
-
// src/
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
dtype;
|
|
27
|
-
device;
|
|
28
|
-
batchSize;
|
|
29
|
-
maxChars;
|
|
30
|
-
allowRemote;
|
|
31
|
-
extractor;
|
|
32
|
-
loadPromise;
|
|
33
|
-
constructor(opts = {}) {
|
|
34
|
-
this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
|
|
35
|
-
this.cacheDir = opts.cacheDir;
|
|
36
|
-
this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
|
|
37
|
-
this.device = opts.device ?? "cpu";
|
|
38
|
-
this.batchSize = opts.batchSize ?? 16;
|
|
39
|
-
this.maxChars = opts.maxChars ?? 2048;
|
|
40
|
-
this.allowRemote = opts.allowRemoteModels ?? true;
|
|
41
|
-
this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
|
|
42
|
-
this.id = `transformers-js:${this.modelId}:${this.dtype}`;
|
|
17
|
+
// src/sage-event-mirror.ts
|
|
18
|
+
import * as fs from "node:fs";
|
|
19
|
+
import * as path from "node:path";
|
|
20
|
+
import { getSageSurface } from "@wrongstack/sage";
|
|
21
|
+
function subscribeVectorMemoryToSage(opts) {
|
|
22
|
+
const { store, memoryStore } = opts;
|
|
23
|
+
const log = opts.logger;
|
|
24
|
+
if (opts.enabled === false) {
|
|
25
|
+
return { dispose: () => void 0 };
|
|
43
26
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
27
|
+
const surface = getSageSurface(memoryStore);
|
|
28
|
+
if (!surface) {
|
|
29
|
+
log?.debug?.("vector-memory mirror disabled: memory store exposes no SAGE surface");
|
|
30
|
+
return { dispose: () => void 0 };
|
|
31
|
+
}
|
|
32
|
+
const events = memoryStore.events;
|
|
33
|
+
if (!events) {
|
|
34
|
+
log?.debug?.("vector-memory mirror disabled: memory store has no event bus");
|
|
35
|
+
return { dispose: () => void 0 };
|
|
36
|
+
}
|
|
37
|
+
const fetch = async (memoryId) => {
|
|
52
38
|
try {
|
|
53
|
-
await
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
return
|
|
39
|
+
return await surface.getSage(memoryId);
|
|
40
|
+
} catch (err) {
|
|
41
|
+
log?.warn?.(`vector-memory mirror fetch failed for ${memoryId}: ${errMsg(err)}`);
|
|
42
|
+
return null;
|
|
57
43
|
}
|
|
58
|
-
}
|
|
59
|
-
async
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
44
|
+
};
|
|
45
|
+
const mirror = async (memoryId) => {
|
|
46
|
+
const memory = await fetch(memoryId);
|
|
47
|
+
if (!memory) return;
|
|
48
|
+
if (memory.scope === "session") return;
|
|
49
|
+
try {
|
|
50
|
+
const existing = store.findBySageId(memoryId);
|
|
51
|
+
if (existing) await store.forget(existing.id);
|
|
52
|
+
await store.remember({
|
|
53
|
+
text: memory.text,
|
|
54
|
+
...memory.summary ? { summary: memory.summary } : {},
|
|
55
|
+
tags: memory.tags ?? [],
|
|
56
|
+
scope: "project",
|
|
57
|
+
kind: "note",
|
|
58
|
+
metadata: {
|
|
59
|
+
source: "sage",
|
|
60
|
+
sageId: memory.id,
|
|
61
|
+
sageKind: memory.kind,
|
|
62
|
+
sageScope: memory.scope,
|
|
63
|
+
importance: memory.importance,
|
|
64
|
+
confidence: memory.confidence
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
} catch (err) {
|
|
68
|
+
log?.warn?.(`vector-memory mirror remember failed for ${memoryId}: ${errMsg(err)}`);
|
|
66
69
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
};
|
|
71
|
+
const forgetMirror = async (memoryId) => {
|
|
72
|
+
try {
|
|
73
|
+
const existing = store.findBySageId(memoryId);
|
|
74
|
+
if (existing) await store.forget(existing.id);
|
|
75
|
+
} catch (err) {
|
|
76
|
+
log?.warn?.(`vector-memory mirror forget failed for ${memoryId}: ${errMsg(err)}`);
|
|
71
77
|
}
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
78
|
+
};
|
|
79
|
+
const offAccepted = events.onPattern("memory.accepted", (_event, payload) => {
|
|
80
|
+
const memoryId = payload?.memoryId;
|
|
81
|
+
if (typeof memoryId !== "string") return;
|
|
82
|
+
void mirror(memoryId);
|
|
83
|
+
});
|
|
84
|
+
const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
|
|
85
|
+
const memoryId = payload?.memoryId;
|
|
86
|
+
if (typeof memoryId !== "string") return;
|
|
87
|
+
void mirror(memoryId);
|
|
88
|
+
});
|
|
89
|
+
const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
|
|
90
|
+
const memoryId = payload?.memoryId;
|
|
91
|
+
if (typeof memoryId !== "string") return;
|
|
92
|
+
const status = payload?.status;
|
|
93
|
+
if (status === "deleted") return;
|
|
94
|
+
void mirror(memoryId);
|
|
95
|
+
});
|
|
96
|
+
const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
|
|
97
|
+
const memoryId = payload?.memoryId;
|
|
98
|
+
if (typeof memoryId !== "string") return;
|
|
99
|
+
void forgetMirror(memoryId);
|
|
100
|
+
});
|
|
101
|
+
return {
|
|
102
|
+
dispose: () => {
|
|
103
|
+
offAccepted();
|
|
104
|
+
offRecovered();
|
|
105
|
+
offUpdated();
|
|
106
|
+
offDeleted();
|
|
87
107
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
|
|
111
|
+
const surface = getSageSurface(memoryStore);
|
|
112
|
+
if (!surface) return { scanned: 0, removed: 0 };
|
|
113
|
+
let scanned = 0;
|
|
114
|
+
let removed = 0;
|
|
115
|
+
const PAGE = Math.max(1, options?.pageSize ?? 500);
|
|
116
|
+
let after;
|
|
117
|
+
for (; ; ) {
|
|
118
|
+
const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
|
|
119
|
+
if (page.length === 0) break;
|
|
120
|
+
const last = page[page.length - 1];
|
|
121
|
+
after = { updatedAt: last.updatedAt, id: last.id };
|
|
122
|
+
for (const entry of page) {
|
|
123
|
+
scanned++;
|
|
124
|
+
const sageId = entry.metadata?.sageId;
|
|
125
|
+
if (typeof sageId !== "string") continue;
|
|
126
|
+
try {
|
|
127
|
+
const memory = await surface.getSage(sageId);
|
|
128
|
+
if (memory !== null && memory.status !== "deleted") continue;
|
|
129
|
+
await store.forget(entry.id);
|
|
130
|
+
removed++;
|
|
131
|
+
} catch (err) {
|
|
132
|
+
logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg(err)}`);
|
|
95
133
|
}
|
|
96
|
-
return vectors;
|
|
97
|
-
}
|
|
98
|
-
if (Array.isArray(flat) && Array.isArray(flat[0])) {
|
|
99
|
-
return flat.map((row) => Float32Array.from(row));
|
|
100
134
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
135
|
+
if (page.length < PAGE) break;
|
|
136
|
+
}
|
|
137
|
+
return { scanned, removed };
|
|
138
|
+
}
|
|
139
|
+
function errMsg(err) {
|
|
140
|
+
return err instanceof Error ? err.message : String(err);
|
|
141
|
+
}
|
|
142
|
+
var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
|
|
143
|
+
var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
|
|
144
|
+
async function sweepStaleSageMirrors(opts) {
|
|
145
|
+
const markerPath2 = path.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
|
|
146
|
+
const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
147
|
+
if (!opts.force) {
|
|
148
|
+
try {
|
|
149
|
+
const raw = JSON.parse(fs.readFileSync(markerPath2, "utf8"));
|
|
150
|
+
const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
|
|
151
|
+
if (Number.isFinite(at) && Date.now() - at < interval) {
|
|
152
|
+
return { swept: false, reason: "throttled" };
|
|
153
|
+
}
|
|
154
|
+
} catch {
|
|
103
155
|
}
|
|
104
|
-
throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
|
|
105
156
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
this.extractor = await this.loadPromise;
|
|
110
|
-
return this.extractor;
|
|
157
|
+
try {
|
|
158
|
+
fs.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
|
|
159
|
+
} catch {
|
|
111
160
|
}
|
|
112
|
-
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
return pipe;
|
|
161
|
+
try {
|
|
162
|
+
const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
|
|
163
|
+
opts.logger?.debug?.(
|
|
164
|
+
`vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
|
|
165
|
+
);
|
|
166
|
+
return { swept: true, ...result };
|
|
167
|
+
} catch (err) {
|
|
168
|
+
opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg(err)}`);
|
|
169
|
+
return { swept: false, reason: errMsg(err) };
|
|
122
170
|
}
|
|
123
|
-
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// src/sage-fusion.ts
|
|
174
|
+
var DEFAULT_RRF_K = 60;
|
|
175
|
+
var DEFAULT_VECTOR_WEIGHT = 0.3;
|
|
176
|
+
function lexicalRankScore(index, total) {
|
|
177
|
+
if (total <= 1) return 1;
|
|
178
|
+
return 1 - index / Math.max(1, total - 1);
|
|
179
|
+
}
|
|
180
|
+
async function fuseWithVectorMemory(query, lexical, options = {}) {
|
|
181
|
+
const weight = clamp01(options.vectorWeight ?? DEFAULT_VECTOR_WEIGHT);
|
|
182
|
+
const k = options.rrfK ?? DEFAULT_RRF_K;
|
|
183
|
+
const limit = options.limit ?? 25;
|
|
184
|
+
let vectorHits = [];
|
|
185
|
+
if (options.vectorHits) {
|
|
186
|
+
vectorHits = [...options.vectorHits];
|
|
187
|
+
} else if (options.store) {
|
|
124
188
|
try {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
189
|
+
vectorHits = await options.store.search(query, {
|
|
190
|
+
...options.threshold !== void 0 ? { threshold: options.threshold } : {},
|
|
191
|
+
limit: Math.max(limit * 2, 50)
|
|
192
|
+
});
|
|
193
|
+
} catch {
|
|
194
|
+
vectorHits = [];
|
|
131
195
|
}
|
|
132
196
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
);
|
|
150
|
-
`);
|
|
151
|
-
db.exec(`
|
|
152
|
-
CREATE TABLE IF NOT EXISTS entries (
|
|
153
|
-
id TEXT PRIMARY KEY,
|
|
154
|
-
text TEXT NOT NULL,
|
|
155
|
-
summary TEXT,
|
|
156
|
-
metadata TEXT NOT NULL DEFAULT '{}',
|
|
157
|
-
tags TEXT NOT NULL DEFAULT '[]',
|
|
158
|
-
scope TEXT NOT NULL DEFAULT 'project',
|
|
159
|
-
kind TEXT NOT NULL DEFAULT 'note',
|
|
160
|
-
content_hash TEXT NOT NULL,
|
|
161
|
-
created_at TEXT NOT NULL,
|
|
162
|
-
updated_at TEXT NOT NULL
|
|
163
|
-
);
|
|
164
|
-
`);
|
|
165
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
|
|
166
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
|
|
167
|
-
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
|
|
168
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
|
|
169
|
-
db.exec(`
|
|
170
|
-
CREATE TABLE IF NOT EXISTS vectors (
|
|
171
|
-
entry_id TEXT NOT NULL,
|
|
172
|
-
provider_id TEXT NOT NULL,
|
|
173
|
-
dimensions INTEGER NOT NULL,
|
|
174
|
-
vector BLOB NOT NULL,
|
|
175
|
-
created_at TEXT NOT NULL,
|
|
176
|
-
PRIMARY KEY (entry_id, provider_id),
|
|
177
|
-
FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
|
|
178
|
-
);
|
|
179
|
-
`);
|
|
180
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
|
|
181
|
-
db.exec(`
|
|
182
|
-
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
183
|
-
content_hash TEXT NOT NULL,
|
|
184
|
-
provider_id TEXT NOT NULL,
|
|
185
|
-
dimensions INTEGER NOT NULL,
|
|
186
|
-
vector BLOB NOT NULL,
|
|
187
|
-
text TEXT NOT NULL,
|
|
188
|
-
created_at TEXT NOT NULL,
|
|
189
|
-
last_used_at TEXT NOT NULL,
|
|
190
|
-
use_count INTEGER NOT NULL DEFAULT 0,
|
|
191
|
-
PRIMARY KEY (content_hash, provider_id, dimensions)
|
|
192
|
-
);
|
|
193
|
-
`);
|
|
194
|
-
db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
|
|
195
|
-
db.exec(`
|
|
196
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
|
|
197
|
-
id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
|
|
198
|
-
);
|
|
199
|
-
`);
|
|
200
|
-
db.exec(`
|
|
201
|
-
CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
|
|
202
|
-
INSERT INTO entries_fts(rowid, text, tags)
|
|
203
|
-
VALUES (new.rowid, new.text, new.tags);
|
|
204
|
-
END;
|
|
205
|
-
`);
|
|
206
|
-
db.exec(`
|
|
207
|
-
CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
|
|
208
|
-
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
209
|
-
VALUES('delete', old.rowid, old.text, old.tags);
|
|
210
|
-
END;
|
|
211
|
-
`);
|
|
212
|
-
db.exec(`
|
|
213
|
-
CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
|
|
214
|
-
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
215
|
-
VALUES('delete', old.rowid, old.text, old.tags);
|
|
216
|
-
INSERT INTO entries_fts(rowid, text, tags)
|
|
217
|
-
VALUES (new.rowid, new.text, new.tags);
|
|
218
|
-
END;
|
|
219
|
-
`);
|
|
220
|
-
}
|
|
221
|
-
function upsertEmbeddingCache(db, row) {
|
|
222
|
-
db.prepare(
|
|
223
|
-
`INSERT INTO embedding_cache
|
|
224
|
-
(content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
|
|
225
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, 1)
|
|
226
|
-
ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
|
|
227
|
-
last_used_at = excluded.last_used_at,
|
|
228
|
-
use_count = embedding_cache.use_count + 1,
|
|
229
|
-
vector = excluded.vector`
|
|
230
|
-
).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
|
|
231
|
-
}
|
|
232
|
-
function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
|
|
233
|
-
const row = db.prepare(
|
|
234
|
-
`SELECT vector FROM embedding_cache
|
|
235
|
-
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
|
|
236
|
-
LIMIT 1`
|
|
237
|
-
).get(contentHash, providerId, dimensions);
|
|
238
|
-
if (!row) return void 0;
|
|
239
|
-
try {
|
|
240
|
-
db.prepare(
|
|
241
|
-
`UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
|
|
242
|
-
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
|
|
243
|
-
).run(now, contentHash, providerId, dimensions);
|
|
244
|
-
} catch {
|
|
197
|
+
const sageById = /* @__PURE__ */ new Map();
|
|
198
|
+
for (const memory of lexical) sageById.set(memory.id, memory);
|
|
199
|
+
const lexicalRanked = lexical.map((memory, index) => ({
|
|
200
|
+
memory,
|
|
201
|
+
rankScore: lexicalRankScore(index, lexical.length),
|
|
202
|
+
lexicalScore: lexicalRankScore(index, lexical.length)
|
|
203
|
+
}));
|
|
204
|
+
const vectorRanked = [];
|
|
205
|
+
const seenVectorSageIds = /* @__PURE__ */ new Set();
|
|
206
|
+
for (let i = 0; i < vectorHits.length; i++) {
|
|
207
|
+
const hit = vectorHits[i];
|
|
208
|
+
const sageId = hit.entry.metadata?.["sageId"];
|
|
209
|
+
if (typeof sageId !== "string" || seenVectorSageIds.has(sageId)) continue;
|
|
210
|
+
const memory = sageById.get(sageId);
|
|
211
|
+
if (!memory) continue;
|
|
212
|
+
seenVectorSageIds.add(sageId);
|
|
213
|
+
vectorRanked.push({ memory, vectorScore: hit.score });
|
|
245
214
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
215
|
+
const fused = /* @__PURE__ */ new Map();
|
|
216
|
+
for (let i = 0; i < lexicalRanked.length; i++) {
|
|
217
|
+
const c = lexicalRanked[i];
|
|
218
|
+
const rrf = (1 - weight) * (1 / (k + i + 1));
|
|
219
|
+
fused.set(c.memory.id, {
|
|
220
|
+
memory: c.memory,
|
|
221
|
+
vectorScore: null,
|
|
222
|
+
lexicalScore: c.lexicalScore,
|
|
223
|
+
finalScore: rrf,
|
|
224
|
+
source: "lexical"
|
|
225
|
+
});
|
|
256
226
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
227
|
+
for (let i = 0; i < vectorRanked.length; i++) {
|
|
228
|
+
const v = vectorRanked[i];
|
|
229
|
+
const rrf = weight * (1 / (k + i + 1));
|
|
230
|
+
const existing = fused.get(v.memory.id);
|
|
231
|
+
if (existing) {
|
|
232
|
+
existing.finalScore += rrf;
|
|
233
|
+
existing.vectorScore = v.vectorScore;
|
|
234
|
+
existing.source = "both";
|
|
235
|
+
}
|
|
261
236
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
// src/store.ts
|
|
266
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
267
|
-
import * as fs from "node:fs";
|
|
268
|
-
import * as path from "node:path";
|
|
269
|
-
import { withFileLock } from "@wrongstack/core/utils";
|
|
270
|
-
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
271
|
-
import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
|
|
272
|
-
var DEFAULT_SEARCH_LIMIT = 10;
|
|
273
|
-
function normalizeLimit(limit) {
|
|
274
|
-
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
|
|
275
|
-
const floored = Math.floor(limit);
|
|
276
|
-
return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
|
|
237
|
+
const out = Array.from(fused.values());
|
|
238
|
+
out.sort((a, b) => b.finalScore - a.finalScore);
|
|
239
|
+
return out.slice(0, limit);
|
|
277
240
|
}
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
const filename = opts.filename ?? DEFAULT_FILENAME;
|
|
294
|
-
if (path.isAbsolute(dir)) {
|
|
295
|
-
throw new Error("Vector memory directory must be project-relative.");
|
|
241
|
+
function asVectorRecallProvider(store) {
|
|
242
|
+
return {
|
|
243
|
+
async search(query, opts) {
|
|
244
|
+
const hits = await store.search(query, {
|
|
245
|
+
limit: opts.limit,
|
|
246
|
+
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
247
|
+
});
|
|
248
|
+
return hits.map((h) => ({
|
|
249
|
+
id: h.entry.id,
|
|
250
|
+
score: h.score,
|
|
251
|
+
text: h.entry.text,
|
|
252
|
+
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
253
|
+
tags: h.entry.tags,
|
|
254
|
+
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
255
|
+
}));
|
|
296
256
|
}
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
function clamp01(value) {
|
|
260
|
+
if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
|
|
261
|
+
if (value < 0) return 0;
|
|
262
|
+
if (value > 1) return 1;
|
|
263
|
+
return value;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// src/sage-port-wrapper.ts
|
|
267
|
+
import {
|
|
268
|
+
augmentLexicalWithVectorRecall,
|
|
269
|
+
isSageVisibleForSearch,
|
|
270
|
+
SAGE_RETRIEVAL_CAPABILITY,
|
|
271
|
+
SAGE_SURFACE_CAPABILITY
|
|
272
|
+
} from "@wrongstack/sage";
|
|
273
|
+
function asVectorRecallProviderAdapter(store) {
|
|
274
|
+
return {
|
|
275
|
+
async search(query, opts) {
|
|
276
|
+
const hits = await store.search(query, {
|
|
277
|
+
limit: opts.limit,
|
|
278
|
+
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
279
|
+
});
|
|
280
|
+
return hits.map((h) => ({
|
|
281
|
+
id: h.entry.id,
|
|
282
|
+
score: h.score,
|
|
283
|
+
text: h.entry.text,
|
|
284
|
+
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
285
|
+
tags: h.entry.tags,
|
|
286
|
+
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
287
|
+
}));
|
|
301
288
|
}
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
this.db.prepare(
|
|
339
|
-
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
340
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
341
|
-
).run(VECTOR_PROVIDER_KEY, this.provider.id);
|
|
342
|
-
this.db.prepare(
|
|
343
|
-
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
344
|
-
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
345
|
-
).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
|
|
346
|
-
this.db.exec("COMMIT");
|
|
347
|
-
} catch (e) {
|
|
348
|
-
this.db.exec("ROLLBACK");
|
|
349
|
-
throw e;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
static contentHash(text) {
|
|
353
|
-
return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
|
|
354
|
-
}
|
|
355
|
-
/**
|
|
356
|
-
* Look up a vector for `text` in the provider-level embedding cache.
|
|
357
|
-
* Cache hit returns the cached vector (no ONNX pass). Miss returns
|
|
358
|
-
* `undefined`.
|
|
359
|
-
*/
|
|
360
|
-
cachedVector(text, now) {
|
|
361
|
-
return lookupEmbeddingCache(
|
|
362
|
-
this.db,
|
|
363
|
-
_VectorMemoryStore.contentHash(text),
|
|
364
|
-
this.provider.id,
|
|
365
|
-
this.provider.dimensions,
|
|
366
|
-
now
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
function wrapMemoryPortWithVectorRecall(port, options) {
|
|
292
|
+
const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
|
|
293
|
+
const materializeFor = (searchOpts) => async (sageId) => {
|
|
294
|
+
const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
|
|
295
|
+
if (!surface?.getSage) return void 0;
|
|
296
|
+
const memory = await surface.getSage(sageId);
|
|
297
|
+
if (!memory) return void 0;
|
|
298
|
+
return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
|
|
299
|
+
};
|
|
300
|
+
const fusionOptions = (searchOpts) => ({
|
|
301
|
+
vectorRecall: recall,
|
|
302
|
+
materializeVectorOnly: materializeFor(searchOpts),
|
|
303
|
+
...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
|
|
304
|
+
...options.threshold !== void 0 ? { threshold: options.threshold } : {},
|
|
305
|
+
...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
|
|
306
|
+
...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
|
|
307
|
+
...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
|
|
308
|
+
});
|
|
309
|
+
const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
|
|
310
|
+
const wrapSearchSage = (original) => async (query, searchOpts) => {
|
|
311
|
+
const opts = searchOpts;
|
|
312
|
+
const lexical = await original(query, searchOpts);
|
|
313
|
+
if (callerOwnsFusion(opts)) return lexical;
|
|
314
|
+
const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
|
|
315
|
+
return fused.map((hit) => hit.memory);
|
|
316
|
+
};
|
|
317
|
+
const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
|
|
318
|
+
const opts = searchOpts;
|
|
319
|
+
const lexicalHits = await original(query, searchOpts);
|
|
320
|
+
if (callerOwnsFusion(opts)) return lexicalHits;
|
|
321
|
+
return augmentLexicalWithVectorRecall(
|
|
322
|
+
query,
|
|
323
|
+
lexicalHits.map((hit) => hit.memory),
|
|
324
|
+
fusionOptions(opts)
|
|
367
325
|
);
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
async embedWithCache(text) {
|
|
387
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
388
|
-
const cached = this.cachedVector(text, now);
|
|
389
|
-
if (cached) return cached;
|
|
390
|
-
try {
|
|
391
|
-
const result = await this.provider.embed([text]);
|
|
392
|
-
const vec = result[0];
|
|
393
|
-
if (vec) this.cacheVector(text, vec, now);
|
|
394
|
-
return vec;
|
|
395
|
-
} catch {
|
|
396
|
-
return void 0;
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
/**
|
|
400
|
-
* Look up an existing entry by `content_hash`. Returns `undefined` when
|
|
401
|
-
* the entry is not present. Used by `remember()` to make writes idempotent
|
|
402
|
-
* and by `syncFromSage()` to skip already-indexed SAGE memories.
|
|
403
|
-
*/
|
|
404
|
-
findByContentHash(contentHash) {
|
|
405
|
-
this.assertOpen();
|
|
406
|
-
const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
|
|
407
|
-
if (!row) return void 0;
|
|
408
|
-
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
409
|
-
return this.rowToEntry(row, vectorRow);
|
|
410
|
-
}
|
|
411
|
-
/**
|
|
412
|
-
* Look up the entry mirroring a given SAGE memory id (i.e. the entry
|
|
413
|
-
* whose `metadata.sageId` equals `sageId`). Returns `undefined` when
|
|
414
|
-
* no such entry exists. Used by the event-driven mirror to delete
|
|
415
|
-
* vector entries on SAGE delete events (the emitter knows the SAGE
|
|
416
|
-
* id, not the vector entry id).
|
|
417
|
-
*
|
|
418
|
-
* Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
|
|
419
|
-
* column is the JSON blob `syncFromSage` writes, so this avoids a
|
|
420
|
-
* full table scan.
|
|
421
|
-
*/
|
|
422
|
-
findBySageId(sageId) {
|
|
423
|
-
this.assertOpen();
|
|
424
|
-
const row = this.db.prepare(
|
|
425
|
-
`SELECT * FROM entries
|
|
426
|
-
WHERE json_extract(metadata, '$.sageId') = ?
|
|
427
|
-
ORDER BY updated_at DESC
|
|
428
|
-
LIMIT 1`
|
|
429
|
-
).get(sageId);
|
|
430
|
-
if (!row) return void 0;
|
|
431
|
-
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
432
|
-
return this.rowToEntry(row, vectorRow);
|
|
433
|
-
}
|
|
434
|
-
/**
|
|
435
|
-
* Persist a new entry. Idempotent: if an entry with the same
|
|
436
|
-
* `content_hash` already exists, that entry is returned unchanged
|
|
437
|
-
* instead of inserting a duplicate. Mutating ops are wrapped in
|
|
438
|
-
* `withFileLock` so two processes cannot race the dedup check.
|
|
439
|
-
*/
|
|
440
|
-
async remember(input) {
|
|
441
|
-
this.assertOpen();
|
|
442
|
-
if (!input.text || input.text.trim().length === 0) {
|
|
443
|
-
throw new Error("VectorMemoryStore.remember: text must be non-empty");
|
|
326
|
+
};
|
|
327
|
+
const wrapped = Object.create(
|
|
328
|
+
Object.getPrototypeOf(port),
|
|
329
|
+
Object.getOwnPropertyDescriptors(port)
|
|
330
|
+
);
|
|
331
|
+
wrapped.getCapability = (capability) => {
|
|
332
|
+
if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
|
|
333
|
+
const original = port.getCapability(capability);
|
|
334
|
+
if (!original) return void 0;
|
|
335
|
+
return {
|
|
336
|
+
...original,
|
|
337
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
338
|
+
...original.searchSageWithBreakdown ? {
|
|
339
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
340
|
+
original.searchSageWithBreakdown
|
|
341
|
+
)
|
|
342
|
+
} : {}
|
|
343
|
+
};
|
|
444
344
|
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
const kind = input.kind ?? "note";
|
|
458
|
-
const id = randomUUID();
|
|
459
|
-
const vector = await this.embedWithCache(input.text);
|
|
460
|
-
const providerId = vector ? this.provider.id : void 0;
|
|
461
|
-
this.db.exec("BEGIN");
|
|
462
|
-
try {
|
|
463
|
-
this.db.prepare(
|
|
464
|
-
`INSERT INTO entries
|
|
465
|
-
(id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
|
|
466
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
467
|
-
).run(
|
|
468
|
-
id,
|
|
469
|
-
input.text,
|
|
470
|
-
input.summary ?? null,
|
|
471
|
-
JSON.stringify(metadata),
|
|
472
|
-
JSON.stringify(tags),
|
|
473
|
-
scope,
|
|
474
|
-
kind,
|
|
475
|
-
contentHash,
|
|
476
|
-
now,
|
|
477
|
-
now
|
|
478
|
-
);
|
|
479
|
-
if (vector && providerId) {
|
|
480
|
-
this.db.prepare(
|
|
481
|
-
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
482
|
-
VALUES (?, ?, ?, ?, ?)
|
|
483
|
-
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
484
|
-
vector = excluded.vector,
|
|
485
|
-
dimensions = excluded.dimensions,
|
|
486
|
-
created_at = excluded.created_at`
|
|
487
|
-
).run(id, providerId, vector.length, encodeVector(vector), now);
|
|
488
|
-
}
|
|
489
|
-
this.db.exec("COMMIT");
|
|
490
|
-
} catch (e) {
|
|
491
|
-
this.db.exec("ROLLBACK");
|
|
492
|
-
throw e;
|
|
345
|
+
if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
|
|
346
|
+
const original = port.getCapability(capability);
|
|
347
|
+
if (!original) return void 0;
|
|
348
|
+
return {
|
|
349
|
+
...original,
|
|
350
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
351
|
+
...original.searchSageWithBreakdown ? {
|
|
352
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
353
|
+
original.searchSageWithBreakdown
|
|
354
|
+
)
|
|
355
|
+
} : {}
|
|
356
|
+
};
|
|
493
357
|
}
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
358
|
+
return port.getCapability(capability);
|
|
359
|
+
};
|
|
360
|
+
return wrapped;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// src/sage-sync.ts
|
|
364
|
+
import * as fs2 from "node:fs";
|
|
365
|
+
import * as path2 from "node:path";
|
|
366
|
+
import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
|
|
367
|
+
|
|
368
|
+
// src/sage-sync-source.ts
|
|
369
|
+
var DEFAULT_PAGE_SIZE = 500;
|
|
370
|
+
var HARD_FLOOR_PAGE = 1;
|
|
371
|
+
var HARD_CEILING_PAGE = 500;
|
|
372
|
+
var HARD_CEILING_TOTAL = 1e6;
|
|
373
|
+
function createSageSurfaceSyncSource(sage, opts = {}) {
|
|
374
|
+
const pageSize = clamp(opts.pageSize ?? DEFAULT_PAGE_SIZE, HARD_FLOOR_PAGE, HARD_CEILING_PAGE);
|
|
375
|
+
const maxTotal = opts.maxTotal === void 0 ? Number.POSITIVE_INFINITY : clamp(opts.maxTotal, 1, HARD_CEILING_TOTAL);
|
|
376
|
+
return {
|
|
377
|
+
async listActiveMemories({ limit }) {
|
|
378
|
+
const requested = limit === void 0 ? maxTotal : clamp(limit, 1, maxTotal === Number.POSITIVE_INFINITY ? limit : maxTotal);
|
|
379
|
+
const memories = [];
|
|
380
|
+
let cursor;
|
|
381
|
+
let noProgressPages = 0;
|
|
382
|
+
while (memories.length < requested) {
|
|
383
|
+
const remaining = requested - memories.length;
|
|
384
|
+
const page = await sage.listSagePage({
|
|
385
|
+
statuses: ["active"],
|
|
386
|
+
limit: Math.min(pageSize, Math.max(1, remaining)),
|
|
387
|
+
...cursor ? { cursor } : {}
|
|
388
|
+
});
|
|
389
|
+
const rows = page.memories ?? [];
|
|
390
|
+
for (const m of rows) {
|
|
391
|
+
memories.push({
|
|
392
|
+
id: m.id,
|
|
393
|
+
text: m.text,
|
|
394
|
+
...m.summary ? { summary: m.summary } : {},
|
|
395
|
+
...m.tags && m.tags.length > 0 ? { tags: m.tags } : {},
|
|
396
|
+
metadata: {
|
|
397
|
+
sageKind: m.kind,
|
|
398
|
+
sageScope: m.scope,
|
|
399
|
+
importance: m.importance,
|
|
400
|
+
confidence: m.confidence
|
|
401
|
+
}
|
|
402
|
+
});
|
|
532
403
|
}
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
const queryVec = await this.embedWithCache(query);
|
|
544
|
-
if (!queryVec || queryVec.length === 0) return [];
|
|
545
|
-
const providerId = this.provider.id;
|
|
546
|
-
const dimensions = this.provider.dimensions;
|
|
547
|
-
const filters = ["v.provider_id = ?", "v.dimensions = ?"];
|
|
548
|
-
const params = [providerId, dimensions];
|
|
549
|
-
if (opts.scope !== void 0) {
|
|
550
|
-
filters.push("e.scope = ?");
|
|
551
|
-
params.push(opts.scope);
|
|
404
|
+
if (!page.nextCursor) break;
|
|
405
|
+
if (rows.length === 0) {
|
|
406
|
+
noProgressPages++;
|
|
407
|
+
if (noProgressPages > 3) break;
|
|
408
|
+
} else {
|
|
409
|
+
noProgressPages = 0;
|
|
410
|
+
}
|
|
411
|
+
cursor = page.nextCursor;
|
|
412
|
+
}
|
|
413
|
+
return memories.slice(0, requested);
|
|
552
414
|
}
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
function clamp(value, min, max) {
|
|
418
|
+
return Math.max(min, Math.min(max, value));
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// src/sage-sync.ts
|
|
422
|
+
var SAGE_SYNC_MARKER_FILENAME = "sage-sync.complete.json";
|
|
423
|
+
var RUNNING_STALE_MS = 10 * 60 * 1e3;
|
|
424
|
+
async function startFirstBootSageSync(opts) {
|
|
425
|
+
const { store, memoryStore } = opts;
|
|
426
|
+
const log = opts.logger;
|
|
427
|
+
try {
|
|
428
|
+
if (opts.force) {
|
|
429
|
+
try {
|
|
430
|
+
fs2.unlinkSync(markerPath(store));
|
|
431
|
+
} catch {
|
|
432
|
+
}
|
|
556
433
|
}
|
|
557
|
-
const
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
)
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
const raw = cosineSimilarity(queryVec, vec);
|
|
567
|
-
const score = Math.max(0, Math.min(1, raw));
|
|
568
|
-
if (!Number.isFinite(score) || score < threshold) continue;
|
|
569
|
-
if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
|
|
570
|
-
let at = top.length;
|
|
571
|
-
while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
|
|
572
|
-
top.splice(at, 0, { id: row.id, score, vector: vec });
|
|
573
|
-
if (top.length > limit) top.length = limit;
|
|
434
|
+
const decision = decideWhetherToSync(
|
|
435
|
+
store,
|
|
436
|
+
opts.staleAfterMs ?? RUNNING_STALE_MS,
|
|
437
|
+
void 0,
|
|
438
|
+
opts.pidAlive
|
|
439
|
+
);
|
|
440
|
+
if (!decision.run) {
|
|
441
|
+
log?.debug?.(`vector-memory sage sync skipped: ${decision.reason}`);
|
|
442
|
+
return { synced: false, reason: decision.reason };
|
|
574
443
|
}
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
).all(...top.map((t) => t.id));
|
|
582
|
-
const entryById = /* @__PURE__ */ new Map();
|
|
583
|
-
for (const row of hydrated) {
|
|
584
|
-
entryById.set(row.id, this.rowToEntry(row));
|
|
444
|
+
const provider = storeProvider(store);
|
|
445
|
+
if (provider && typeof provider.isAvailable === "function" && !await provider.isAvailable()) {
|
|
446
|
+
log?.debug?.(
|
|
447
|
+
"vector-memory sage sync deferred: embedding provider unavailable (optional dependency not installed?)"
|
|
448
|
+
);
|
|
449
|
+
return { synced: false, reason: "provider-unavailable" };
|
|
585
450
|
}
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
451
|
+
if (provider && typeof provider.embed === "function") {
|
|
452
|
+
try {
|
|
453
|
+
const probe = await provider.embed(["wrongstack vector memory warmup probe"]);
|
|
454
|
+
if (!probe[0] || probe[0].length === 0) throw new Error("empty embedding");
|
|
455
|
+
} catch {
|
|
456
|
+
log?.debug?.(
|
|
457
|
+
"vector-memory sage sync deferred: embedding probe failed (model not cached / backend error)"
|
|
458
|
+
);
|
|
459
|
+
return { synced: false, reason: "provider-unavailable" };
|
|
460
|
+
}
|
|
593
461
|
}
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
*
|
|
599
|
-
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
600
|
-
* without the id tiebreak two entries written in the same millisecond can
|
|
601
|
-
* swap places between calls and a paging caller silently skips one.
|
|
602
|
-
*
|
|
603
|
-
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
604
|
-
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
605
|
-
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
606
|
-
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
607
|
-
* resumes from a position, and the rows a deletion removes are ones the
|
|
608
|
-
* sweep has already passed.
|
|
609
|
-
*/
|
|
610
|
-
list(opts = {}) {
|
|
611
|
-
this.assertOpen();
|
|
612
|
-
const where = [];
|
|
613
|
-
const params = [];
|
|
614
|
-
if (opts.scope !== void 0) {
|
|
615
|
-
where.push("scope = ?");
|
|
616
|
-
params.push(opts.scope);
|
|
462
|
+
const surface = getSageSurface2(memoryStore);
|
|
463
|
+
if (!surface) {
|
|
464
|
+
log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
|
|
465
|
+
return { synced: false, reason: "no-sage-surface" };
|
|
617
466
|
}
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
467
|
+
writeMarker(store, {
|
|
468
|
+
phase: "running",
|
|
469
|
+
pid: process.pid,
|
|
470
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
471
|
+
});
|
|
472
|
+
const report = await store.syncFromSage(createSageSurfaceSyncSource(surface));
|
|
473
|
+
if (report.failed > 0) {
|
|
474
|
+
log?.warn?.(
|
|
475
|
+
`vector-memory sage sync finished with ${report.failed} failure(s) \u2014 will retry on next boot`
|
|
476
|
+
);
|
|
477
|
+
return {
|
|
478
|
+
synced: false,
|
|
479
|
+
reason: "partial-failure",
|
|
480
|
+
marker: { phase: "running", ...counts(report) }
|
|
481
|
+
};
|
|
621
482
|
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
483
|
+
let stats = store.stats();
|
|
484
|
+
if (stats.vectors !== stats.entries) {
|
|
485
|
+
log?.warn?.(
|
|
486
|
+
`vector-memory sage sync healed ${stats.entries - stats.vectors} vector-less entr(ies) via reindexAll()`
|
|
487
|
+
);
|
|
488
|
+
await store.reindexAll();
|
|
489
|
+
stats = store.stats();
|
|
625
490
|
}
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
651
|
-
VALUES (?, ?, ?, ?, ?)
|
|
652
|
-
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
653
|
-
vector = excluded.vector,
|
|
654
|
-
dimensions = excluded.dimensions,
|
|
655
|
-
created_at = excluded.created_at`
|
|
656
|
-
).run(row.id, this.provider.id, v.length, encodeVector(v), now);
|
|
657
|
-
this.cacheVector(row.text, v, now);
|
|
658
|
-
processed++;
|
|
659
|
-
} catch {
|
|
660
|
-
errors++;
|
|
661
|
-
}
|
|
662
|
-
}
|
|
663
|
-
return { processed, errors };
|
|
664
|
-
},
|
|
665
|
-
{ timeoutMs: 6e4 }
|
|
491
|
+
if (stats.vectors !== stats.entries) {
|
|
492
|
+
log?.warn?.(
|
|
493
|
+
`vector-memory sage sync incomplete: ${stats.entries - stats.vectors} entr(ies) still vector-less \u2014 will retry on next boot`
|
|
494
|
+
);
|
|
495
|
+
return {
|
|
496
|
+
synced: false,
|
|
497
|
+
reason: "vector-incomplete",
|
|
498
|
+
marker: { phase: "running", ...counts(report) }
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
const marker = {
|
|
502
|
+
phase: "complete",
|
|
503
|
+
completedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
504
|
+
...provider ? { providerId: provider.id } : {},
|
|
505
|
+
...counts(report)
|
|
506
|
+
};
|
|
507
|
+
writeMarker(store, marker);
|
|
508
|
+
log?.info?.(
|
|
509
|
+
`vector-memory sage sync complete: ${report.indexed} indexed, ${report.skipped} skipped (already present)`
|
|
510
|
+
);
|
|
511
|
+
return { synced: true, reason: "synced", marker };
|
|
512
|
+
} catch (error) {
|
|
513
|
+
log?.warn?.(
|
|
514
|
+
`vector-memory sage sync failed: ${error instanceof Error ? error.message : String(error)} \u2014 will retry on next boot`
|
|
666
515
|
);
|
|
516
|
+
return { synced: false, reason: "error" };
|
|
667
517
|
}
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
vectors: vectorCount,
|
|
676
|
-
providers: providerRows.map((r) => r.provider_id),
|
|
677
|
-
modelAvailable: true,
|
|
678
|
-
modelId: this.provider.id,
|
|
679
|
-
dimensions: this.provider.dimensions
|
|
680
|
-
};
|
|
518
|
+
}
|
|
519
|
+
function decideWhetherToSync(store, staleAfterMs, now = /* @__PURE__ */ new Date(), pidAlive = defaultPidAlive) {
|
|
520
|
+
const existing = readMarker(store);
|
|
521
|
+
if (!existing) return { run: true, reason: "no-marker" };
|
|
522
|
+
if (existing.phase === "complete") return { run: false, reason: "already-complete" };
|
|
523
|
+
if (existing.pid === process.pid) {
|
|
524
|
+
return { run: true, reason: "running-own-pid" };
|
|
681
525
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
|
|
691
|
-
const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
|
|
692
|
-
const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
|
|
693
|
-
return {
|
|
694
|
-
entries,
|
|
695
|
-
providers,
|
|
696
|
-
totalUseCount,
|
|
697
|
-
oldestLastUsedAt: oldest?.t ?? null
|
|
698
|
-
};
|
|
526
|
+
const startedAt = existing.startedAt ? Date.parse(existing.startedAt) : Number.NaN;
|
|
527
|
+
const ageMs = Number.isNaN(startedAt) ? Number.POSITIVE_INFINITY : now.getTime() - startedAt;
|
|
528
|
+
const stale = ageMs > staleAfterMs;
|
|
529
|
+
if (existing.pid === void 0) {
|
|
530
|
+
return Number.isNaN(startedAt) || stale ? {
|
|
531
|
+
run: true,
|
|
532
|
+
reason: Number.isNaN(startedAt) ? "running-marker-undated" : "running-marker-stale"
|
|
533
|
+
} : { run: false, reason: "running-unknown-pid" };
|
|
699
534
|
}
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
async evictCache(keepMostRecent) {
|
|
707
|
-
this.assertOpen();
|
|
708
|
-
if (keepMostRecent < 0) {
|
|
709
|
-
throw new Error("evictCache: keepMostRecent must be >= 0");
|
|
535
|
+
try {
|
|
536
|
+
if (pidAlive(existing.pid)) {
|
|
537
|
+
return {
|
|
538
|
+
run: false,
|
|
539
|
+
reason: `running-pid-${existing.pid}${stale ? "-stale-but-alive" : ""}`
|
|
540
|
+
};
|
|
710
541
|
}
|
|
711
|
-
return
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
715
|
-
if (total <= keepMostRecent) return { removed: 0 };
|
|
716
|
-
const toRemove = total - keepMostRecent;
|
|
717
|
-
const stmt = this.db.prepare(
|
|
718
|
-
`DELETE FROM embedding_cache
|
|
719
|
-
WHERE content_hash IN (
|
|
720
|
-
SELECT content_hash FROM embedding_cache
|
|
721
|
-
ORDER BY last_used_at ASC
|
|
722
|
-
LIMIT ?
|
|
723
|
-
)`
|
|
724
|
-
);
|
|
725
|
-
const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
|
|
726
|
-
return { removed: Number(info.changes) };
|
|
727
|
-
},
|
|
728
|
-
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
729
|
-
);
|
|
542
|
+
return { run: true, reason: `running-pid-${existing.pid}-dead` };
|
|
543
|
+
} catch {
|
|
544
|
+
return { run: false, reason: `running-pid-${existing.pid}-probe-failed` };
|
|
730
545
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
546
|
+
}
|
|
547
|
+
function defaultPidAlive(pid) {
|
|
548
|
+
try {
|
|
549
|
+
process.kill(pid, 0);
|
|
550
|
+
return true;
|
|
551
|
+
} catch (error) {
|
|
552
|
+
return error.code === "EPERM";
|
|
735
553
|
}
|
|
736
|
-
|
|
737
|
-
|
|
554
|
+
}
|
|
555
|
+
function readMarker(store) {
|
|
556
|
+
try {
|
|
557
|
+
const raw = fs2.readFileSync(markerPath(store), "utf8");
|
|
558
|
+
const parsed = JSON.parse(raw);
|
|
559
|
+
return parsed && (parsed.phase === "running" || parsed.phase === "complete") ? parsed : void 0;
|
|
560
|
+
} catch {
|
|
561
|
+
return void 0;
|
|
738
562
|
}
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
updatedAt: row.updated_at,
|
|
752
|
-
providerId: vectorRow?.provider_id ?? "",
|
|
753
|
-
dimensions: vectorRow?.dimensions ?? 0
|
|
754
|
-
};
|
|
755
|
-
if (vectorRow?.vector) {
|
|
756
|
-
entry.vector = decodeVector(vectorRow.vector);
|
|
757
|
-
}
|
|
758
|
-
return entry;
|
|
563
|
+
}
|
|
564
|
+
function writeMarker(store, marker) {
|
|
565
|
+
try {
|
|
566
|
+
fs2.writeFileSync(markerPath(store), `${JSON.stringify(marker, null, 2)}
|
|
567
|
+
`, "utf8");
|
|
568
|
+
} catch {
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
function markerPath(store) {
|
|
572
|
+
const dir = store.directory;
|
|
573
|
+
if (typeof dir !== "string") {
|
|
574
|
+
throw new Error("VectorMemoryStore.directory is required to place the sage sync marker");
|
|
759
575
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
const errors = [];
|
|
767
|
-
for (const memory of memories) {
|
|
768
|
-
try {
|
|
769
|
-
const hash = _VectorMemoryStore.contentHash(memory.text);
|
|
770
|
-
const existing = this.findByContentHash(hash);
|
|
771
|
-
if (existing) {
|
|
772
|
-
skipped++;
|
|
773
|
-
continue;
|
|
774
|
-
}
|
|
775
|
-
await this.rememberUnlocked({
|
|
776
|
-
text: memory.text,
|
|
777
|
-
summary: memory.summary ?? void 0,
|
|
778
|
-
metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
|
|
779
|
-
tags: memory.tags ?? [],
|
|
780
|
-
scope: "project",
|
|
781
|
-
kind: "note"
|
|
782
|
-
});
|
|
783
|
-
indexed++;
|
|
784
|
-
} catch (err) {
|
|
785
|
-
failed++;
|
|
786
|
-
errors.push({ memoryId: memory.id, message: errMsg(err) });
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
return { scanned: memories.length, indexed, skipped, failed, errors };
|
|
576
|
+
return path2.join(dir, SAGE_SYNC_MARKER_FILENAME);
|
|
577
|
+
}
|
|
578
|
+
function storeProvider(store) {
|
|
579
|
+
const provider = store.provider;
|
|
580
|
+
if (provider && typeof provider.id === "string") {
|
|
581
|
+
return provider;
|
|
790
582
|
}
|
|
791
|
-
|
|
792
|
-
function fallbackHashingProvider(dimensions) {
|
|
793
|
-
return new HashingEmbeddingProvider({ dimensions });
|
|
583
|
+
return void 0;
|
|
794
584
|
}
|
|
795
|
-
function
|
|
796
|
-
|
|
585
|
+
function counts(report) {
|
|
586
|
+
return {
|
|
587
|
+
scanned: report.scanned,
|
|
588
|
+
indexed: report.indexed,
|
|
589
|
+
skipped: report.skipped,
|
|
590
|
+
failed: report.failed
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
// src/schema.ts
|
|
595
|
+
var VECTOR_SCHEMA_VERSION = 2;
|
|
596
|
+
var VECTOR_PROVIDER_KEY = "active_provider_id";
|
|
597
|
+
var VECTOR_DIMENSIONS_KEY = "active_provider_dimensions";
|
|
598
|
+
function initVectorSchema(db) {
|
|
599
|
+
db.exec("PRAGMA journal_mode = WAL");
|
|
600
|
+
db.exec("PRAGMA synchronous = NORMAL");
|
|
601
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
602
|
+
db.exec("PRAGMA temp_store = MEMORY");
|
|
603
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
604
|
+
db.exec(`
|
|
605
|
+
CREATE TABLE IF NOT EXISTS schema_meta (
|
|
606
|
+
key TEXT PRIMARY KEY,
|
|
607
|
+
value TEXT NOT NULL
|
|
608
|
+
);
|
|
609
|
+
`);
|
|
610
|
+
db.exec(`
|
|
611
|
+
CREATE TABLE IF NOT EXISTS entries (
|
|
612
|
+
id TEXT PRIMARY KEY,
|
|
613
|
+
text TEXT NOT NULL,
|
|
614
|
+
summary TEXT,
|
|
615
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
616
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
617
|
+
scope TEXT NOT NULL DEFAULT 'project',
|
|
618
|
+
kind TEXT NOT NULL DEFAULT 'note',
|
|
619
|
+
content_hash TEXT NOT NULL,
|
|
620
|
+
created_at TEXT NOT NULL,
|
|
621
|
+
updated_at TEXT NOT NULL
|
|
622
|
+
);
|
|
623
|
+
`);
|
|
624
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_scope ON entries(scope)");
|
|
625
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind)");
|
|
626
|
+
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_hash ON entries(content_hash)");
|
|
627
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_entries_updated ON entries(updated_at DESC)");
|
|
628
|
+
db.exec(`
|
|
629
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
630
|
+
entry_id TEXT NOT NULL,
|
|
631
|
+
provider_id TEXT NOT NULL,
|
|
632
|
+
dimensions INTEGER NOT NULL,
|
|
633
|
+
vector BLOB NOT NULL,
|
|
634
|
+
created_at TEXT NOT NULL,
|
|
635
|
+
PRIMARY KEY (entry_id, provider_id),
|
|
636
|
+
FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
|
|
637
|
+
);
|
|
638
|
+
`);
|
|
639
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_vectors_provider ON vectors(provider_id)");
|
|
640
|
+
db.exec(`
|
|
641
|
+
CREATE TABLE IF NOT EXISTS embedding_cache (
|
|
642
|
+
content_hash TEXT NOT NULL,
|
|
643
|
+
provider_id TEXT NOT NULL,
|
|
644
|
+
dimensions INTEGER NOT NULL,
|
|
645
|
+
vector BLOB NOT NULL,
|
|
646
|
+
text TEXT NOT NULL,
|
|
647
|
+
created_at TEXT NOT NULL,
|
|
648
|
+
last_used_at TEXT NOT NULL,
|
|
649
|
+
use_count INTEGER NOT NULL DEFAULT 0,
|
|
650
|
+
PRIMARY KEY (content_hash, provider_id, dimensions)
|
|
651
|
+
);
|
|
652
|
+
`);
|
|
653
|
+
db.exec("CREATE INDEX IF NOT EXISTS idx_cache_last_used ON embedding_cache(last_used_at)");
|
|
654
|
+
db.exec(`
|
|
655
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(
|
|
656
|
+
id UNINDEXED, text, tags, content='entries', content_rowid='rowid'
|
|
657
|
+
);
|
|
658
|
+
`);
|
|
659
|
+
db.exec(`
|
|
660
|
+
CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN
|
|
661
|
+
INSERT INTO entries_fts(rowid, text, tags)
|
|
662
|
+
VALUES (new.rowid, new.text, new.tags);
|
|
663
|
+
END;
|
|
664
|
+
`);
|
|
665
|
+
db.exec(`
|
|
666
|
+
CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN
|
|
667
|
+
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
668
|
+
VALUES('delete', old.rowid, old.text, old.tags);
|
|
669
|
+
END;
|
|
670
|
+
`);
|
|
671
|
+
db.exec(`
|
|
672
|
+
CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE ON entries BEGIN
|
|
673
|
+
INSERT INTO entries_fts(entries_fts, rowid, text, tags)
|
|
674
|
+
VALUES('delete', old.rowid, old.text, old.tags);
|
|
675
|
+
INSERT INTO entries_fts(rowid, text, tags)
|
|
676
|
+
VALUES (new.rowid, new.text, new.tags);
|
|
677
|
+
END;
|
|
678
|
+
`);
|
|
679
|
+
}
|
|
680
|
+
function upsertEmbeddingCache(db, row) {
|
|
681
|
+
db.prepare(
|
|
682
|
+
`INSERT INTO embedding_cache
|
|
683
|
+
(content_hash, provider_id, dimensions, vector, text, created_at, last_used_at, use_count)
|
|
684
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 1)
|
|
685
|
+
ON CONFLICT(content_hash, provider_id, dimensions) DO UPDATE SET
|
|
686
|
+
last_used_at = excluded.last_used_at,
|
|
687
|
+
use_count = embedding_cache.use_count + 1,
|
|
688
|
+
vector = excluded.vector`
|
|
689
|
+
).run(row.contentHash, row.providerId, row.dimensions, row.vector, row.text, row.now, row.now);
|
|
690
|
+
}
|
|
691
|
+
function lookupEmbeddingCache(db, contentHash, providerId, dimensions, now) {
|
|
692
|
+
const row = db.prepare(
|
|
693
|
+
`SELECT vector FROM embedding_cache
|
|
694
|
+
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?
|
|
695
|
+
LIMIT 1`
|
|
696
|
+
).get(contentHash, providerId, dimensions);
|
|
697
|
+
if (!row) return void 0;
|
|
797
698
|
try {
|
|
798
|
-
|
|
699
|
+
db.prepare(
|
|
700
|
+
`UPDATE embedding_cache SET last_used_at = ?, use_count = use_count + 1
|
|
701
|
+
WHERE content_hash = ? AND provider_id = ? AND dimensions = ?`
|
|
702
|
+
).run(now, contentHash, providerId, dimensions);
|
|
799
703
|
} catch {
|
|
800
|
-
return fallback;
|
|
801
704
|
}
|
|
705
|
+
return decodeVector(row.vector);
|
|
802
706
|
}
|
|
803
|
-
function
|
|
804
|
-
return
|
|
707
|
+
function encodeVector(vec) {
|
|
708
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
709
|
+
}
|
|
710
|
+
function decodeVector(buf) {
|
|
711
|
+
if (buf.byteLength % 4 !== 0) {
|
|
712
|
+
throw new Error(
|
|
713
|
+
`decodeVector: invalid vector byteLength ${buf.byteLength} (must be a multiple of 4)`
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
717
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
718
|
+
for (let i = 0; i < copy.length; i++) {
|
|
719
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
720
|
+
}
|
|
721
|
+
return copy;
|
|
805
722
|
}
|
|
806
723
|
|
|
807
|
-
// src/
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
function
|
|
813
|
-
const
|
|
814
|
-
const
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
724
|
+
// src/search-race.ts
|
|
725
|
+
function previewText(text, maxLen) {
|
|
726
|
+
if (text.length <= maxLen) return text;
|
|
727
|
+
return text.slice(0, maxLen - 1) + "\u2026";
|
|
728
|
+
}
|
|
729
|
+
async function runSearchRace(query, lexical, vectorStore, options = {}) {
|
|
730
|
+
const limit = options.limit ?? 20;
|
|
731
|
+
const threshold = options.threshold ?? 0;
|
|
732
|
+
let vectorHits = [];
|
|
733
|
+
try {
|
|
734
|
+
vectorHits = await vectorStore.search(query, {
|
|
735
|
+
limit,
|
|
736
|
+
...threshold > 0 ? { threshold } : {}
|
|
737
|
+
});
|
|
738
|
+
} catch {
|
|
739
|
+
vectorHits = [];
|
|
740
|
+
}
|
|
741
|
+
const lexicalOnly = [];
|
|
742
|
+
const vectorOnly = [];
|
|
743
|
+
const overlap = [];
|
|
744
|
+
const seenIds = /* @__PURE__ */ new Set();
|
|
745
|
+
const lexicalCapped = lexical.slice(0, limit);
|
|
746
|
+
for (let i = 0; i < lexicalCapped.length; i++) {
|
|
747
|
+
const mem = lexicalCapped[i];
|
|
748
|
+
const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
|
|
749
|
+
const id = mem.id;
|
|
750
|
+
seenIds.add(id);
|
|
751
|
+
overlap.push({
|
|
752
|
+
id,
|
|
753
|
+
lexicalScore: score,
|
|
754
|
+
vectorScore: null,
|
|
755
|
+
// patched below when a vector hit carries this id
|
|
756
|
+
preview: previewText(mem.text, 140)
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
const overlapById = /* @__PURE__ */ new Map();
|
|
760
|
+
for (const row of overlap) overlapById.set(row.id, row);
|
|
761
|
+
for (const hit of vectorHits) {
|
|
762
|
+
const sageId = hit.entry.metadata?.["sageId"];
|
|
763
|
+
if (typeof sageId !== "string") continue;
|
|
764
|
+
const existing = overlapById.get(sageId);
|
|
765
|
+
if (existing) {
|
|
766
|
+
if (existing.vectorScore === null) {
|
|
767
|
+
existing.vectorScore = hit.score;
|
|
851
768
|
}
|
|
852
|
-
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
if (seenIds.has(sageId)) continue;
|
|
772
|
+
seenIds.add(sageId);
|
|
773
|
+
vectorOnly.push({
|
|
774
|
+
id: sageId,
|
|
775
|
+
lexicalScore: null,
|
|
776
|
+
vectorScore: hit.score,
|
|
777
|
+
preview: previewText(hit.entry.text, 140)
|
|
778
|
+
});
|
|
779
|
+
}
|
|
780
|
+
const finalOverlap = [];
|
|
781
|
+
for (const row of overlap) {
|
|
782
|
+
if (row.vectorScore === null) {
|
|
783
|
+
lexicalOnly.push({
|
|
784
|
+
id: row.id,
|
|
785
|
+
lexicalScore: row.lexicalScore,
|
|
786
|
+
vectorScore: null,
|
|
787
|
+
preview: row.preview
|
|
788
|
+
});
|
|
789
|
+
} else {
|
|
790
|
+
finalOverlap.push(row);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
const lexicalCount = lexicalOnly.length + finalOverlap.length;
|
|
794
|
+
const vectorCount = vectorOnly.length + finalOverlap.length;
|
|
795
|
+
const denom = Math.max(lexicalCount, vectorCount, 1);
|
|
796
|
+
return {
|
|
797
|
+
query,
|
|
798
|
+
lexicalOnly,
|
|
799
|
+
vectorOnly,
|
|
800
|
+
overlap: finalOverlap,
|
|
801
|
+
metrics: {
|
|
802
|
+
lexicalCount,
|
|
803
|
+
vectorCount,
|
|
804
|
+
overlapCount: finalOverlap.length,
|
|
805
|
+
lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
|
|
806
|
+
vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
|
|
807
|
+
agreementRatio: finalOverlap.length / denom
|
|
853
808
|
}
|
|
854
809
|
};
|
|
855
810
|
}
|
|
856
|
-
function clamp(value, min, max) {
|
|
857
|
-
return Math.max(min, Math.min(max, value));
|
|
858
|
-
}
|
|
859
811
|
|
|
860
|
-
// src/
|
|
861
|
-
import {
|
|
862
|
-
import * as
|
|
863
|
-
import * as
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
);
|
|
891
|
-
return { synced: false, reason: "provider-unavailable" };
|
|
892
|
-
}
|
|
893
|
-
if (provider && typeof provider.embed === "function") {
|
|
894
|
-
try {
|
|
895
|
-
const probe = await provider.embed(["wrongstack vector memory warmup probe"]);
|
|
896
|
-
if (!probe[0] || probe[0].length === 0) throw new Error("empty embedding");
|
|
897
|
-
} catch {
|
|
898
|
-
log?.debug?.(
|
|
899
|
-
"vector-memory sage sync deferred: embedding probe failed (model not cached / backend error)"
|
|
900
|
-
);
|
|
901
|
-
return { synced: false, reason: "provider-unavailable" };
|
|
902
|
-
}
|
|
903
|
-
}
|
|
904
|
-
const surface = getSageSurface(memoryStore);
|
|
905
|
-
if (!surface) {
|
|
906
|
-
log?.debug?.("vector-memory sage sync skipped: memory store exposes no SAGE surface");
|
|
907
|
-
return { synced: false, reason: "no-sage-surface" };
|
|
908
|
-
}
|
|
909
|
-
writeMarker(store, {
|
|
910
|
-
phase: "running",
|
|
911
|
-
pid: process.pid,
|
|
912
|
-
startedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
913
|
-
});
|
|
914
|
-
const report = await store.syncFromSage(createSageSurfaceSyncSource(surface));
|
|
915
|
-
if (report.failed > 0) {
|
|
916
|
-
log?.warn?.(
|
|
917
|
-
`vector-memory sage sync finished with ${report.failed} failure(s) \u2014 will retry on next boot`
|
|
918
|
-
);
|
|
919
|
-
return {
|
|
920
|
-
synced: false,
|
|
921
|
-
reason: "partial-failure",
|
|
922
|
-
marker: { phase: "running", ...counts(report) }
|
|
923
|
-
};
|
|
924
|
-
}
|
|
925
|
-
let stats = store.stats();
|
|
926
|
-
if (stats.vectors !== stats.entries) {
|
|
927
|
-
log?.warn?.(
|
|
928
|
-
`vector-memory sage sync healed ${stats.entries - stats.vectors} vector-less entr(ies) via reindexAll()`
|
|
929
|
-
);
|
|
930
|
-
await store.reindexAll();
|
|
931
|
-
stats = store.stats();
|
|
812
|
+
// src/store.ts
|
|
813
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
814
|
+
import * as fs3 from "node:fs";
|
|
815
|
+
import * as path3 from "node:path";
|
|
816
|
+
import { withFileLock } from "@wrongstack/core/utils";
|
|
817
|
+
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
818
|
+
import { cosineSimilarity, HashingEmbeddingProvider } from "@wrongstack/sage";
|
|
819
|
+
var DEFAULT_SEARCH_LIMIT = 10;
|
|
820
|
+
function normalizeLimit(limit) {
|
|
821
|
+
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_SEARCH_LIMIT;
|
|
822
|
+
const floored = Math.floor(limit);
|
|
823
|
+
return floored < 1 ? DEFAULT_SEARCH_LIMIT : floored;
|
|
824
|
+
}
|
|
825
|
+
var DEFAULT_DIRECTORY = ".wrongstack/vector-memory";
|
|
826
|
+
var DEFAULT_FILENAME = "vector-memory.db";
|
|
827
|
+
var DEFAULT_LOCK_TIMEOUT_MS = 5e3;
|
|
828
|
+
var CACHE_EVICT_BATCH = 256;
|
|
829
|
+
var VectorMemoryStore = class _VectorMemoryStore {
|
|
830
|
+
db;
|
|
831
|
+
dbPath;
|
|
832
|
+
rootDir;
|
|
833
|
+
provider;
|
|
834
|
+
closed = false;
|
|
835
|
+
constructor(opts) {
|
|
836
|
+
if (!opts.provider) throw new Error("VectorMemoryStore: provider is required");
|
|
837
|
+
if (!opts.projectRoot) throw new Error("VectorMemoryStore: projectRoot is required");
|
|
838
|
+
this.provider = opts.provider;
|
|
839
|
+
const dir = opts.directory ?? DEFAULT_DIRECTORY;
|
|
840
|
+
const filename = opts.filename ?? DEFAULT_FILENAME;
|
|
841
|
+
if (path3.isAbsolute(dir)) {
|
|
842
|
+
throw new Error("Vector memory directory must be project-relative.");
|
|
932
843
|
}
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
);
|
|
937
|
-
return {
|
|
938
|
-
synced: false,
|
|
939
|
-
reason: "vector-incomplete",
|
|
940
|
-
marker: { phase: "running", ...counts(report) }
|
|
941
|
-
};
|
|
844
|
+
const rootDir = path3.resolve(opts.projectRoot, dir);
|
|
845
|
+
const rel = path3.relative(path3.resolve(opts.projectRoot), rootDir);
|
|
846
|
+
if (rel.startsWith("..") || path3.isAbsolute(rel)) {
|
|
847
|
+
throw new Error("Vector memory directory must stay inside the project root.");
|
|
942
848
|
}
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
log?.info?.(
|
|
951
|
-
`vector-memory sage sync complete: ${report.indexed} indexed, ${report.skipped} skipped (already present)`
|
|
952
|
-
);
|
|
953
|
-
return { synced: true, reason: "synced", marker };
|
|
954
|
-
} catch (error) {
|
|
955
|
-
log?.warn?.(
|
|
956
|
-
`vector-memory sage sync failed: ${error instanceof Error ? error.message : String(error)} \u2014 will retry on next boot`
|
|
957
|
-
);
|
|
958
|
-
return { synced: false, reason: "error" };
|
|
849
|
+
fs3.mkdirSync(rootDir, { recursive: true });
|
|
850
|
+
this.rootDir = rootDir;
|
|
851
|
+
this.dbPath = path3.join(rootDir, filename);
|
|
852
|
+
const Database = loadRuntimeDatabaseSync();
|
|
853
|
+
this.db = new Database(this.dbPath);
|
|
854
|
+
initVectorSchema(this.db);
|
|
855
|
+
this.recordActiveProvider();
|
|
959
856
|
}
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
857
|
+
/**
|
|
858
|
+
* Absolute path of the store's data directory (the resolved
|
|
859
|
+
* `opts.directory`, default `.wrongstack/vector-memory`). Hosts use this
|
|
860
|
+
* to place sidecar state (e.g. the first-boot SAGE sync marker) next to
|
|
861
|
+
* the db instead of re-deriving the path and drifting from it.
|
|
862
|
+
*/
|
|
863
|
+
get directory() {
|
|
864
|
+
return this.rootDir;
|
|
967
865
|
}
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
} : { run: false, reason: "running-unknown-pid" };
|
|
866
|
+
/**
|
|
867
|
+
* Absolute path of the SQLite database file. Hosts use this to take a
|
|
868
|
+
* file-level lock that covers all mutating operations (see
|
|
869
|
+
* `withFileLock(this.dbPath + '.lock', …)`).
|
|
870
|
+
*/
|
|
871
|
+
get databasePath() {
|
|
872
|
+
return this.dbPath;
|
|
976
873
|
}
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
874
|
+
/** The lockfile path used to serialize mutating operations. */
|
|
875
|
+
get lockPath() {
|
|
876
|
+
return `${this.dbPath}.lock`;
|
|
877
|
+
}
|
|
878
|
+
get activeProviderId() {
|
|
879
|
+
const row = this.db.prepare("SELECT value FROM schema_meta WHERE key = ?").get(VECTOR_PROVIDER_KEY);
|
|
880
|
+
return row?.value ?? this.provider.id;
|
|
881
|
+
}
|
|
882
|
+
recordActiveProvider() {
|
|
883
|
+
this.db.exec("BEGIN");
|
|
884
|
+
try {
|
|
885
|
+
this.db.prepare(
|
|
886
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
887
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
888
|
+
).run(VECTOR_PROVIDER_KEY, this.provider.id);
|
|
889
|
+
this.db.prepare(
|
|
890
|
+
`INSERT INTO schema_meta (key, value) VALUES (?, ?)
|
|
891
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`
|
|
892
|
+
).run(VECTOR_DIMENSIONS_KEY, String(this.provider.dimensions));
|
|
893
|
+
this.db.exec("COMMIT");
|
|
894
|
+
} catch (e) {
|
|
895
|
+
this.db.exec("ROLLBACK");
|
|
896
|
+
throw e;
|
|
983
897
|
}
|
|
984
|
-
return { run: true, reason: `running-pid-${existing.pid}-dead` };
|
|
985
|
-
} catch {
|
|
986
|
-
return { run: false, reason: `running-pid-${existing.pid}-probe-failed` };
|
|
987
898
|
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
try {
|
|
991
|
-
process.kill(pid, 0);
|
|
992
|
-
return true;
|
|
993
|
-
} catch (error) {
|
|
994
|
-
return error.code === "EPERM";
|
|
899
|
+
static contentHash(text) {
|
|
900
|
+
return createHash("sha256").update(text.normalize("NFKC").trim()).digest("hex");
|
|
995
901
|
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
902
|
+
/**
|
|
903
|
+
* Look up a vector for `text` in the provider-level embedding cache.
|
|
904
|
+
* Cache hit returns the cached vector (no ONNX pass). Miss returns
|
|
905
|
+
* `undefined`.
|
|
906
|
+
*/
|
|
907
|
+
cachedVector(text, now) {
|
|
908
|
+
return lookupEmbeddingCache(
|
|
909
|
+
this.db,
|
|
910
|
+
_VectorMemoryStore.contentHash(text),
|
|
911
|
+
this.provider.id,
|
|
912
|
+
this.provider.dimensions,
|
|
913
|
+
now
|
|
914
|
+
);
|
|
1004
915
|
}
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
916
|
+
/** Persist `vec` for `text` to the embedding cache. */
|
|
917
|
+
cacheVector(text, vec, now) {
|
|
918
|
+
upsertEmbeddingCache(this.db, {
|
|
919
|
+
contentHash: _VectorMemoryStore.contentHash(text),
|
|
920
|
+
providerId: this.provider.id,
|
|
921
|
+
dimensions: vec.length,
|
|
922
|
+
vector: encodeVector(vec),
|
|
923
|
+
text,
|
|
924
|
+
now
|
|
925
|
+
});
|
|
1011
926
|
}
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
927
|
+
/**
|
|
928
|
+
* Embed `text`, hitting the provider-level cache first. Cache miss falls
|
|
929
|
+
* through to the configured provider and writes the result back. Returns
|
|
930
|
+
* `undefined` when the provider fails — the caller can persist the entry
|
|
931
|
+
* without a vector (fail-open).
|
|
932
|
+
*/
|
|
933
|
+
async embedWithCache(text) {
|
|
934
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
935
|
+
const cached = this.cachedVector(text, now);
|
|
936
|
+
if (cached) return cached;
|
|
937
|
+
try {
|
|
938
|
+
const result = await this.provider.embed([text]);
|
|
939
|
+
const vec = result[0];
|
|
940
|
+
if (vec) this.cacheVector(text, vec, now);
|
|
941
|
+
return vec;
|
|
942
|
+
} catch {
|
|
943
|
+
return void 0;
|
|
944
|
+
}
|
|
1017
945
|
}
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
946
|
+
/**
|
|
947
|
+
* Look up an existing entry by `content_hash`. Returns `undefined` when
|
|
948
|
+
* the entry is not present. Used by `remember()` to make writes idempotent
|
|
949
|
+
* and by `syncFromSage()` to skip already-indexed SAGE memories.
|
|
950
|
+
*/
|
|
951
|
+
findByContentHash(contentHash) {
|
|
952
|
+
this.assertOpen();
|
|
953
|
+
const row = this.db.prepare("SELECT * FROM entries WHERE content_hash = ? LIMIT 1").get(contentHash);
|
|
954
|
+
if (!row) return void 0;
|
|
955
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
956
|
+
return this.rowToEntry(row, vectorRow);
|
|
1024
957
|
}
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
type: "object",
|
|
1059
|
-
properties: {
|
|
1060
|
-
text: { type: "string", minLength: 1, description: "The text to embed and store." },
|
|
1061
|
-
summary: { type: "string", description: "Optional short label." },
|
|
1062
|
-
tags: {
|
|
1063
|
-
type: "array",
|
|
1064
|
-
items: { type: "string" },
|
|
1065
|
-
description: "Optional tags for later filtering."
|
|
1066
|
-
},
|
|
1067
|
-
scope: {
|
|
1068
|
-
type: "string",
|
|
1069
|
-
enum: ["project", "user", "session"],
|
|
1070
|
-
description: "Visibility scope. Defaults to `project`."
|
|
1071
|
-
},
|
|
1072
|
-
kind: {
|
|
1073
|
-
type: "string",
|
|
1074
|
-
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1075
|
-
description: "Entry kind. Defaults to `note`."
|
|
1076
|
-
},
|
|
1077
|
-
metadata: {
|
|
1078
|
-
type: "object",
|
|
1079
|
-
additionalProperties: true,
|
|
1080
|
-
description: "Free-form metadata persisted as JSON."
|
|
1081
|
-
}
|
|
1082
|
-
},
|
|
1083
|
-
required: ["text"],
|
|
1084
|
-
additionalProperties: false
|
|
1085
|
-
},
|
|
1086
|
-
execute: async (input) => {
|
|
1087
|
-
const entry = await store.remember(input);
|
|
1088
|
-
return { id: entry.id, hasVector: entry.vector !== void 0 };
|
|
958
|
+
/**
|
|
959
|
+
* Look up the entry mirroring a given SAGE memory id (i.e. the entry
|
|
960
|
+
* whose `metadata.sageId` equals `sageId`). Returns `undefined` when
|
|
961
|
+
* no such entry exists. Used by the event-driven mirror to delete
|
|
962
|
+
* vector entries on SAGE delete events (the emitter knows the SAGE
|
|
963
|
+
* id, not the vector entry id).
|
|
964
|
+
*
|
|
965
|
+
* Index lookup is `json_extract(metadata, '$.sageId')` — the metadata
|
|
966
|
+
* column is the JSON blob `syncFromSage` writes, so this avoids a
|
|
967
|
+
* full table scan.
|
|
968
|
+
*/
|
|
969
|
+
findBySageId(sageId) {
|
|
970
|
+
this.assertOpen();
|
|
971
|
+
const row = this.db.prepare(
|
|
972
|
+
`SELECT * FROM entries
|
|
973
|
+
WHERE json_extract(metadata, '$.sageId') = ?
|
|
974
|
+
ORDER BY updated_at DESC
|
|
975
|
+
LIMIT 1`
|
|
976
|
+
).get(sageId);
|
|
977
|
+
if (!row) return void 0;
|
|
978
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(row.id);
|
|
979
|
+
return this.rowToEntry(row, vectorRow);
|
|
980
|
+
}
|
|
981
|
+
/**
|
|
982
|
+
* Persist a new entry. Idempotent: if an entry with the same
|
|
983
|
+
* `content_hash` already exists, that entry is returned unchanged
|
|
984
|
+
* instead of inserting a duplicate. Mutating ops are wrapped in
|
|
985
|
+
* `withFileLock` so two processes cannot race the dedup check.
|
|
986
|
+
*/
|
|
987
|
+
async remember(input) {
|
|
988
|
+
this.assertOpen();
|
|
989
|
+
if (!input.text || input.text.trim().length === 0) {
|
|
990
|
+
throw new Error("VectorMemoryStore.remember: text must be non-empty");
|
|
1089
991
|
}
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
scope: input.scope,
|
|
1139
|
-
kind: input.kind
|
|
1140
|
-
});
|
|
1141
|
-
return {
|
|
1142
|
-
hits: hits.map((h) => ({
|
|
1143
|
-
id: h.entry.id,
|
|
1144
|
-
score: h.score,
|
|
1145
|
-
text: h.entry.text,
|
|
1146
|
-
summary: h.entry.summary ?? void 0,
|
|
1147
|
-
tags: h.entry.tags
|
|
1148
|
-
}))
|
|
1149
|
-
};
|
|
992
|
+
return withFileLock(this.lockPath, () => this.rememberUnlocked(input), {
|
|
993
|
+
timeoutMs: DEFAULT_LOCK_TIMEOUT_MS
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
async rememberUnlocked(input) {
|
|
997
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
998
|
+
const contentHash = _VectorMemoryStore.contentHash(input.text);
|
|
999
|
+
const existing = this.findByContentHash(contentHash);
|
|
1000
|
+
if (existing) return existing;
|
|
1001
|
+
const metadata = input.metadata ?? {};
|
|
1002
|
+
const tags = input.tags ?? [];
|
|
1003
|
+
const scope = input.scope ?? "project";
|
|
1004
|
+
const kind = input.kind ?? "note";
|
|
1005
|
+
const id = randomUUID();
|
|
1006
|
+
const vector = await this.embedWithCache(input.text);
|
|
1007
|
+
const providerId = vector ? this.provider.id : void 0;
|
|
1008
|
+
this.db.exec("BEGIN");
|
|
1009
|
+
try {
|
|
1010
|
+
this.db.prepare(
|
|
1011
|
+
`INSERT INTO entries
|
|
1012
|
+
(id, text, summary, metadata, tags, scope, kind, content_hash, created_at, updated_at)
|
|
1013
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1014
|
+
).run(
|
|
1015
|
+
id,
|
|
1016
|
+
input.text,
|
|
1017
|
+
input.summary ?? null,
|
|
1018
|
+
JSON.stringify(metadata),
|
|
1019
|
+
JSON.stringify(tags),
|
|
1020
|
+
scope,
|
|
1021
|
+
kind,
|
|
1022
|
+
contentHash,
|
|
1023
|
+
now,
|
|
1024
|
+
now
|
|
1025
|
+
);
|
|
1026
|
+
if (vector && providerId) {
|
|
1027
|
+
this.db.prepare(
|
|
1028
|
+
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
1029
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1030
|
+
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
1031
|
+
vector = excluded.vector,
|
|
1032
|
+
dimensions = excluded.dimensions,
|
|
1033
|
+
created_at = excluded.created_at`
|
|
1034
|
+
).run(id, providerId, vector.length, encodeVector(vector), now);
|
|
1035
|
+
}
|
|
1036
|
+
this.db.exec("COMMIT");
|
|
1037
|
+
} catch (e) {
|
|
1038
|
+
this.db.exec("ROLLBACK");
|
|
1039
|
+
throw e;
|
|
1150
1040
|
}
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
type: "string",
|
|
1190
|
-
minLength: 1,
|
|
1191
|
-
description: "Entry id returned by `vector_memory_remember`."
|
|
1041
|
+
const result = {
|
|
1042
|
+
id,
|
|
1043
|
+
text: input.text,
|
|
1044
|
+
summary: input.summary ?? void 0,
|
|
1045
|
+
metadata,
|
|
1046
|
+
tags,
|
|
1047
|
+
scope,
|
|
1048
|
+
kind,
|
|
1049
|
+
contentHash,
|
|
1050
|
+
createdAt: now,
|
|
1051
|
+
updatedAt: now,
|
|
1052
|
+
providerId: providerId ?? "",
|
|
1053
|
+
dimensions: vector?.length ?? 0
|
|
1054
|
+
};
|
|
1055
|
+
if (vector) result.vector = vector;
|
|
1056
|
+
return result;
|
|
1057
|
+
}
|
|
1058
|
+
get(id) {
|
|
1059
|
+
this.assertOpen();
|
|
1060
|
+
const row = this.db.prepare("SELECT * FROM entries WHERE id = ?").get(id);
|
|
1061
|
+
if (!row) return void 0;
|
|
1062
|
+
const vectorRow = this.db.prepare("SELECT * FROM vectors WHERE entry_id = ?").get(id);
|
|
1063
|
+
return this.rowToEntry(row, vectorRow);
|
|
1064
|
+
}
|
|
1065
|
+
/** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
|
|
1066
|
+
async forget(id) {
|
|
1067
|
+
this.assertOpen();
|
|
1068
|
+
return withFileLock(
|
|
1069
|
+
this.lockPath,
|
|
1070
|
+
async () => {
|
|
1071
|
+
this.db.exec("BEGIN");
|
|
1072
|
+
try {
|
|
1073
|
+
const info = this.db.prepare("DELETE FROM entries WHERE id = ?").run(id);
|
|
1074
|
+
this.db.exec("COMMIT");
|
|
1075
|
+
return info.changes > 0;
|
|
1076
|
+
} catch (e) {
|
|
1077
|
+
this.db.exec("ROLLBACK");
|
|
1078
|
+
throw e;
|
|
1192
1079
|
}
|
|
1193
1080
|
},
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1081
|
+
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
1082
|
+
);
|
|
1083
|
+
}
|
|
1084
|
+
async search(query, opts = {}) {
|
|
1085
|
+
this.assertOpen();
|
|
1086
|
+
const limit = normalizeLimit(opts.limit);
|
|
1087
|
+
const threshold = opts.threshold ?? 0;
|
|
1088
|
+
const includeVectors = opts.includeVectors === true;
|
|
1089
|
+
if (typeof query !== "string" || query.trim().length === 0) return [];
|
|
1090
|
+
const queryVec = await this.embedWithCache(query);
|
|
1091
|
+
if (!queryVec || queryVec.length === 0) return [];
|
|
1092
|
+
const providerId = this.provider.id;
|
|
1093
|
+
const dimensions = this.provider.dimensions;
|
|
1094
|
+
const filters = ["v.provider_id = ?", "v.dimensions = ?"];
|
|
1095
|
+
const params = [providerId, dimensions];
|
|
1096
|
+
if (opts.scope !== void 0) {
|
|
1097
|
+
filters.push("e.scope = ?");
|
|
1098
|
+
params.push(opts.scope);
|
|
1099
|
+
}
|
|
1100
|
+
if (opts.kind !== void 0) {
|
|
1101
|
+
filters.push("e.kind = ?");
|
|
1102
|
+
params.push(opts.kind);
|
|
1103
|
+
}
|
|
1104
|
+
const scanRows = this.db.prepare(
|
|
1105
|
+
`SELECT e.id AS id, v.vector AS vec_blob
|
|
1106
|
+
FROM entries e
|
|
1107
|
+
JOIN vectors v ON v.entry_id = e.id
|
|
1108
|
+
WHERE ${filters.join(" AND ")}`
|
|
1109
|
+
).all(...params);
|
|
1110
|
+
const top = [];
|
|
1111
|
+
for (const row of scanRows) {
|
|
1112
|
+
const vec = decodeVector(row.vec_blob);
|
|
1113
|
+
const raw = cosineSimilarity(queryVec, vec);
|
|
1114
|
+
const score = Math.max(0, Math.min(1, raw));
|
|
1115
|
+
if (!Number.isFinite(score) || score < threshold) continue;
|
|
1116
|
+
if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
|
|
1117
|
+
let at = top.length;
|
|
1118
|
+
while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
|
|
1119
|
+
top.splice(at, 0, { id: row.id, score, vector: vec });
|
|
1120
|
+
if (top.length > limit) top.length = limit;
|
|
1121
|
+
}
|
|
1122
|
+
if (top.length === 0) return [];
|
|
1123
|
+
const placeholders = top.map(() => "?").join(",");
|
|
1124
|
+
const hydrated = this.db.prepare(
|
|
1125
|
+
`SELECT id, text, summary, metadata, tags, scope, kind,
|
|
1126
|
+
content_hash, created_at, updated_at
|
|
1127
|
+
FROM entries WHERE id IN (${placeholders})`
|
|
1128
|
+
).all(...top.map((t) => t.id));
|
|
1129
|
+
const entryById = /* @__PURE__ */ new Map();
|
|
1130
|
+
for (const row of hydrated) {
|
|
1131
|
+
entryById.set(row.id, this.rowToEntry(row));
|
|
1132
|
+
}
|
|
1133
|
+
const scored = [];
|
|
1134
|
+
for (const candidate of top) {
|
|
1135
|
+
const entry = entryById.get(candidate.id);
|
|
1136
|
+
if (!entry) continue;
|
|
1137
|
+
const hit = { entry, score: candidate.score, providerId };
|
|
1138
|
+
if (includeVectors) hit.vector = candidate.vector;
|
|
1139
|
+
scored.push(hit);
|
|
1224
1140
|
}
|
|
1141
|
+
return scored;
|
|
1225
1142
|
}
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1143
|
+
/**
|
|
1144
|
+
* Page through entries, newest first.
|
|
1145
|
+
*
|
|
1146
|
+
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
1147
|
+
* without the id tiebreak two entries written in the same millisecond can
|
|
1148
|
+
* swap places between calls and a paging caller silently skips one.
|
|
1149
|
+
*
|
|
1150
|
+
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
1151
|
+
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
1152
|
+
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
1153
|
+
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
1154
|
+
* resumes from a position, and the rows a deletion removes are ones the
|
|
1155
|
+
* sweep has already passed.
|
|
1156
|
+
*/
|
|
1157
|
+
list(opts = {}) {
|
|
1158
|
+
this.assertOpen();
|
|
1159
|
+
const where = [];
|
|
1160
|
+
const params = [];
|
|
1161
|
+
if (opts.scope !== void 0) {
|
|
1162
|
+
where.push("scope = ?");
|
|
1163
|
+
params.push(opts.scope);
|
|
1164
|
+
}
|
|
1165
|
+
if (opts.kind !== void 0) {
|
|
1166
|
+
where.push("kind = ?");
|
|
1167
|
+
params.push(opts.kind);
|
|
1168
|
+
}
|
|
1169
|
+
if (opts.after) {
|
|
1170
|
+
where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
|
|
1171
|
+
params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
|
|
1172
|
+
}
|
|
1173
|
+
const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
|
|
1174
|
+
ORDER BY updated_at DESC, id DESC LIMIT ?`;
|
|
1175
|
+
params.push(opts.limit ?? 100);
|
|
1176
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
1177
|
+
return rows.map((r) => this.rowToEntry(r));
|
|
1178
|
+
}
|
|
1179
|
+
async reindexAll() {
|
|
1180
|
+
this.assertOpen();
|
|
1181
|
+
return withFileLock(
|
|
1182
|
+
this.lockPath,
|
|
1183
|
+
async () => {
|
|
1184
|
+
const rows = this.db.prepare("SELECT id, text FROM entries").all();
|
|
1185
|
+
let processed = 0;
|
|
1186
|
+
let errors = 0;
|
|
1187
|
+
for (const row of rows) {
|
|
1188
|
+
try {
|
|
1189
|
+
const result = await this.provider.embed([row.text]);
|
|
1190
|
+
const v = result[0];
|
|
1191
|
+
if (!v) {
|
|
1192
|
+
errors++;
|
|
1193
|
+
continue;
|
|
1194
|
+
}
|
|
1195
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1196
|
+
this.db.prepare(
|
|
1197
|
+
`INSERT INTO vectors (entry_id, provider_id, dimensions, vector, created_at)
|
|
1198
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1199
|
+
ON CONFLICT(entry_id, provider_id) DO UPDATE SET
|
|
1200
|
+
vector = excluded.vector,
|
|
1201
|
+
dimensions = excluded.dimensions,
|
|
1202
|
+
created_at = excluded.created_at`
|
|
1203
|
+
).run(row.id, this.provider.id, v.length, encodeVector(v), now);
|
|
1204
|
+
this.cacheVector(row.text, v, now);
|
|
1205
|
+
processed++;
|
|
1206
|
+
} catch {
|
|
1207
|
+
errors++;
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
return { processed, errors };
|
|
1211
|
+
},
|
|
1212
|
+
{ timeoutMs: 6e4 }
|
|
1213
|
+
);
|
|
1243
1214
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
const
|
|
1247
|
-
const
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1215
|
+
stats() {
|
|
1216
|
+
this.assertOpen();
|
|
1217
|
+
const entryCount = this.db.prepare("SELECT COUNT(*) AS n FROM entries").get().n;
|
|
1218
|
+
const vectorCount = this.db.prepare("SELECT COUNT(*) AS n FROM vectors").get().n;
|
|
1219
|
+
const providerRows = this.db.prepare("SELECT DISTINCT provider_id FROM vectors").all();
|
|
1220
|
+
return {
|
|
1221
|
+
entries: entryCount,
|
|
1222
|
+
vectors: vectorCount,
|
|
1223
|
+
providers: providerRows.map((r) => r.provider_id),
|
|
1224
|
+
modelAvailable: true,
|
|
1225
|
+
modelId: this.provider.id,
|
|
1226
|
+
dimensions: this.provider.dimensions
|
|
1227
|
+
};
|
|
1255
1228
|
}
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Embedding-cache diagnostics — entries, hit/miss counters, oldest entry.
|
|
1231
|
+
* Useful for the WebUI's vector-memory panel and for diagnosing the
|
|
1232
|
+
* "why is search slow?" question.
|
|
1233
|
+
*/
|
|
1234
|
+
cacheStats() {
|
|
1235
|
+
this.assertOpen();
|
|
1236
|
+
const entries = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
1237
|
+
const providers = this.db.prepare("SELECT COUNT(DISTINCT provider_id) AS n FROM embedding_cache").get().n;
|
|
1238
|
+
const totalUseCount = this.db.prepare("SELECT COALESCE(SUM(use_count), 0) AS n FROM embedding_cache").get().n;
|
|
1239
|
+
const oldest = this.db.prepare("SELECT MIN(last_used_at) AS t FROM embedding_cache").get();
|
|
1240
|
+
return {
|
|
1241
|
+
entries,
|
|
1242
|
+
providers,
|
|
1243
|
+
totalUseCount,
|
|
1244
|
+
oldestLastUsedAt: oldest?.t ?? null
|
|
1245
|
+
};
|
|
1273
1246
|
}
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
});
|
|
1285
|
-
return hits.map((h) => ({
|
|
1286
|
-
id: h.entry.id,
|
|
1287
|
-
score: h.score,
|
|
1288
|
-
text: h.entry.text,
|
|
1289
|
-
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
1290
|
-
tags: h.entry.tags,
|
|
1291
|
-
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
1292
|
-
}));
|
|
1293
|
-
}
|
|
1294
|
-
};
|
|
1295
|
-
}
|
|
1296
|
-
function clamp01(value) {
|
|
1297
|
-
if (!Number.isFinite(value)) return DEFAULT_VECTOR_WEIGHT;
|
|
1298
|
-
if (value < 0) return 0;
|
|
1299
|
-
if (value > 1) return 1;
|
|
1300
|
-
return value;
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
// src/sage-port-wrapper.ts
|
|
1304
|
-
import {
|
|
1305
|
-
augmentLexicalWithVectorRecall,
|
|
1306
|
-
isSageVisibleForSearch,
|
|
1307
|
-
SAGE_RETRIEVAL_CAPABILITY,
|
|
1308
|
-
SAGE_SURFACE_CAPABILITY
|
|
1309
|
-
} from "@wrongstack/sage";
|
|
1310
|
-
function asVectorRecallProviderAdapter(store) {
|
|
1311
|
-
return {
|
|
1312
|
-
async search(query, opts) {
|
|
1313
|
-
const hits = await store.search(query, {
|
|
1314
|
-
limit: opts.limit,
|
|
1315
|
-
...opts.threshold !== void 0 ? { threshold: opts.threshold } : {}
|
|
1316
|
-
});
|
|
1317
|
-
return hits.map((h) => ({
|
|
1318
|
-
id: h.entry.id,
|
|
1319
|
-
score: h.score,
|
|
1320
|
-
text: h.entry.text,
|
|
1321
|
-
...h.entry.summary ? { summary: h.entry.summary } : {},
|
|
1322
|
-
tags: h.entry.tags,
|
|
1323
|
-
...h.entry.metadata ? { metadata: h.entry.metadata } : {}
|
|
1324
|
-
}));
|
|
1247
|
+
/**
|
|
1248
|
+
* LRU-evict the embedding cache down to `keepMostRecent` rows. The
|
|
1249
|
+
* `embedding_cache` table is independent of `entries`, so a sweep here
|
|
1250
|
+
* only removes cached vectors — never stored entries. Called by hosts
|
|
1251
|
+
* that want to bound cache growth on long-lived processes.
|
|
1252
|
+
*/
|
|
1253
|
+
async evictCache(keepMostRecent) {
|
|
1254
|
+
this.assertOpen();
|
|
1255
|
+
if (keepMostRecent < 0) {
|
|
1256
|
+
throw new Error("evictCache: keepMostRecent must be >= 0");
|
|
1325
1257
|
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
|
|
1345
|
-
});
|
|
1346
|
-
const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
|
|
1347
|
-
const wrapSearchSage = (original) => async (query, searchOpts) => {
|
|
1348
|
-
const opts = searchOpts;
|
|
1349
|
-
const lexical = await original(query, searchOpts);
|
|
1350
|
-
if (callerOwnsFusion(opts)) return lexical;
|
|
1351
|
-
const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
|
|
1352
|
-
return fused.map((hit) => hit.memory);
|
|
1353
|
-
};
|
|
1354
|
-
const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
|
|
1355
|
-
const opts = searchOpts;
|
|
1356
|
-
const lexicalHits = await original(query, searchOpts);
|
|
1357
|
-
if (callerOwnsFusion(opts)) return lexicalHits;
|
|
1358
|
-
return augmentLexicalWithVectorRecall(
|
|
1359
|
-
query,
|
|
1360
|
-
lexicalHits.map((hit) => hit.memory),
|
|
1361
|
-
fusionOptions(opts)
|
|
1258
|
+
return withFileLock(
|
|
1259
|
+
this.lockPath,
|
|
1260
|
+
async () => {
|
|
1261
|
+
const total = this.db.prepare("SELECT COUNT(*) AS n FROM embedding_cache").get().n;
|
|
1262
|
+
if (total <= keepMostRecent) return { removed: 0 };
|
|
1263
|
+
const toRemove = total - keepMostRecent;
|
|
1264
|
+
const stmt = this.db.prepare(
|
|
1265
|
+
`DELETE FROM embedding_cache
|
|
1266
|
+
WHERE content_hash IN (
|
|
1267
|
+
SELECT content_hash FROM embedding_cache
|
|
1268
|
+
ORDER BY last_used_at ASC
|
|
1269
|
+
LIMIT ?
|
|
1270
|
+
)`
|
|
1271
|
+
);
|
|
1272
|
+
const info = stmt.run(Math.min(toRemove, CACHE_EVICT_BATCH));
|
|
1273
|
+
return { removed: Number(info.changes) };
|
|
1274
|
+
},
|
|
1275
|
+
{ timeoutMs: DEFAULT_LOCK_TIMEOUT_MS }
|
|
1362
1276
|
);
|
|
1363
|
-
};
|
|
1364
|
-
const wrapped = Object.create(
|
|
1365
|
-
Object.getPrototypeOf(port),
|
|
1366
|
-
Object.getOwnPropertyDescriptors(port)
|
|
1367
|
-
);
|
|
1368
|
-
wrapped.getCapability = (capability) => {
|
|
1369
|
-
if (capability.id === SAGE_RETRIEVAL_CAPABILITY.id) {
|
|
1370
|
-
const original = port.getCapability(capability);
|
|
1371
|
-
if (!original) return void 0;
|
|
1372
|
-
return {
|
|
1373
|
-
...original,
|
|
1374
|
-
searchSage: wrapSearchSage(original.searchSage),
|
|
1375
|
-
...original.searchSageWithBreakdown ? {
|
|
1376
|
-
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
1377
|
-
original.searchSageWithBreakdown
|
|
1378
|
-
)
|
|
1379
|
-
} : {}
|
|
1380
|
-
};
|
|
1381
|
-
}
|
|
1382
|
-
if (capability.id === SAGE_SURFACE_CAPABILITY.id) {
|
|
1383
|
-
const original = port.getCapability(capability);
|
|
1384
|
-
if (!original) return void 0;
|
|
1385
|
-
return {
|
|
1386
|
-
...original,
|
|
1387
|
-
searchSage: wrapSearchSage(original.searchSage),
|
|
1388
|
-
...original.searchSageWithBreakdown ? {
|
|
1389
|
-
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
1390
|
-
original.searchSageWithBreakdown
|
|
1391
|
-
)
|
|
1392
|
-
} : {}
|
|
1393
|
-
};
|
|
1394
|
-
}
|
|
1395
|
-
return port.getCapability(capability);
|
|
1396
|
-
};
|
|
1397
|
-
return wrapped;
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
// src/sage-event-mirror.ts
|
|
1401
|
-
import * as fs3 from "node:fs";
|
|
1402
|
-
import * as path3 from "node:path";
|
|
1403
|
-
import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
|
|
1404
|
-
function subscribeVectorMemoryToSage(opts) {
|
|
1405
|
-
const { store, memoryStore } = opts;
|
|
1406
|
-
const log = opts.logger;
|
|
1407
|
-
if (opts.enabled === false) {
|
|
1408
|
-
return { dispose: () => void 0 };
|
|
1409
1277
|
}
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1278
|
+
close() {
|
|
1279
|
+
if (this.closed) return;
|
|
1280
|
+
this.closed = true;
|
|
1281
|
+
this.db.close();
|
|
1282
|
+
}
|
|
1283
|
+
assertOpen() {
|
|
1284
|
+
if (this.closed) throw new Error("VectorMemoryStore is closed");
|
|
1285
|
+
}
|
|
1286
|
+
rowToEntry(row, vectorRow) {
|
|
1287
|
+
const summaryValue = row.summary;
|
|
1288
|
+
const entry = {
|
|
1289
|
+
id: row.id,
|
|
1290
|
+
text: row.text,
|
|
1291
|
+
summary: summaryValue ?? void 0,
|
|
1292
|
+
metadata: safeParseJson(row.metadata, {}),
|
|
1293
|
+
tags: safeParseJson(row.tags, []),
|
|
1294
|
+
scope: row.scope,
|
|
1295
|
+
kind: row.kind,
|
|
1296
|
+
contentHash: row.content_hash,
|
|
1297
|
+
createdAt: row.created_at,
|
|
1298
|
+
updatedAt: row.updated_at,
|
|
1299
|
+
providerId: vectorRow?.provider_id ?? "",
|
|
1300
|
+
dimensions: vectorRow?.dimensions ?? 0
|
|
1301
|
+
};
|
|
1302
|
+
if (vectorRow?.vector) {
|
|
1303
|
+
entry.vector = decodeVector(vectorRow.vector);
|
|
1304
|
+
}
|
|
1305
|
+
return entry;
|
|
1306
|
+
}
|
|
1307
|
+
async syncFromSage(sage) {
|
|
1308
|
+
this.assertOpen();
|
|
1309
|
+
const memories = await sage.listActiveMemories({ limit: Number.POSITIVE_INFINITY });
|
|
1310
|
+
let indexed = 0;
|
|
1311
|
+
let skipped = 0;
|
|
1312
|
+
let failed = 0;
|
|
1313
|
+
const errors = [];
|
|
1314
|
+
for (const memory of memories) {
|
|
1315
|
+
try {
|
|
1316
|
+
const hash = _VectorMemoryStore.contentHash(memory.text);
|
|
1317
|
+
const existing = this.findByContentHash(hash);
|
|
1318
|
+
if (existing) {
|
|
1319
|
+
skipped++;
|
|
1320
|
+
continue;
|
|
1321
|
+
}
|
|
1322
|
+
await this.rememberUnlocked({
|
|
1323
|
+
text: memory.text,
|
|
1324
|
+
summary: memory.summary ?? void 0,
|
|
1325
|
+
metadata: { source: "sage", sageId: memory.id, ...memory.metadata ?? {} },
|
|
1326
|
+
tags: memory.tags ?? [],
|
|
1327
|
+
scope: "project",
|
|
1328
|
+
kind: "note"
|
|
1329
|
+
});
|
|
1330
|
+
indexed++;
|
|
1331
|
+
} catch (err) {
|
|
1332
|
+
failed++;
|
|
1333
|
+
errors.push({ memoryId: memory.id, message: errMsg2(err) });
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
return { scanned: memories.length, indexed, skipped, failed, errors };
|
|
1414
1337
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1338
|
+
};
|
|
1339
|
+
function fallbackHashingProvider(dimensions) {
|
|
1340
|
+
return new HashingEmbeddingProvider({ dimensions });
|
|
1341
|
+
}
|
|
1342
|
+
function safeParseJson(value, fallback) {
|
|
1343
|
+
if (typeof value !== "string") return fallback;
|
|
1344
|
+
try {
|
|
1345
|
+
return JSON.parse(value);
|
|
1346
|
+
} catch {
|
|
1347
|
+
return fallback;
|
|
1419
1348
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1349
|
+
}
|
|
1350
|
+
function errMsg2(err) {
|
|
1351
|
+
return err instanceof Error ? err.message : String(err);
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
// src/tools.ts
|
|
1355
|
+
function createVectorMemoryTools(store) {
|
|
1356
|
+
return [
|
|
1357
|
+
vectorMemoryRememberTool(store),
|
|
1358
|
+
vectorMemorySearchTool(store),
|
|
1359
|
+
vectorMemoryStatsTool(store),
|
|
1360
|
+
vectorMemoryForgetTool(store)
|
|
1361
|
+
];
|
|
1362
|
+
}
|
|
1363
|
+
function vectorMemoryRememberTool(store) {
|
|
1364
|
+
return {
|
|
1365
|
+
name: "vector_memory_remember",
|
|
1366
|
+
category: "Memory",
|
|
1367
|
+
description: "Persist a piece of knowledge into the local vector memory store. The text is embedded with the active embedding provider (transformers.js when available, otherwise the sage hashing provider). Returns the new entry id and whether an embedding was stored.",
|
|
1368
|
+
usageHint: "Store text you want to find later by *meaning*, not just exact keywords. Embeddings happen locally \u2014 no project text leaves the machine.",
|
|
1369
|
+
permission: "confirm",
|
|
1370
|
+
mutating: true,
|
|
1371
|
+
riskTier: "standard",
|
|
1372
|
+
timeoutMs: 5e3,
|
|
1373
|
+
capabilities: ["memory.write"],
|
|
1374
|
+
icon: "settings",
|
|
1375
|
+
inputSchema: {
|
|
1376
|
+
type: "object",
|
|
1377
|
+
properties: {
|
|
1378
|
+
text: { type: "string", minLength: 1, description: "The text to embed and store." },
|
|
1379
|
+
summary: { type: "string", description: "Optional short label." },
|
|
1380
|
+
tags: {
|
|
1381
|
+
type: "array",
|
|
1382
|
+
items: { type: "string" },
|
|
1383
|
+
description: "Optional tags for later filtering."
|
|
1384
|
+
},
|
|
1385
|
+
scope: {
|
|
1386
|
+
type: "string",
|
|
1387
|
+
enum: ["project", "user", "session"],
|
|
1388
|
+
description: "Visibility scope. Defaults to `project`."
|
|
1389
|
+
},
|
|
1390
|
+
kind: {
|
|
1391
|
+
type: "string",
|
|
1392
|
+
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1393
|
+
description: "Entry kind. Defaults to `note`."
|
|
1394
|
+
},
|
|
1395
|
+
metadata: {
|
|
1396
|
+
type: "object",
|
|
1397
|
+
additionalProperties: true,
|
|
1398
|
+
description: "Free-form metadata persisted as JSON."
|
|
1399
|
+
}
|
|
1400
|
+
},
|
|
1401
|
+
required: ["text"],
|
|
1402
|
+
additionalProperties: false
|
|
1403
|
+
},
|
|
1404
|
+
execute: async (input) => {
|
|
1405
|
+
const entry = await store.remember(input);
|
|
1406
|
+
return { id: entry.id, hasVector: entry.vector !== void 0 };
|
|
1426
1407
|
}
|
|
1427
1408
|
};
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1409
|
+
}
|
|
1410
|
+
function vectorMemorySearchTool(store) {
|
|
1411
|
+
return {
|
|
1412
|
+
name: "vector_memory_search",
|
|
1413
|
+
category: "Memory",
|
|
1414
|
+
description: "Semantic search over the vector memory store. Embeds the query with the active provider and returns the top-k entries ranked by cosine similarity. Returns an empty list when the embedding provider is unavailable \u2014 callers should fall back to lexical search.",
|
|
1415
|
+
usageHint: "Use when you want results ranked by meaning. Pairs well with sage `memory_search` for keyword precision.",
|
|
1416
|
+
permission: "auto",
|
|
1417
|
+
mutating: false,
|
|
1418
|
+
riskTier: "safe",
|
|
1419
|
+
timeoutMs: 5e3,
|
|
1420
|
+
capabilities: ["memory.read"],
|
|
1421
|
+
icon: "search",
|
|
1422
|
+
inputSchema: {
|
|
1423
|
+
type: "object",
|
|
1424
|
+
properties: {
|
|
1425
|
+
query: { type: "string", minLength: 1, description: "The natural-language query." },
|
|
1426
|
+
limit: {
|
|
1427
|
+
type: "number",
|
|
1428
|
+
minimum: 1,
|
|
1429
|
+
maximum: 100,
|
|
1430
|
+
description: "Max results (default 10)."
|
|
1431
|
+
},
|
|
1432
|
+
threshold: {
|
|
1433
|
+
type: "number",
|
|
1434
|
+
minimum: 0,
|
|
1435
|
+
maximum: 1,
|
|
1436
|
+
description: "Minimum cosine similarity. Results below the floor are dropped."
|
|
1437
|
+
},
|
|
1438
|
+
scope: {
|
|
1439
|
+
type: "string",
|
|
1440
|
+
enum: ["project", "user", "session"],
|
|
1441
|
+
description: "Restrict to a scope."
|
|
1442
|
+
},
|
|
1443
|
+
kind: {
|
|
1444
|
+
type: "string",
|
|
1445
|
+
enum: ["note", "fact", "summary", "snippet", "link"],
|
|
1446
|
+
description: "Restrict to a kind."
|
|
1448
1447
|
}
|
|
1448
|
+
},
|
|
1449
|
+
required: ["query"],
|
|
1450
|
+
additionalProperties: false
|
|
1451
|
+
},
|
|
1452
|
+
execute: async (input) => {
|
|
1453
|
+
const hits = await store.search(input.query, {
|
|
1454
|
+
limit: input.limit !== void 0 ? input.limit : void 0,
|
|
1455
|
+
threshold: input.threshold !== void 0 ? input.threshold : void 0,
|
|
1456
|
+
scope: input.scope,
|
|
1457
|
+
kind: input.kind
|
|
1449
1458
|
});
|
|
1450
|
-
|
|
1451
|
-
|
|
1459
|
+
return {
|
|
1460
|
+
hits: hits.map((h) => ({
|
|
1461
|
+
id: h.entry.id,
|
|
1462
|
+
score: h.score,
|
|
1463
|
+
text: h.entry.text,
|
|
1464
|
+
summary: h.entry.summary ?? void 0,
|
|
1465
|
+
tags: h.entry.tags
|
|
1466
|
+
}))
|
|
1467
|
+
};
|
|
1452
1468
|
}
|
|
1453
1469
|
};
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1470
|
+
}
|
|
1471
|
+
function vectorMemoryStatsTool(store) {
|
|
1472
|
+
return {
|
|
1473
|
+
name: "vector_memory_stats",
|
|
1474
|
+
category: "Memory",
|
|
1475
|
+
description: "Return counts, providers, and dimensions for the local vector memory store.",
|
|
1476
|
+
usageHint: "Cheap diagnostic \u2014 safe to call any time.",
|
|
1477
|
+
permission: "auto",
|
|
1478
|
+
mutating: false,
|
|
1479
|
+
riskTier: "safe",
|
|
1480
|
+
timeoutMs: 1e3,
|
|
1481
|
+
capabilities: ["memory.read"],
|
|
1482
|
+
icon: "search",
|
|
1483
|
+
inputSchema: {
|
|
1484
|
+
type: "object",
|
|
1485
|
+
properties: {},
|
|
1486
|
+
additionalProperties: false
|
|
1487
|
+
},
|
|
1488
|
+
execute: async () => store.stats()
|
|
1461
1489
|
};
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
if (typeof memoryId !== "string") return;
|
|
1465
|
-
void mirror(memoryId);
|
|
1466
|
-
});
|
|
1467
|
-
const offRecovered = events.onPattern("memory.recovered", (_event, payload) => {
|
|
1468
|
-
const memoryId = payload?.memoryId;
|
|
1469
|
-
if (typeof memoryId !== "string") return;
|
|
1470
|
-
void mirror(memoryId);
|
|
1471
|
-
});
|
|
1472
|
-
const offUpdated = events.onPattern("memory.updated", (_event, payload) => {
|
|
1473
|
-
const memoryId = payload?.memoryId;
|
|
1474
|
-
if (typeof memoryId !== "string") return;
|
|
1475
|
-
const status = payload?.status;
|
|
1476
|
-
if (status === "deleted") return;
|
|
1477
|
-
void mirror(memoryId);
|
|
1478
|
-
});
|
|
1479
|
-
const offDeleted = events.onPattern("memory.deleted", (_event, payload) => {
|
|
1480
|
-
const memoryId = payload?.memoryId;
|
|
1481
|
-
if (typeof memoryId !== "string") return;
|
|
1482
|
-
void forgetMirror(memoryId);
|
|
1483
|
-
});
|
|
1490
|
+
}
|
|
1491
|
+
function vectorMemoryForgetTool(store) {
|
|
1484
1492
|
return {
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1493
|
+
name: "vector_memory_forget",
|
|
1494
|
+
category: "Memory",
|
|
1495
|
+
description: "Remove an entry (and its vector) from the vector memory store.",
|
|
1496
|
+
usageHint: "Hard delete \u2014 no soft-delete tombstone. Use `vector_memory_search` to find the id first if you only have text.",
|
|
1497
|
+
permission: "confirm",
|
|
1498
|
+
mutating: true,
|
|
1499
|
+
riskTier: "standard",
|
|
1500
|
+
timeoutMs: 1e3,
|
|
1501
|
+
capabilities: ["memory.write"],
|
|
1502
|
+
icon: "settings",
|
|
1503
|
+
inputSchema: {
|
|
1504
|
+
type: "object",
|
|
1505
|
+
properties: {
|
|
1506
|
+
id: {
|
|
1507
|
+
type: "string",
|
|
1508
|
+
minLength: 1,
|
|
1509
|
+
description: "Entry id returned by `vector_memory_remember`."
|
|
1510
|
+
}
|
|
1511
|
+
},
|
|
1512
|
+
required: ["id"],
|
|
1513
|
+
additionalProperties: false
|
|
1514
|
+
},
|
|
1515
|
+
execute: async (input) => ({ removed: await store.forget(input.id) })
|
|
1491
1516
|
};
|
|
1492
1517
|
}
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1518
|
+
|
|
1519
|
+
// src/transformers-provider.ts
|
|
1520
|
+
var DEFAULT_VECTOR_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
1521
|
+
var DEFAULT_VECTOR_DIMENSIONS = 384;
|
|
1522
|
+
var DEFAULT_VECTOR_DTYPE = "q8";
|
|
1523
|
+
var TransformersEmbeddingProvider = class {
|
|
1524
|
+
id;
|
|
1525
|
+
dimensions;
|
|
1526
|
+
modelId;
|
|
1527
|
+
cacheDir;
|
|
1528
|
+
dtype;
|
|
1529
|
+
device;
|
|
1530
|
+
batchSize;
|
|
1531
|
+
maxChars;
|
|
1532
|
+
allowRemote;
|
|
1533
|
+
extractor;
|
|
1534
|
+
loadPromise;
|
|
1535
|
+
constructor(opts = {}) {
|
|
1536
|
+
this.modelId = opts.modelId ?? DEFAULT_VECTOR_MODEL_ID;
|
|
1537
|
+
this.cacheDir = opts.cacheDir;
|
|
1538
|
+
this.dtype = opts.dtype ?? DEFAULT_VECTOR_DTYPE;
|
|
1539
|
+
this.device = opts.device ?? "cpu";
|
|
1540
|
+
this.batchSize = opts.batchSize ?? 16;
|
|
1541
|
+
this.maxChars = opts.maxChars ?? 2048;
|
|
1542
|
+
this.allowRemote = opts.allowRemoteModels ?? true;
|
|
1543
|
+
this.dimensions = DEFAULT_VECTOR_DIMENSIONS;
|
|
1544
|
+
this.id = `transformers-js:${this.modelId}:${this.dtype}`;
|
|
1519
1545
|
}
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
async
|
|
1528
|
-
const markerPath2 = path3.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
|
|
1529
|
-
const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
1530
|
-
if (!opts.force) {
|
|
1546
|
+
/**
|
|
1547
|
+
* Synchronous capability check. Returns false when the optional
|
|
1548
|
+
* `@huggingface/transformers` dependency is not installed.
|
|
1549
|
+
*
|
|
1550
|
+
* NOTE: this probes via dynamic import and caches the result, but does
|
|
1551
|
+
* NOT load the model itself — model loading is deferred to `embed()`.
|
|
1552
|
+
*/
|
|
1553
|
+
async isAvailable() {
|
|
1531
1554
|
try {
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
if (Number.isFinite(at) && Date.now() - at < interval) {
|
|
1535
|
-
return { swept: false, reason: "throttled" };
|
|
1536
|
-
}
|
|
1555
|
+
await this.loadModule();
|
|
1556
|
+
return true;
|
|
1537
1557
|
} catch {
|
|
1558
|
+
return false;
|
|
1538
1559
|
}
|
|
1539
1560
|
}
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
}
|
|
1555
|
-
|
|
1556
|
-
// src/search-race.ts
|
|
1557
|
-
function previewText(text, maxLen) {
|
|
1558
|
-
if (text.length <= maxLen) return text;
|
|
1559
|
-
return text.slice(0, maxLen - 1) + "\u2026";
|
|
1560
|
-
}
|
|
1561
|
-
async function runSearchRace(query, lexical, vectorStore, options = {}) {
|
|
1562
|
-
const limit = options.limit ?? 20;
|
|
1563
|
-
const threshold = options.threshold ?? 0;
|
|
1564
|
-
let vectorHits = [];
|
|
1565
|
-
try {
|
|
1566
|
-
vectorHits = await vectorStore.search(query, {
|
|
1567
|
-
limit,
|
|
1568
|
-
...threshold > 0 ? { threshold } : {}
|
|
1569
|
-
});
|
|
1570
|
-
} catch {
|
|
1571
|
-
vectorHits = [];
|
|
1561
|
+
async embed(texts) {
|
|
1562
|
+
if (texts.length === 0) return [];
|
|
1563
|
+
const extractor = await this.getExtractor();
|
|
1564
|
+
const prepared = texts.map((t) => this.prepare(t));
|
|
1565
|
+
const batches = [];
|
|
1566
|
+
for (let i = 0; i < prepared.length; i += this.batchSize) {
|
|
1567
|
+
batches.push(prepared.slice(i, i + this.batchSize));
|
|
1568
|
+
}
|
|
1569
|
+
const results = [];
|
|
1570
|
+
for (const batch of batches) {
|
|
1571
|
+
const out = await extractor(batch, { pooling: "mean", normalize: true });
|
|
1572
|
+
results.push(...this.tensorToVectors(out, batch.length));
|
|
1573
|
+
}
|
|
1574
|
+
return results;
|
|
1572
1575
|
}
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
for (let i = 0; i < lexicalCapped.length; i++) {
|
|
1579
|
-
const mem = lexicalCapped[i];
|
|
1580
|
-
const score = lexicalCapped.length <= 1 ? 1 : 1 - i / Math.max(1, lexicalCapped.length - 1);
|
|
1581
|
-
const id = mem.id;
|
|
1582
|
-
seenIds.add(id);
|
|
1583
|
-
overlap.push({
|
|
1584
|
-
id,
|
|
1585
|
-
lexicalScore: score,
|
|
1586
|
-
vectorScore: null,
|
|
1587
|
-
// patched below when a vector hit carries this id
|
|
1588
|
-
preview: previewText(mem.text, 140)
|
|
1589
|
-
});
|
|
1576
|
+
/** Truncate + normalize text before embedding. */
|
|
1577
|
+
prepare(text) {
|
|
1578
|
+
if (!text) return "";
|
|
1579
|
+
const normalized = text.normalize("NFKC").trim();
|
|
1580
|
+
return normalized.length > this.maxChars ? normalized.slice(0, this.maxChars) : normalized;
|
|
1590
1581
|
}
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
const existing = overlapById.get(sageId);
|
|
1597
|
-
if (existing) {
|
|
1598
|
-
if (existing.vectorScore === null) {
|
|
1599
|
-
existing.vectorScore = hit.score;
|
|
1582
|
+
tensorToVectors(out, batchSize) {
|
|
1583
|
+
if (typeof out.tolist === "function") {
|
|
1584
|
+
const nested = out.tolist();
|
|
1585
|
+
if (Array.isArray(nested) && Array.isArray(nested[0])) {
|
|
1586
|
+
return nested.map((row) => Float32Array.from(row));
|
|
1600
1587
|
}
|
|
1601
|
-
|
|
1588
|
+
return [Float32Array.from(nested)];
|
|
1602
1589
|
}
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1590
|
+
const flat = out.data;
|
|
1591
|
+
if (flat instanceof Float32Array) {
|
|
1592
|
+
if (batchSize === 1) return [flat];
|
|
1593
|
+
const dim = flat.length / batchSize;
|
|
1594
|
+
const vectors = [];
|
|
1595
|
+
for (let i = 0; i < batchSize; i++) {
|
|
1596
|
+
vectors.push(Float32Array.from(flat.subarray(i * dim, (i + 1) * dim)));
|
|
1597
|
+
}
|
|
1598
|
+
return vectors;
|
|
1599
|
+
}
|
|
1600
|
+
if (Array.isArray(flat) && Array.isArray(flat[0])) {
|
|
1601
|
+
return flat.map((row) => Float32Array.from(row));
|
|
1602
|
+
}
|
|
1603
|
+
if (Array.isArray(flat)) {
|
|
1604
|
+
return [Float32Array.from(flat)];
|
|
1605
|
+
}
|
|
1606
|
+
throw new Error("TransformersEmbeddingProvider: unexpected pipeline output shape");
|
|
1607
|
+
}
|
|
1608
|
+
async getExtractor() {
|
|
1609
|
+
if (this.extractor) return this.extractor;
|
|
1610
|
+
if (!this.loadPromise) this.loadPromise = this.loadExtractor();
|
|
1611
|
+
this.extractor = await this.loadPromise;
|
|
1612
|
+
return this.extractor;
|
|
1613
|
+
}
|
|
1614
|
+
async loadExtractor() {
|
|
1615
|
+
const mod = await this.loadModule();
|
|
1616
|
+
if (this.cacheDir) mod.env.cacheDir = this.cacheDir;
|
|
1617
|
+
mod.env.allowRemoteModels = this.allowRemote;
|
|
1618
|
+
if (!this.allowRemote) mod.env.localModelPath = this.cacheDir ?? "";
|
|
1619
|
+
const pipe = await mod.pipeline("feature-extraction", this.modelId, {
|
|
1620
|
+
dtype: this.dtype,
|
|
1621
|
+
device: this.device
|
|
1610
1622
|
});
|
|
1623
|
+
return pipe;
|
|
1611
1624
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
});
|
|
1621
|
-
} else {
|
|
1622
|
-
finalOverlap.push(row);
|
|
1625
|
+
async loadModule() {
|
|
1626
|
+
try {
|
|
1627
|
+
return await import("@huggingface/transformers");
|
|
1628
|
+
} catch (err) {
|
|
1629
|
+
throw new VectorMemoryProviderUnavailableError(
|
|
1630
|
+
"@huggingface/transformers is not installed. Install it (pnpm add @huggingface/transformers) or wire a fallback EmbeddingProvider.",
|
|
1631
|
+
err
|
|
1632
|
+
);
|
|
1623
1633
|
}
|
|
1624
1634
|
}
|
|
1625
|
-
|
|
1626
|
-
const vectorCount = vectorOnly.length + finalOverlap.length;
|
|
1627
|
-
const denom = Math.max(lexicalCount, vectorCount, 1);
|
|
1628
|
-
return {
|
|
1629
|
-
query,
|
|
1630
|
-
lexicalOnly,
|
|
1631
|
-
vectorOnly,
|
|
1632
|
-
overlap: finalOverlap,
|
|
1633
|
-
metrics: {
|
|
1634
|
-
lexicalCount,
|
|
1635
|
-
vectorCount,
|
|
1636
|
-
overlapCount: finalOverlap.length,
|
|
1637
|
-
lexicalOnlyRatio: lexicalCount === 0 ? 0 : lexicalOnly.length / lexicalCount,
|
|
1638
|
-
vectorOnlyRatio: vectorCount === 0 ? 0 : vectorOnly.length / vectorCount,
|
|
1639
|
-
agreementRatio: finalOverlap.length / denom
|
|
1640
|
-
}
|
|
1641
|
-
};
|
|
1642
|
-
}
|
|
1635
|
+
};
|
|
1643
1636
|
export {
|
|
1644
1637
|
DEFAULT_SWEEP_INTERVAL_MS,
|
|
1645
1638
|
DEFAULT_VECTOR_DIMENSIONS,
|