@wrongstack/vector-memory 0.319.1 → 0.320.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +133 -48
- package/dist/sage-event-mirror.d.ts +53 -3
- package/dist/sage-port-wrapper.d.ts +52 -16
- package/dist/store.d.ts +19 -0
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export { SAGE_SYNC_MARKER_FILENAME, decideWhetherToSync, startFirstBootSageSync,
|
|
|
9
9
|
export { createVectorMemoryTools } from './tools.js';
|
|
10
10
|
export { asVectorRecallProvider, fuseWithVectorMemory, type SageFusionHit, type SageFusionOptions, } from './sage-fusion.js';
|
|
11
11
|
export { wrapMemoryPortWithVectorRecall, type VectorPortWrappingOptions, } from './sage-port-wrapper.js';
|
|
12
|
-
export { forgetStaleSageMirrors, subscribeVectorMemoryToSage, type VectorMemoryMirrorHandle, type VectorMemoryMirrorOptions, } from './sage-event-mirror.js';
|
|
12
|
+
export { DEFAULT_SWEEP_INTERVAL_MS, forgetStaleSageMirrors, SAGE_SWEEP_MARKER_FILENAME, subscribeVectorMemoryToSage, sweepStaleSageMirrors, type SweepStaleSageMirrorsOptions, type SweepStaleSageMirrorsResult, type VectorMemoryMirrorHandle, type VectorMemoryMirrorOptions, } from './sage-event-mirror.js';
|
|
13
13
|
export { runSearchRace, type SearchRaceChannelHit, type SearchRaceOptions, type SearchRaceResult, } from './search-race.js';
|
|
14
14
|
export type { SageSyncReport, VectorEntry, VectorEntryInput, VectorEntryWithVector, VectorKind, VectorMemoryStoreOptions, VectorScope, VectorSearchHit, VectorSearchOptions, VectorStoreStats, } from './types.js';
|
|
15
15
|
export { VectorMemoryError, VectorMemoryProviderUnavailableError, } from './errors.js';
|
package/dist/index.js
CHANGED
|
@@ -543,29 +543,59 @@ var VectorMemoryStore = class _VectorMemoryStore {
|
|
|
543
543
|
filters.push("e.kind = ?");
|
|
544
544
|
params.push(opts.kind);
|
|
545
545
|
}
|
|
546
|
-
const
|
|
547
|
-
`SELECT e.id
|
|
548
|
-
e.content_hash, e.created_at, e.updated_at,
|
|
549
|
-
v.vector AS vec_blob
|
|
546
|
+
const scanRows = this.db.prepare(
|
|
547
|
+
`SELECT e.id AS id, v.vector AS vec_blob
|
|
550
548
|
FROM entries e
|
|
551
549
|
JOIN vectors v ON v.entry_id = e.id
|
|
552
550
|
WHERE ${filters.join(" AND ")}`
|
|
553
551
|
).all(...params);
|
|
554
|
-
const
|
|
555
|
-
for (const row of
|
|
556
|
-
const
|
|
557
|
-
const vec = decodeVector(blob);
|
|
552
|
+
const top = [];
|
|
553
|
+
for (const row of scanRows) {
|
|
554
|
+
const vec = decodeVector(row.vec_blob);
|
|
558
555
|
const raw = cosineSimilarity(queryVec, vec);
|
|
559
556
|
const score = Math.max(0, Math.min(1, raw));
|
|
560
557
|
if (score < threshold) continue;
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
558
|
+
if (top.length >= limit && score <= (top[top.length - 1]?.score ?? 0)) continue;
|
|
559
|
+
let at = top.length;
|
|
560
|
+
while (at > 0 && (top[at - 1]?.score ?? 0) < score) at--;
|
|
561
|
+
top.splice(at, 0, { id: row.id, score, vector: vec });
|
|
562
|
+
if (top.length > limit) top.length = limit;
|
|
563
|
+
}
|
|
564
|
+
if (top.length === 0) return [];
|
|
565
|
+
const placeholders = top.map(() => "?").join(",");
|
|
566
|
+
const hydrated = this.db.prepare(
|
|
567
|
+
`SELECT id, text, summary, metadata, tags, scope, kind,
|
|
568
|
+
content_hash, created_at, updated_at
|
|
569
|
+
FROM entries WHERE id IN (${placeholders})`
|
|
570
|
+
).all(...top.map((t) => t.id));
|
|
571
|
+
const entryById = /* @__PURE__ */ new Map();
|
|
572
|
+
for (const row of hydrated) {
|
|
573
|
+
entryById.set(row.id, this.rowToEntry(row));
|
|
574
|
+
}
|
|
575
|
+
const scored = [];
|
|
576
|
+
for (const candidate of top) {
|
|
577
|
+
const entry = entryById.get(candidate.id);
|
|
578
|
+
if (!entry) continue;
|
|
579
|
+
const hit = { entry, score: candidate.score, providerId };
|
|
580
|
+
if (includeVectors) hit.vector = candidate.vector;
|
|
564
581
|
scored.push(hit);
|
|
565
582
|
}
|
|
566
|
-
scored
|
|
567
|
-
return scored.slice(0, limit);
|
|
583
|
+
return scored;
|
|
568
584
|
}
|
|
585
|
+
/**
|
|
586
|
+
* Page through entries, newest first.
|
|
587
|
+
*
|
|
588
|
+
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
589
|
+
* without the id tiebreak two entries written in the same millisecond can
|
|
590
|
+
* swap places between calls and a paging caller silently skips one.
|
|
591
|
+
*
|
|
592
|
+
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
593
|
+
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
594
|
+
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
595
|
+
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
596
|
+
* resumes from a position, and the rows a deletion removes are ones the
|
|
597
|
+
* sweep has already passed.
|
|
598
|
+
*/
|
|
569
599
|
list(opts = {}) {
|
|
570
600
|
this.assertOpen();
|
|
571
601
|
const where = [];
|
|
@@ -578,8 +608,12 @@ var VectorMemoryStore = class _VectorMemoryStore {
|
|
|
578
608
|
where.push("kind = ?");
|
|
579
609
|
params.push(opts.kind);
|
|
580
610
|
}
|
|
611
|
+
if (opts.after) {
|
|
612
|
+
where.push("(updated_at < ? OR (updated_at = ? AND id < ?))");
|
|
613
|
+
params.push(opts.after.updatedAt, opts.after.updatedAt, opts.after.id);
|
|
614
|
+
}
|
|
581
615
|
const sql = `SELECT * FROM entries ${where.length ? `WHERE ${where.join(" AND ")}` : ""}
|
|
582
|
-
ORDER BY updated_at DESC LIMIT ?`;
|
|
616
|
+
ORDER BY updated_at DESC, id DESC LIMIT ?`;
|
|
583
617
|
params.push(opts.limit ?? 100);
|
|
584
618
|
const rows = this.db.prepare(sql).all(...params);
|
|
585
619
|
return rows.map((r) => this.rowToEntry(r));
|
|
@@ -1261,20 +1295,11 @@ function clamp01(value) {
|
|
|
1261
1295
|
|
|
1262
1296
|
// src/sage-port-wrapper.ts
|
|
1263
1297
|
import {
|
|
1298
|
+
augmentLexicalWithVectorRecall,
|
|
1299
|
+
isSageVisibleForSearch,
|
|
1264
1300
|
SAGE_RETRIEVAL_CAPABILITY,
|
|
1265
1301
|
SAGE_SURFACE_CAPABILITY
|
|
1266
1302
|
} from "@wrongstack/sage";
|
|
1267
|
-
function mergeVectorRecall(options, recall, weight, threshold) {
|
|
1268
|
-
if (options && typeof options === "object" && "vectorRecall" in options && options["vectorRecall"]) {
|
|
1269
|
-
return options;
|
|
1270
|
-
}
|
|
1271
|
-
return {
|
|
1272
|
-
...options ?? {},
|
|
1273
|
-
vectorRecall: recall,
|
|
1274
|
-
...weight !== void 0 ? { vectorRecallWeight: weight } : {},
|
|
1275
|
-
...threshold !== void 0 ? { vectorRecallMinScore: threshold } : {}
|
|
1276
|
-
};
|
|
1277
|
-
}
|
|
1278
1303
|
function asVectorRecallProviderAdapter(store) {
|
|
1279
1304
|
return {
|
|
1280
1305
|
async search(query, opts) {
|
|
@@ -1295,18 +1320,39 @@ function asVectorRecallProviderAdapter(store) {
|
|
|
1295
1320
|
}
|
|
1296
1321
|
function wrapMemoryPortWithVectorRecall(port, options) {
|
|
1297
1322
|
const recall = options.vectorRecall ?? asVectorRecallProviderAdapter(options.store);
|
|
1298
|
-
const
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1323
|
+
const materializeFor = (searchOpts) => async (sageId) => {
|
|
1324
|
+
const surface = port.getCapability(SAGE_SURFACE_CAPABILITY);
|
|
1325
|
+
if (!surface?.getSage) return void 0;
|
|
1326
|
+
const memory = await surface.getSage(sageId);
|
|
1327
|
+
if (!memory) return void 0;
|
|
1328
|
+
return isSageVisibleForSearch(memory, searchOpts) ? memory : void 0;
|
|
1329
|
+
};
|
|
1330
|
+
const fusionOptions = (searchOpts) => ({
|
|
1331
|
+
vectorRecall: recall,
|
|
1332
|
+
materializeVectorOnly: materializeFor(searchOpts),
|
|
1333
|
+
...options.weight !== void 0 ? { vectorWeight: options.weight } : {},
|
|
1334
|
+
...options.threshold !== void 0 ? { threshold: options.threshold } : {},
|
|
1335
|
+
...options.vectorOnlyThreshold !== void 0 ? { vectorOnlyThreshold: options.vectorOnlyThreshold } : {},
|
|
1336
|
+
...options.maxMaterializations !== void 0 ? { maxMaterializations: options.maxMaterializations } : {},
|
|
1337
|
+
...searchOpts?.limit !== void 0 ? { limit: searchOpts.limit } : {}
|
|
1338
|
+
});
|
|
1339
|
+
const callerOwnsFusion = (searchOpts) => Boolean(searchOpts?.vectorRecall);
|
|
1340
|
+
const wrapSearchSage = (original) => async (query, searchOpts) => {
|
|
1341
|
+
const opts = searchOpts;
|
|
1342
|
+
const lexical = await original(query, searchOpts);
|
|
1343
|
+
if (callerOwnsFusion(opts)) return lexical;
|
|
1344
|
+
const fused = await augmentLexicalWithVectorRecall(query, lexical, fusionOptions(opts));
|
|
1345
|
+
return fused.map((hit) => hit.memory);
|
|
1346
|
+
};
|
|
1347
|
+
const wrapSearchWithBreakdown = (original) => async (query, searchOpts) => {
|
|
1348
|
+
const opts = searchOpts;
|
|
1349
|
+
const lexicalHits = await original(query, searchOpts);
|
|
1350
|
+
if (callerOwnsFusion(opts)) return lexicalHits;
|
|
1351
|
+
return augmentLexicalWithVectorRecall(
|
|
1302
1352
|
query,
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
weight,
|
|
1307
|
-
threshold
|
|
1308
|
-
)
|
|
1309
|
-
));
|
|
1353
|
+
lexicalHits.map((hit) => hit.memory),
|
|
1354
|
+
fusionOptions(opts)
|
|
1355
|
+
);
|
|
1310
1356
|
};
|
|
1311
1357
|
const wrapped = Object.create(
|
|
1312
1358
|
Object.getPrototypeOf(port),
|
|
@@ -1318,13 +1364,11 @@ function wrapMemoryPortWithVectorRecall(port, options) {
|
|
|
1318
1364
|
if (!original) return void 0;
|
|
1319
1365
|
return {
|
|
1320
1366
|
...original,
|
|
1321
|
-
searchSage:
|
|
1322
|
-
// The rich-breakdown variant uses the same options-merge
|
|
1323
|
-
// helper — pass the vector recall through so consumers that
|
|
1324
|
-
// want the per-channel score breakdown get the same fusion
|
|
1325
|
-
// behaviour as `searchSage`.
|
|
1367
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
1326
1368
|
...original.searchSageWithBreakdown ? {
|
|
1327
|
-
searchSageWithBreakdown:
|
|
1369
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
1370
|
+
original.searchSageWithBreakdown
|
|
1371
|
+
)
|
|
1328
1372
|
} : {}
|
|
1329
1373
|
};
|
|
1330
1374
|
}
|
|
@@ -1333,9 +1377,11 @@ function wrapMemoryPortWithVectorRecall(port, options) {
|
|
|
1333
1377
|
if (!original) return void 0;
|
|
1334
1378
|
return {
|
|
1335
1379
|
...original,
|
|
1336
|
-
searchSage:
|
|
1380
|
+
searchSage: wrapSearchSage(original.searchSage),
|
|
1337
1381
|
...original.searchSageWithBreakdown ? {
|
|
1338
|
-
searchSageWithBreakdown:
|
|
1382
|
+
searchSageWithBreakdown: wrapSearchWithBreakdown(
|
|
1383
|
+
original.searchSageWithBreakdown
|
|
1384
|
+
)
|
|
1339
1385
|
} : {}
|
|
1340
1386
|
};
|
|
1341
1387
|
}
|
|
@@ -1345,6 +1391,8 @@ function wrapMemoryPortWithVectorRecall(port, options) {
|
|
|
1345
1391
|
}
|
|
1346
1392
|
|
|
1347
1393
|
// src/sage-event-mirror.ts
|
|
1394
|
+
import * as fs3 from "node:fs";
|
|
1395
|
+
import * as path3 from "node:path";
|
|
1348
1396
|
import { getSageSurface as getSageSurface2 } from "@wrongstack/sage";
|
|
1349
1397
|
function subscribeVectorMemoryToSage(opts) {
|
|
1350
1398
|
const { store, memoryStore } = opts;
|
|
@@ -1435,14 +1483,18 @@ function subscribeVectorMemoryToSage(opts) {
|
|
|
1435
1483
|
}
|
|
1436
1484
|
};
|
|
1437
1485
|
}
|
|
1438
|
-
async function forgetStaleSageMirrors(store, memoryStore, logger) {
|
|
1486
|
+
async function forgetStaleSageMirrors(store, memoryStore, logger, options) {
|
|
1439
1487
|
const surface = getSageSurface2(memoryStore);
|
|
1440
1488
|
if (!surface) return { scanned: 0, removed: 0 };
|
|
1441
1489
|
let scanned = 0;
|
|
1442
1490
|
let removed = 0;
|
|
1443
|
-
|
|
1444
|
-
|
|
1491
|
+
const PAGE = Math.max(1, options?.pageSize ?? 500);
|
|
1492
|
+
let after;
|
|
1493
|
+
for (; ; ) {
|
|
1494
|
+
const page = store.list(after ? { limit: PAGE, after } : { limit: PAGE });
|
|
1445
1495
|
if (page.length === 0) break;
|
|
1496
|
+
const last = page[page.length - 1];
|
|
1497
|
+
after = { updatedAt: last.updatedAt, id: last.id };
|
|
1446
1498
|
for (const entry of page) {
|
|
1447
1499
|
scanned++;
|
|
1448
1500
|
const sageId = entry.metadata?.sageId;
|
|
@@ -1456,13 +1508,43 @@ async function forgetStaleSageMirrors(store, memoryStore, logger) {
|
|
|
1456
1508
|
logger?.warn?.(`vector-memory stale-mirror sweep failed for ${sageId}: ${errMsg2(err)}`);
|
|
1457
1509
|
}
|
|
1458
1510
|
}
|
|
1459
|
-
if (page.length <
|
|
1511
|
+
if (page.length < PAGE) break;
|
|
1460
1512
|
}
|
|
1461
1513
|
return { scanned, removed };
|
|
1462
1514
|
}
|
|
1463
1515
|
function errMsg2(err) {
|
|
1464
1516
|
return err instanceof Error ? err.message : String(err);
|
|
1465
1517
|
}
|
|
1518
|
+
var SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
|
|
1519
|
+
var DEFAULT_SWEEP_INTERVAL_MS = 60 * 6e4;
|
|
1520
|
+
async function sweepStaleSageMirrors(opts) {
|
|
1521
|
+
const markerPath2 = path3.join(opts.store.directory, SAGE_SWEEP_MARKER_FILENAME);
|
|
1522
|
+
const interval = opts.minIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
1523
|
+
if (!opts.force) {
|
|
1524
|
+
try {
|
|
1525
|
+
const raw = JSON.parse(fs3.readFileSync(markerPath2, "utf8"));
|
|
1526
|
+
const at = typeof raw.at === "string" ? Date.parse(raw.at) : Number.NaN;
|
|
1527
|
+
if (Number.isFinite(at) && Date.now() - at < interval) {
|
|
1528
|
+
return { swept: false, reason: "throttled" };
|
|
1529
|
+
}
|
|
1530
|
+
} catch {
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
try {
|
|
1534
|
+
fs3.writeFileSync(markerPath2, JSON.stringify({ at: (/* @__PURE__ */ new Date()).toISOString() }), "utf8");
|
|
1535
|
+
} catch {
|
|
1536
|
+
}
|
|
1537
|
+
try {
|
|
1538
|
+
const result = await forgetStaleSageMirrors(opts.store, opts.memoryStore, opts.logger);
|
|
1539
|
+
opts.logger?.debug?.(
|
|
1540
|
+
`vector-memory stale-mirror sweep: scanned=${result.scanned} removed=${result.removed}`
|
|
1541
|
+
);
|
|
1542
|
+
return { swept: true, ...result };
|
|
1543
|
+
} catch (err) {
|
|
1544
|
+
opts.logger?.warn?.(`vector-memory stale-mirror sweep failed: ${errMsg2(err)}`);
|
|
1545
|
+
return { swept: false, reason: errMsg2(err) };
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1466
1548
|
|
|
1467
1549
|
// src/search-race.ts
|
|
1468
1550
|
function previewText(text, maxLen) {
|
|
@@ -1549,9 +1631,11 @@ async function runSearchRace(query, lexical, vectorStore, options = {}) {
|
|
|
1549
1631
|
};
|
|
1550
1632
|
}
|
|
1551
1633
|
export {
|
|
1634
|
+
DEFAULT_SWEEP_INTERVAL_MS,
|
|
1552
1635
|
DEFAULT_VECTOR_DIMENSIONS,
|
|
1553
1636
|
DEFAULT_VECTOR_DTYPE,
|
|
1554
1637
|
DEFAULT_VECTOR_MODEL_ID,
|
|
1638
|
+
SAGE_SWEEP_MARKER_FILENAME,
|
|
1555
1639
|
SAGE_SYNC_MARKER_FILENAME,
|
|
1556
1640
|
TransformersEmbeddingProvider,
|
|
1557
1641
|
VECTOR_DIMENSIONS_KEY,
|
|
@@ -1574,6 +1658,7 @@ export {
|
|
|
1574
1658
|
runSearchRace,
|
|
1575
1659
|
startFirstBootSageSync,
|
|
1576
1660
|
subscribeVectorMemoryToSage,
|
|
1661
|
+
sweepStaleSageMirrors,
|
|
1577
1662
|
upsertEmbeddingCache,
|
|
1578
1663
|
wrapMemoryPortWithVectorRecall
|
|
1579
1664
|
};
|
|
@@ -34,14 +34,64 @@ export declare function subscribeVectorMemoryToSage(opts: VectorMemoryMirrorOpti
|
|
|
34
34
|
* Walks the store, looks up each `metadata.sageId` in the SAGE surface,
|
|
35
35
|
* and forgets entries whose SAGE id no longer resolves.
|
|
36
36
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* `memory.deleted
|
|
37
|
+
* This is the safety net for bulk operations — hygiene's archive/purge
|
|
38
|
+
* passes, `memory.cleared` — which emit a single top-level event rather than
|
|
39
|
+
* a per-memory `memory.deleted`, so the live mirror never sees them. Without
|
|
40
|
+
* a periodic sweep those rows stay in the vector store forever.
|
|
41
|
+
*
|
|
42
|
+
* A stale row is not a *correctness* hole: a semantic-only hit is resolved
|
|
43
|
+
* through `SageSurface.getSage` and re-checked with `isSageVisibleForSearch`,
|
|
44
|
+
* which rejects an archived or deleted memory. It is a *cost* — every stale
|
|
45
|
+
* row is scanned on every cosine pass and can consume one of the fusion's
|
|
46
|
+
* bounded `maxMaterializations` slots before being dropped.
|
|
47
|
+
*
|
|
48
|
+
* Hosts run this from the session-end teardown, throttled alongside SAGE
|
|
49
|
+
* hygiene (see `setupVectorMemory` / `startWebUI`).
|
|
40
50
|
*/
|
|
41
51
|
export declare function forgetStaleSageMirrors(store: VectorMemoryStore, memoryStore: MemoryPort, logger?: {
|
|
42
52
|
warn?(msg: string, ctx?: unknown): void | undefined;
|
|
53
|
+
},
|
|
54
|
+
/** Rows per keyset page. Exposed so tests can exercise multi-page walks. */
|
|
55
|
+
options?: {
|
|
56
|
+
pageSize?: number | undefined;
|
|
43
57
|
}): Promise<{
|
|
44
58
|
scanned: number;
|
|
45
59
|
removed: number;
|
|
46
60
|
}>;
|
|
61
|
+
/** Sidecar recording the last stale-mirror sweep, next to the vector db. */
|
|
62
|
+
export declare const SAGE_SWEEP_MARKER_FILENAME = "sage-mirror-sweep.json";
|
|
63
|
+
/** Default minimum gap between sweeps. Matches SAGE's auto-hygiene throttle. */
|
|
64
|
+
export declare const DEFAULT_SWEEP_INTERVAL_MS: number;
|
|
65
|
+
export interface SweepStaleSageMirrorsOptions {
|
|
66
|
+
store: VectorMemoryStore;
|
|
67
|
+
memoryStore: MemoryPort;
|
|
68
|
+
logger?: {
|
|
69
|
+
debug?(msg: string): void | undefined;
|
|
70
|
+
warn?(msg: string): void | undefined;
|
|
71
|
+
} | undefined;
|
|
72
|
+
/** Skip when the last sweep was more recent than this. Default 1 hour. */
|
|
73
|
+
minIntervalMs?: number | undefined;
|
|
74
|
+
/** Run regardless of the throttle (operator-forced re-sync). */
|
|
75
|
+
force?: boolean | undefined;
|
|
76
|
+
}
|
|
77
|
+
export interface SweepStaleSageMirrorsResult {
|
|
78
|
+
swept: boolean;
|
|
79
|
+
reason?: string;
|
|
80
|
+
scanned?: number;
|
|
81
|
+
removed?: number;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Throttled wrapper around {@link forgetStaleSageMirrors} for host wiring.
|
|
85
|
+
*
|
|
86
|
+
* The sweep is O(corpus) with one `getSage` per mirrored row, so it must not
|
|
87
|
+
* run on every boot of every surface — a project with the CLI and the WebUI
|
|
88
|
+
* open would otherwise sweep twice per session start. The throttle is a
|
|
89
|
+
* timestamp file beside the vector database rather than a process-local
|
|
90
|
+
* variable, precisely so that those two independent processes share it.
|
|
91
|
+
*
|
|
92
|
+
* Fail-open in every direction: an unreadable or corrupt marker is treated as
|
|
93
|
+
* "never swept", and a failed sweep is logged and swallowed. Callers
|
|
94
|
+
* fire-and-forget this during boot.
|
|
95
|
+
*/
|
|
96
|
+
export declare function sweepStaleSageMirrors(opts: SweepStaleSageMirrorsOptions): Promise<SweepStaleSageMirrorsResult>;
|
|
47
97
|
//# sourceMappingURL=sage-event-mirror.d.ts.map
|
|
@@ -1,21 +1,47 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Wrap an existing SAGE `MemoryPort` so that every `searchSage` /
|
|
3
|
-
* `
|
|
4
|
-
*
|
|
5
|
-
* underlying port's identity for callers that compare ports, but routes
|
|
6
|
-
* the read-side capability methods through a vector-augmented
|
|
7
|
-
* `searchSage`.
|
|
3
|
+
* `searchSageWithBreakdown` call fuses the port's lexical candidate set with
|
|
4
|
+
* a semantic recall from the local vector store.
|
|
8
5
|
*
|
|
9
6
|
* Why a wrapper and not a direct constructor change:
|
|
10
7
|
* - non-invasive: no migration needed for existing host construction
|
|
11
8
|
* - opt-in: hosts that don't want vector augmentation just don't wrap
|
|
12
9
|
* - testable: easy to mock the wrapper in unit tests
|
|
13
10
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
* `
|
|
17
|
-
*
|
|
18
|
-
*
|
|
11
|
+
* ## Why the fusion runs HERE and not inside the store
|
|
12
|
+
*
|
|
13
|
+
* The historical implementation merged a `vectorRecall` provider into the
|
|
14
|
+
* search *options* and let `SqliteSageStore.searchSage` do the fusion. That
|
|
15
|
+
* works only when the store is in-process. In production it is not: hosts
|
|
16
|
+
* build the port with `createProjectSageMemoryPort`, which returns a
|
|
17
|
+
* `ProjectSageMemoryPort` speaking line-delimited JSON to the per-project
|
|
18
|
+
* SAGE daemon (`encodeSageProjectServerMessage` = `JSON.stringify`).
|
|
19
|
+
* `JSON.stringify({ vectorRecall: { search: fn } })` yields
|
|
20
|
+
* `{"vectorRecall":{}}` — functions do not survive the wire — so the daemon
|
|
21
|
+
* saw a truthy-but-empty provider, threw `search is not a function` inside
|
|
22
|
+
* the fusion's fail-open `try`, and silently returned the lexical list.
|
|
23
|
+
* The entire semantic channel was dead in every production surface while
|
|
24
|
+
* every diagnostic reported it as wired.
|
|
25
|
+
*
|
|
26
|
+
* The vector store also *cannot* simply move into the daemon: it owns an
|
|
27
|
+
* ONNX embedding provider and `@wrongstack/vector-memory` already depends on
|
|
28
|
+
* `@wrongstack/sage`, so wiring it the other way is a dependency cycle.
|
|
29
|
+
*
|
|
30
|
+
* So the fusion runs on the host side of the boundary:
|
|
31
|
+
* 1. call the port's `searchSage` (remote or in-process) for the lexical list
|
|
32
|
+
* 2. query the local vector store for the semantic list
|
|
33
|
+
* 3. fuse with RRF via `augmentLexicalWithVectorRecall`
|
|
34
|
+
* 4. resolve vector-only hits by id through the port's surface capability,
|
|
35
|
+
* re-applying every visibility rule the lexical channel enforces in SQL
|
|
36
|
+
* (`isSageVisibleForSearch`)
|
|
37
|
+
*
|
|
38
|
+
* Step 4 is one round-trip per admitted vector-only hit, which is why the
|
|
39
|
+
* fusion is called with a `maxMaterializations` bound.
|
|
40
|
+
*
|
|
41
|
+
* The wrapper only augments read-side capabilities
|
|
42
|
+
* (`SAGE_RETRIEVAL_CAPABILITY` / `SAGE_SURFACE_CAPABILITY`). Other
|
|
43
|
+
* capabilities (write-side, hygiene, audit) pass through unchanged so the
|
|
44
|
+
* wrapper never widens the trust boundary.
|
|
19
45
|
*/
|
|
20
46
|
import type { MemoryPort } from '@wrongstack/core/types';
|
|
21
47
|
import { type VectorRecallProvider } from '@wrongstack/sage';
|
|
@@ -25,18 +51,28 @@ export interface VectorPortWrappingOptions {
|
|
|
25
51
|
store: VectorMemoryStore;
|
|
26
52
|
/**
|
|
27
53
|
* Optional pre-built provider. When omitted, the wrapper builds one via
|
|
28
|
-
* `
|
|
54
|
+
* `asVectorRecallProviderAdapter(store)`.
|
|
29
55
|
*/
|
|
30
56
|
vectorRecall?: VectorRecallProvider | undefined;
|
|
31
57
|
/**
|
|
32
|
-
* Cosine threshold forwarded to the vector backend.
|
|
33
|
-
* (keep all hits, let RRF decide).
|
|
58
|
+
* Cosine threshold forwarded to the vector backend. Undefined = no
|
|
59
|
+
* threshold (keep all hits, let RRF decide).
|
|
34
60
|
*/
|
|
35
61
|
threshold?: number | undefined;
|
|
36
62
|
/**
|
|
37
63
|
* Weight of the vector channel in the RRF blend. Default 0.3.
|
|
38
64
|
*/
|
|
39
65
|
weight?: number | undefined;
|
|
66
|
+
/**
|
|
67
|
+
* Cosine floor a semantic-only hit must clear before it is resolved and
|
|
68
|
+
* admitted. Falls back to the fusion's own default (0.62).
|
|
69
|
+
*/
|
|
70
|
+
vectorOnlyThreshold?: number | undefined;
|
|
71
|
+
/**
|
|
72
|
+
* Cap on by-id resolutions of semantic-only hits per search. Each one is a
|
|
73
|
+
* round-trip when the port is remote. Falls back to the fusion default.
|
|
74
|
+
*/
|
|
75
|
+
maxMaterializations?: number | undefined;
|
|
40
76
|
}
|
|
41
77
|
/**
|
|
42
78
|
* Adapt a `VectorMemoryStore` to the SAGE `VectorRecallProvider` contract.
|
|
@@ -45,9 +81,9 @@ export interface VectorPortWrappingOptions {
|
|
|
45
81
|
*/
|
|
46
82
|
export declare function asVectorRecallProviderAdapter(store: VectorMemoryStore): VectorRecallProvider;
|
|
47
83
|
/**
|
|
48
|
-
* Return a new `MemoryPort`
|
|
49
|
-
*
|
|
50
|
-
*
|
|
84
|
+
* Return a new `MemoryPort` whose `searchSage` / `searchSageWithBreakdown`
|
|
85
|
+
* fuse lexical and semantic recall. All other capabilities pass through
|
|
86
|
+
* unchanged.
|
|
51
87
|
*/
|
|
52
88
|
export declare function wrapMemoryPortWithVectorRecall(port: MemoryPort, options: VectorPortWrappingOptions): MemoryPort;
|
|
53
89
|
//# sourceMappingURL=sage-port-wrapper.d.ts.map
|
package/dist/store.d.ts
CHANGED
|
@@ -70,10 +70,29 @@ export declare class VectorMemoryStore {
|
|
|
70
70
|
/** Hard-delete an entry by id. Wrapped in `withFileLock` for cross-process safety. */
|
|
71
71
|
forget(id: string): Promise<boolean>;
|
|
72
72
|
search(query: string, opts?: VectorSearchOptions): Promise<VectorSearchHit[]>;
|
|
73
|
+
/**
|
|
74
|
+
* Page through entries, newest first.
|
|
75
|
+
*
|
|
76
|
+
* Ordering is `(updated_at, id)` DESC — `updated_at` alone is not unique, so
|
|
77
|
+
* without the id tiebreak two entries written in the same millisecond can
|
|
78
|
+
* swap places between calls and a paging caller silently skips one.
|
|
79
|
+
*
|
|
80
|
+
* Pagination is keyset (`after`), not offset, because the only caller that
|
|
81
|
+
* pages is `forgetStaleSageMirrors`, which *deletes as it walks*. Under
|
|
82
|
+
* `OFFSET` every deletion shifts the remaining rows left and the next page
|
|
83
|
+
* skips exactly as many entries as were removed. Keyset is immune: it
|
|
84
|
+
* resumes from a position, and the rows a deletion removes are ones the
|
|
85
|
+
* sweep has already passed.
|
|
86
|
+
*/
|
|
73
87
|
list(opts?: {
|
|
74
88
|
limit?: number;
|
|
75
89
|
scope?: VectorScope;
|
|
76
90
|
kind?: VectorKind;
|
|
91
|
+
/** Resume after this entry — pass the last row of the previous page. */
|
|
92
|
+
after?: {
|
|
93
|
+
updatedAt: string;
|
|
94
|
+
id: string;
|
|
95
|
+
} | undefined;
|
|
77
96
|
}): VectorEntry[];
|
|
78
97
|
reindexAll(): Promise<{
|
|
79
98
|
processed: number;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wrongstack/vector-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.320.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "WrongStack Vector Memory — an additional vector-search memory store powered by @huggingface/transformers (local ONNX embeddings), alongside the SAGE lexical memory system.",
|
|
6
6
|
"repository": {
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"README.md"
|
|
28
28
|
],
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"@wrongstack/core": "0.
|
|
31
|
-
"@wrongstack/persistence": "0.
|
|
32
|
-
"@wrongstack/sage": "0.
|
|
30
|
+
"@wrongstack/core": "0.320.1",
|
|
31
|
+
"@wrongstack/persistence": "0.320.1",
|
|
32
|
+
"@wrongstack/sage": "0.320.1"
|
|
33
33
|
},
|
|
34
34
|
"optionalDependencies": {
|
|
35
35
|
"@huggingface/transformers": "^4.2.0"
|