@avantf/dsh-mem 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +60 -4
- package/lib/engine/mem/db/dao/shared.d.ts +153 -0
- package/lib/engine/mem/dispatch.d.ts +19 -1
- package/lib/engine/mem/eval/index.d.ts +4 -0
- package/lib/engine/mem/index.d.ts +2 -0
- package/lib/engine/mem/runtime.d.ts +2 -2
- package/lib/engine/mem/store/entity_sweep.d.ts +121 -0
- package/lib/engine/mem/store/hybrid.d.ts +50 -0
- package/lib/engine/mem/store/knowledge.d.ts +116 -12
- package/lib/engine/mem/store/legs.d.ts +171 -0
- package/lib/engine/mem/store/lexical.d.ts +8 -0
- package/lib/engine/mem/store/memory.d.ts +84 -72
- package/lib/engine/mem/store/vector_repair.d.ts +248 -0
- package/lib/engine/mem/trustHeartbeat.d.ts +47 -0
- package/lib/engine/mem-contract/tools.d.ts +4 -0
- package/lib/engine/mem-contract/types.d.ts +37 -6
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/index.js +1653 -703
- package/lib/interface-version.json +1 -1
- package/lib/types/envinit-bootstrap.d.ts +1 -1
- package/lib/types/envinit.d.ts +0 -3
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/interface_gate.d.ts +70 -0
- package/lib/types/interface_gate.d.ts.map +1 -1
- package/lib/types/provision.d.ts +6 -0
- package/lib/types/provision.d.ts.map +1 -1
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/vectorMigration.d.ts +22 -18
- package/lib/types/vectorMigration.d.ts.map +1 -1
- package/package.json +5 -5
package/lib/index.js
CHANGED
|
@@ -708,7 +708,8 @@ const AdminUnion = z.discriminatedUnion("action", [
|
|
|
708
708
|
z.object({ action: z.literal("vectors_diagnose") }),
|
|
709
709
|
z.object({
|
|
710
710
|
action: z.literal("vectors_fix"),
|
|
711
|
-
dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。")
|
|
711
|
+
dry_run: z.boolean().optional().describe("true=只诊断不写入;可选,默认 false。"),
|
|
712
|
+
store: z.enum(["memory", "knowledge"]).optional().describe("只修某一个库(memory / knowledge);省略=两库都修。")
|
|
712
713
|
}),
|
|
713
714
|
z.object({ action: z.literal("contradict_check") }),
|
|
714
715
|
z.object({
|
|
@@ -1324,6 +1325,170 @@ function promptTextWarnings(sections) {
|
|
|
1324
1325
|
return out;
|
|
1325
1326
|
}
|
|
1326
1327
|
//#endregion
|
|
1328
|
+
//#region lib/types/interface_gate.js
|
|
1329
|
+
/** Read one member without letting a hostile module's getter escape. */
|
|
1330
|
+
function member(module, name) {
|
|
1331
|
+
try {
|
|
1332
|
+
return module?.[name];
|
|
1333
|
+
} catch {
|
|
1334
|
+
return;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
function cannotTell(reason) {
|
|
1338
|
+
return {
|
|
1339
|
+
status: "cannot-tell",
|
|
1340
|
+
reason
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
/**
|
|
1344
|
+
* Whether the base's shared capabilities may be used under `verdict`.
|
|
1345
|
+
*
|
|
1346
|
+
* The ONE mapping from the gate's verdict to the loader's decision, exported so it is asserted
|
|
1347
|
+
* directly and the loader and its test cannot drift: only `incompatible` withholds the base;
|
|
1348
|
+
* `cannot-tell` does not, because it is not a negative answer. A `false` here is still a DEGRADATION
|
|
1349
|
+
* (own prompt defaults, gate skipped, legacy provisioning), never a refused mount.
|
|
1350
|
+
*/
|
|
1351
|
+
function baseIsUsable(verdict) {
|
|
1352
|
+
return verdict.status !== "incompatible";
|
|
1353
|
+
}
|
|
1354
|
+
/**
|
|
1355
|
+
* Where the build's bake record sits when no explicit URL is given: beside the built ENTRY
|
|
1356
|
+
* (`lib/index.js`), which is where `scripts/link-envinit.mjs` writes it. A per-file `tsc` emit of
|
|
1357
|
+
* this module lives one directory below that entry and looks one level up; the first readable record
|
|
1358
|
+
* wins. Under `vitest` (running from `src/`) neither exists, which is the correct "not baked".
|
|
1359
|
+
*/
|
|
1360
|
+
const BAKED_CANDIDATES = ["./interface-version.json", "../interface-version.json"];
|
|
1361
|
+
/**
|
|
1362
|
+
* Read this artifact's baked requirement through the loaded base's own reader.
|
|
1363
|
+
* @param read - the base's `readInterfaceRequirement`, taken off the loaded module.
|
|
1364
|
+
* @param bakedUrl - test seam: read EXACTLY this record instead of walking {@link BAKED_CANDIDATES}.
|
|
1365
|
+
*/
|
|
1366
|
+
function requirementOf(read, bakedUrl) {
|
|
1367
|
+
const candidates = bakedUrl === void 0 ? BAKED_CANDIDATES.map((relative) => new URL(relative, import.meta.url)) : [bakedUrl];
|
|
1368
|
+
for (const url of candidates) {
|
|
1369
|
+
let requirement;
|
|
1370
|
+
try {
|
|
1371
|
+
requirement = read(url);
|
|
1372
|
+
} catch {
|
|
1373
|
+
continue;
|
|
1374
|
+
}
|
|
1375
|
+
const version = requirement?.interfaceVersion;
|
|
1376
|
+
if (typeof version === "number" && Number.isInteger(version) && version > 0) return { interfaceVersion: version };
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
/**
|
|
1380
|
+
* Ask the loaded base whether it implements the interface generation this artifact was built for.
|
|
1381
|
+
*
|
|
1382
|
+
* @param module - the loaded base module (only `checkInterface` / `readInterfaceRequirement` are
|
|
1383
|
+
* touched, each through a guarded read).
|
|
1384
|
+
* @param bakedUrl - test seam: the bake record to read; production leaves it undefined.
|
|
1385
|
+
* @returns a verdict that is never `incompatible` unless BOTH sides were readable and differ, and
|
|
1386
|
+
* never a throw.
|
|
1387
|
+
*/
|
|
1388
|
+
function interfaceVerdict(module, bakedUrl) {
|
|
1389
|
+
const check = member(module, "checkInterface");
|
|
1390
|
+
const read = member(module, "readInterfaceRequirement");
|
|
1391
|
+
if (typeof check !== "function" || typeof read !== "function") return cannotTell("the loaded base has no interface gate (checkInterface / readInterfaceRequirement missing) — it predates the runtime interface contract, so which generation it implements cannot be told");
|
|
1392
|
+
const requirement = requirementOf(read, bakedUrl);
|
|
1393
|
+
if (requirement === void 0) return cannotTell("this build has no baked interface requirement (lib/interface-version.json is missing or malformed), so the runtime interface gate cannot run");
|
|
1394
|
+
let verdict;
|
|
1395
|
+
try {
|
|
1396
|
+
verdict = check(requirement.interfaceVersion, module);
|
|
1397
|
+
} catch {
|
|
1398
|
+
return cannotTell("the loaded base threw while judging the interface generation");
|
|
1399
|
+
}
|
|
1400
|
+
const status = verdict?.status;
|
|
1401
|
+
if (status !== "ok" && status !== "incompatible" && status !== "cannot-tell") return cannotTell("the loaded base returned no usable interface verdict");
|
|
1402
|
+
return verdict;
|
|
1403
|
+
}
|
|
1404
|
+
/** Emit a warning without ever letting the logger turn it into a crash. */
|
|
1405
|
+
function warn$1(log, message) {
|
|
1406
|
+
try {
|
|
1407
|
+
log?.warn(message);
|
|
1408
|
+
} catch {}
|
|
1409
|
+
}
|
|
1410
|
+
function reasonOf$1(error) {
|
|
1411
|
+
return error instanceof Error ? error.message : String(error);
|
|
1412
|
+
}
|
|
1413
|
+
/**
|
|
1414
|
+
* Run the runtime interface gate and answer whether the loaded base's shared capabilities may be
|
|
1415
|
+
* used.
|
|
1416
|
+
*
|
|
1417
|
+
* `false` is a DEGRADATION the caller answers by taking its own full-degradation route (no runtime,
|
|
1418
|
+
* own prompt defaults, the compatibility gate skipped, legacy provisioning) — never a refused mount,
|
|
1419
|
+
* and never a throw. `cannot-tell` warns and answers `true`: "cannot tell" is not a negative answer,
|
|
1420
|
+
* so the base is used normally. The verdict's own reason is the base's; only the surrounding sentence
|
|
1421
|
+
* is the caller's ({@link DegradeWords}).
|
|
1422
|
+
*/
|
|
1423
|
+
function gateOrDegrade(module, log, words) {
|
|
1424
|
+
const verdict = interfaceVerdict(module);
|
|
1425
|
+
if (!baseIsUsable(verdict)) {
|
|
1426
|
+
warn$1(log, `${words.prefix} WARNING — interface: ${verdict.reason ?? "the loaded base implements another interface generation"}; the base's shared capabilities are NOT used (${words.withheld}) and the plugin mounts anyway`);
|
|
1427
|
+
return false;
|
|
1428
|
+
}
|
|
1429
|
+
if (verdict.status === "cannot-tell") warn$1(log, `${words.prefix} WARNING — interface: ${verdict.reason ?? "the interface generation cannot be told"}; using the loaded base anyway ("cannot tell" is never "incompatible")`);
|
|
1430
|
+
return true;
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* The guard that makes a loader's "never throws" true: every failure is one warning plus
|
|
1434
|
+
* `undefined`. The caller owns the sentence's tail (`words`) and its own cache discipline — a
|
|
1435
|
+
* FAILED load is not cached, so this must return `undefined` rather than reject (see
|
|
1436
|
+
* {@link createLoadCache}).
|
|
1437
|
+
*/
|
|
1438
|
+
async function loadGuarded(options, loadOnce, log, words) {
|
|
1439
|
+
try {
|
|
1440
|
+
return await loadOnce(options);
|
|
1441
|
+
} catch (error) {
|
|
1442
|
+
warn$1(log, `${words.prefix} WARNING — environment initialisation failed (${reasonOf$1(error)}); ${words.loadFailure}`);
|
|
1443
|
+
return;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1447
|
+
* Wrap a guarded loader in the cache discipline both startups share: the first call runs it, every
|
|
1448
|
+
* later and concurrent call shares its result, and a FAILED load is NOT cached. A failure is "cannot
|
|
1449
|
+
* tell", not a verdict — and it may be transient (offline at boot, a half-written install) — so a
|
|
1450
|
+
* profile reload, or a second instance in the same process, retries instead of replaying the first
|
|
1451
|
+
* failure for the whole process lifetime.
|
|
1452
|
+
*/
|
|
1453
|
+
function createLoadCache(guarded) {
|
|
1454
|
+
let loaded;
|
|
1455
|
+
let loading;
|
|
1456
|
+
return {
|
|
1457
|
+
load(options) {
|
|
1458
|
+
if (loaded !== void 0) return Promise.resolve(loaded);
|
|
1459
|
+
if (loading !== void 0) return loading;
|
|
1460
|
+
loading = guarded(options).then((runtime) => {
|
|
1461
|
+
if (runtime === void 0) loading = void 0;
|
|
1462
|
+
else loaded = runtime;
|
|
1463
|
+
return runtime;
|
|
1464
|
+
}, () => {
|
|
1465
|
+
loading = void 0;
|
|
1466
|
+
});
|
|
1467
|
+
return loading;
|
|
1468
|
+
},
|
|
1469
|
+
clear() {
|
|
1470
|
+
loaded = void 0;
|
|
1471
|
+
loading = void 0;
|
|
1472
|
+
}
|
|
1473
|
+
};
|
|
1474
|
+
}
|
|
1475
|
+
/**
|
|
1476
|
+
* Whether a Cordis context is still alive.
|
|
1477
|
+
*
|
|
1478
|
+
* Cordis clears `fiber.uid` on disposal, and every later context call then throws
|
|
1479
|
+
* `INACTIVE_EFFECT`. `apply` awaits environment preparation, so a profile reload or unload can land
|
|
1480
|
+
* inside that window and half-register a plugin (the Remote face cannot be rolled back). A fiberless
|
|
1481
|
+
* stub counts as active: an unknown shape is "cannot tell", never "disposed".
|
|
1482
|
+
*
|
|
1483
|
+
* Generic Cordis liveness rather than interface-gate knowledge — but the two plugins' startup paths
|
|
1484
|
+
* share it byte-for-byte, so it lives with their one shared shim, where the two readings of "is my
|
|
1485
|
+
* context alive" cannot drift.
|
|
1486
|
+
*/
|
|
1487
|
+
function stillActive(ctx) {
|
|
1488
|
+
const fiber = ctx?.fiber;
|
|
1489
|
+
return fiber === void 0 || fiber.uid !== null;
|
|
1490
|
+
}
|
|
1491
|
+
//#endregion
|
|
1327
1492
|
//#region lib/types/hints.js
|
|
1328
1493
|
/**
|
|
1329
1494
|
* The plugin's CONDITIONAL hint: ONE line, contributed only when either store already holds
|
|
@@ -3687,6 +3852,79 @@ async function dispatchToolKey(rt, key, args) {
|
|
|
3687
3852
|
if (dispatch === void 0) throw new Error(`unknown tool key ${key}`);
|
|
3688
3853
|
return dispatch(rt, args);
|
|
3689
3854
|
}
|
|
3855
|
+
/**
|
|
3856
|
+
* The ONE model-facing boundary: validate against the contract spec, dispatch the key, and shape the
|
|
3857
|
+
* result into the agent envelope.
|
|
3858
|
+
*
|
|
3859
|
+
* Every model-facing surface shares this — the DSH tool runner (`@avantf/dsh-mem`) and the MCP
|
|
3860
|
+
* server — because the envelope is `ToolEnvelope`, a wire shape: an MCP caller and a DSH caller
|
|
3861
|
+
* asking the same question must get the same fields. It used to live only in the plugin, so the MCP
|
|
3862
|
+
* surface re-implemented it and the two could drift field by field.
|
|
3863
|
+
*
|
|
3864
|
+
* What stays in the CALLERS is everything AROUND the envelope: MCP's `textResult`/`isError` wrapper
|
|
3865
|
+
* and the plugin's `present` view plus its Remote gateway. The Remote gateway is deliberately NOT
|
|
3866
|
+
* routed through here (`mem/DESIGN.md` §239: it keeps `mem_admin`'s retention diagnostics and uses
|
|
3867
|
+
* the wire error type, not `toolErr`).
|
|
3868
|
+
*
|
|
3869
|
+
* A validation failure answers `validationError` (named for the spec), an unknown key answers
|
|
3870
|
+
* `toolErr`, and a throwing runtime call answers `toolErr` too — a raw exception never escapes.
|
|
3871
|
+
*/
|
|
3872
|
+
async function runToolSpec$1(rt, spec, args) {
|
|
3873
|
+
const parsed = spec.input.safeParse(args ?? {});
|
|
3874
|
+
if (!parsed.success) return validationError(spec.name, parsed.error.issues);
|
|
3875
|
+
if (!supportsToolKey(spec.key)) return toolErr(`unknown tool ${spec.key}`);
|
|
3876
|
+
try {
|
|
3877
|
+
const value = await dispatchToolKey(rt, spec.key, parsed.data);
|
|
3878
|
+
return toolOk(modelFacingToolResult(spec.key, parsed.data.action, value));
|
|
3879
|
+
} catch (error) {
|
|
3880
|
+
return toolErr(error);
|
|
3881
|
+
}
|
|
3882
|
+
}
|
|
3883
|
+
//#endregion
|
|
3884
|
+
//#region ../core/lib/trustHeartbeat.js
|
|
3885
|
+
/**
|
|
3886
|
+
* The trust heartbeat: the interval beat that advances the active-day clock and performs the
|
|
3887
|
+
* settle / forget / idle / purge sweeps (TRUST_MODEL.md §5), shared by the DSH plugin and the MCP
|
|
3888
|
+
* server.
|
|
3889
|
+
*
|
|
3890
|
+
* The two surfaces used to install it separately, and the copies drifted in exactly the places a
|
|
3891
|
+
* copy drifts: the wording of the failure line and the work each beat does after the tick. The tick
|
|
3892
|
+
* itself — the one thing that MUST be identical, because it is the trust clock — is what lives
|
|
3893
|
+
* here. The differences are data: {@link TrustHeartbeatOptions.onBeat} is the extra work (the plugin
|
|
3894
|
+
* sweeps entities and retries the vector migration), {@link TrustHeartbeatOptions.tickFailed} is how
|
|
3895
|
+
* that surface names a failed sweep.
|
|
3896
|
+
*
|
|
3897
|
+
* `heartbeatMinutes <= 0` is the caller's decision, not this function's: a surface that disables the
|
|
3898
|
+
* beat also decides whether it logs the interval line.
|
|
3899
|
+
*
|
|
3900
|
+
* Zero dependencies beyond `setInterval` so it stays in the engine (the plugin's index and the MCP
|
|
3901
|
+
* entry both consume it).
|
|
3902
|
+
*/
|
|
3903
|
+
/** The generic one-line reason for a caught value. */
|
|
3904
|
+
function messageOf$1(error) {
|
|
3905
|
+
return error instanceof Error ? error.message : String(error);
|
|
3906
|
+
}
|
|
3907
|
+
/**
|
|
3908
|
+
* Install the heartbeat and return its disposer.
|
|
3909
|
+
*
|
|
3910
|
+
* A `trustTick` that throws is warned about and swallowed: a failed sweep must never take the host
|
|
3911
|
+
* or the server down, and it must not be invisible either — a trust clock that stopped advancing
|
|
3912
|
+
* shows up much later as facts that never settle, forget or purge. The timer is `unref`'d where the
|
|
3913
|
+
* runtime supports it, so it never holds a process open by itself.
|
|
3914
|
+
*/
|
|
3915
|
+
function installTrustHeartbeat(options) {
|
|
3916
|
+
const timer = setInterval(() => {
|
|
3917
|
+
try {
|
|
3918
|
+
options.memory.trustTick();
|
|
3919
|
+
} catch (error) {
|
|
3920
|
+
options.logger.warn((options.tickFailed ?? ((message) => `trust tick failed: ${message}`))(messageOf$1(error)));
|
|
3921
|
+
}
|
|
3922
|
+
options.onBeat?.();
|
|
3923
|
+
}, options.heartbeatMinutes * 6e4);
|
|
3924
|
+
const unref = timer.unref;
|
|
3925
|
+
if (typeof unref === "function") unref.call(timer);
|
|
3926
|
+
return () => clearInterval(timer);
|
|
3927
|
+
}
|
|
3690
3928
|
//#endregion
|
|
3691
3929
|
//#region ../retrieval-core/lib/representation.js
|
|
3692
3930
|
/**
|
|
@@ -6870,6 +7108,18 @@ function probeTerms(text, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
|
6870
7108
|
matched
|
|
6871
7109
|
};
|
|
6872
7110
|
}
|
|
7111
|
+
/**
|
|
7112
|
+
* The store probe both stores' `lexicalProbe` used to spell out verbatim: build THIS store's MATCH
|
|
7113
|
+
* expression for one term (the tokenizer is a property of the store's FTS table) and ask that
|
|
7114
|
+
* table whether anything holds it. `probeTerms` owns the term set and the counting; this owns the
|
|
7115
|
+
* one part that has to differ, which is now a single `holds` closure per store.
|
|
7116
|
+
*/
|
|
7117
|
+
function probeStoreTerms(text, tokenizer, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
7118
|
+
return probeTerms(text, (term) => {
|
|
7119
|
+
const fts = buildFtsQuery(term, tokenizer);
|
|
7120
|
+
return fts !== null && holds(fts);
|
|
7121
|
+
}, stopAt);
|
|
7122
|
+
}
|
|
6873
7123
|
//#endregion
|
|
6874
7124
|
//#region ../core/lib/store/self_query.js
|
|
6875
7125
|
/**
|
|
@@ -7393,6 +7643,60 @@ async function hybridSearch(deps, plan) {
|
|
|
7393
7643
|
};
|
|
7394
7644
|
}
|
|
7395
7645
|
/**
|
|
7646
|
+
* Map a store's search arguments onto {@link HybridPlan} and run {@link hybridSearch}. ONE copy.
|
|
7647
|
+
*
|
|
7648
|
+
* Before this, `MemoryStore.search` and `KnowledgeStore.search` carried the mapping verbatim side by
|
|
7649
|
+
* side: the same field list, the same conditional spreads, the same call. That list is part of the
|
|
7650
|
+
* orchestrator's contract, so an option added to one entry point and not the other would silently
|
|
7651
|
+
* make the two stores query differently — the same shape of drift this module already documents
|
|
7652
|
+
* (`NaN` limit, `over_fetch_factor`, unrecorded capped legs).
|
|
7653
|
+
*
|
|
7654
|
+
* The conditional spreads are deliberate, not cosmetic: a caller that did not pass `overFetch` /
|
|
7655
|
+
* `includeScores` / `relaxLegs` / `rewriteQuery` leaves the plan key ABSENT rather than
|
|
7656
|
+
* present-with-`undefined`. The orchestrator reads all four through `??` / `=== true` today, so the
|
|
7657
|
+
* two are indistinguishable now — but the plan is a contract, and widening it with keys the caller
|
|
7658
|
+
* never set is how "absent means default" decays into "present-undefined means set to nothing".
|
|
7659
|
+
*/
|
|
7660
|
+
async function runHybridSearch(deps, query, fields) {
|
|
7661
|
+
return hybridSearch(deps, hybridPlanOf(query, fields));
|
|
7662
|
+
}
|
|
7663
|
+
/**
|
|
7664
|
+
* The mapping itself, separated from the call so it can be pinned directly: the contract here is
|
|
7665
|
+
* the SHAPE of the plan (which keys exist), which no behavioural assertion inside `hybridSearch`
|
|
7666
|
+
* could observe once every optional field is read through `??` / `=== true`.
|
|
7667
|
+
*/
|
|
7668
|
+
function hybridPlanOf(query, fields) {
|
|
7669
|
+
return {
|
|
7670
|
+
query,
|
|
7671
|
+
limit: fields?.limit,
|
|
7672
|
+
...fields?.overFetch === void 0 ? {} : { overFetch: fields.overFetch },
|
|
7673
|
+
maxTokens: fields?.maxTokens,
|
|
7674
|
+
queryVector: fields?.queryVector,
|
|
7675
|
+
recordStats: fields?.recordStats,
|
|
7676
|
+
floors: fields?.floors,
|
|
7677
|
+
...fields?.includeScores === void 0 ? {} : { includeScores: fields.includeScores },
|
|
7678
|
+
...fields?.relaxLegs === void 0 ? {} : { relaxLegs: fields.relaxLegs },
|
|
7679
|
+
...fields?.rewriteQuery === void 0 ? {} : { rewriteQuery: fields.rewriteQuery }
|
|
7680
|
+
};
|
|
7681
|
+
}
|
|
7682
|
+
/**
|
|
7683
|
+
* Build a store's {@link HybridDeps} — the ONE construction site for the contract fields both stores
|
|
7684
|
+
* share (`kind` / `config` / `semantic` / `legs` / `texts` / `hits` / `onReturn`). The per-store half
|
|
7685
|
+
* is `parts`; the stores used to spell the whole object out twice, so a new required `HybridDeps`
|
|
7686
|
+
* field was a change to remember in two places.
|
|
7687
|
+
*/
|
|
7688
|
+
function makeHybridDeps(kind, config, semantic, parts) {
|
|
7689
|
+
return {
|
|
7690
|
+
kind,
|
|
7691
|
+
config,
|
|
7692
|
+
semantic,
|
|
7693
|
+
legs: parts.legs,
|
|
7694
|
+
texts: parts.texts,
|
|
7695
|
+
hits: parts.hits,
|
|
7696
|
+
...parts.onReturn === void 0 ? {} : { onReturn: parts.onReturn }
|
|
7697
|
+
};
|
|
7698
|
+
}
|
|
7699
|
+
/**
|
|
7396
7700
|
* P-01: put the fused per-leg evidence onto the caller-facing hits.
|
|
7397
7701
|
*
|
|
7398
7702
|
* Matched by `ref_id`, not by array index: a store's `hits` mapper may legitimately drop an entry
|
|
@@ -7496,6 +7800,266 @@ function describe(error) {
|
|
|
7496
7800
|
return error instanceof Error ? error.message : String(error);
|
|
7497
7801
|
}
|
|
7498
7802
|
//#endregion
|
|
7803
|
+
//#region ../core/lib/store/legs.js
|
|
7804
|
+
/**
|
|
7805
|
+
* The leg runner: how BOTH stores turn their own raw leg scores into `HybridLeg`s — plus the two
|
|
7806
|
+
* leg SOURCES that were the same flow twice ({@link ftsLeg}, {@link semanticLeg}, §4.6 of
|
|
7807
|
+
* `docs/vector-repair-shared-flow.md`). Both stores keep only their DAO/vstore primitives and their
|
|
7808
|
+
* scope: the `-rank`/`rank` sign convention, the short-query fallback, `max(50, k)`, the dimension
|
|
7809
|
+
* check's error and the Map insertion order live here once.
|
|
7810
|
+
*
|
|
7811
|
+
* `MemoryStore.searchLegs` and `KnowledgeStore.searchLegs` were the same skeleton written twice —
|
|
7812
|
+
* a `leg()` wrapper whose `capped` flag is measured on the RAW (pre-floor) score set, the
|
|
7813
|
+
* `applyTermFloor` / `applyScoreFloor` application, and the "semantic backend down ⇒ an EQUAL-WEIGHT
|
|
7814
|
+
* empty leg with `droppedByFloor: 0`" fallback — and even the comments explaining them were the same
|
|
7815
|
+
* paragraph twice. What legitimately differs is only the leg LIST and where each leg's scores come
|
|
7816
|
+
* from: memory computes anchors once and shares them between the entity and HRR legs, has the HRR
|
|
7817
|
+
* probe and the time-window scan; knowledge's entity leg is a query-driven async Jaccard. Those stay
|
|
7818
|
+
* at the store, exactly as §2 of `docs/vector-repair-shared-flow.md` requires.
|
|
7819
|
+
*
|
|
7820
|
+
* WHY THE CAP IS MEASURED ON THE RAW SET. `size === cap` is the only observable "this leg was cut"
|
|
7821
|
+
* signal (a leg that finished under the cap cannot have been trimmed), and it feeds the health
|
|
7822
|
+
* counters so the headroom can be sized from data — what makes the cap measurable instead of a silent
|
|
7823
|
+
* quality cliff (DESIGN §20.17). The relevance floors remove the tail anyway, so
|
|
7824
|
+
* deriving the flag from the FLOORED size would erase the signal exactly when the cap bound. The
|
|
7825
|
+
* floor functions only ever REMOVE entries, so the raw set is the honest measurement.
|
|
7826
|
+
*
|
|
7827
|
+
* WHY THE SEMANTIC LEG CARRIES NO `capped`. It is capped by the POOL size (`overFetch`), not by
|
|
7828
|
+
* `legCap`, so comparing it against `legCap` would report a trim that never happened. That is why
|
|
7829
|
+
* {@link LegRunner.semantic} is a separate method rather than a flag on {@link LegRunner.scored}:
|
|
7830
|
+
* the absence is structural, not something a caller can forget to set.
|
|
7831
|
+
*
|
|
7832
|
+
* WHY COMPOSITION. No base class (see the spec §2): the two stores share no construction, DB, DAO or
|
|
7833
|
+
* table, and a template-method base class would be a structural rewrite of two hub files. This is
|
|
7834
|
+
* the same style as `store/common.ts` — free functions plus a small interface — so a THIRD store can
|
|
7835
|
+
* be added by supplying its own leg list and sources and driving THIS runner (proved by a fake
|
|
7836
|
+
* third store in `test/legs.spec.ts`).
|
|
7837
|
+
*
|
|
7838
|
+
* CONCURRENCY lives one level up, in `store/hybrid.ts`'s `runLegs`: it awaits the returned
|
|
7839
|
+
* (value | promise) array concurrently and isolates a failing leg to itself. This module only BUILDS
|
|
7840
|
+
* legs, so both stores get the same awaiting and isolation by construction.
|
|
7841
|
+
*
|
|
7842
|
+
* @module store/legs
|
|
7843
|
+
*/
|
|
7844
|
+
/**
|
|
7845
|
+
* The one `capped` derivation (raw-set size vs cap). Kept as a named helper so the three call sites
|
|
7846
|
+
* cannot drift into measuring the floored set.
|
|
7847
|
+
*/
|
|
7848
|
+
function wasCapped(raw, cap) {
|
|
7849
|
+
return raw.size === cap;
|
|
7850
|
+
}
|
|
7851
|
+
/**
|
|
7852
|
+
* Build the leg runner for one query.
|
|
7853
|
+
*
|
|
7854
|
+
* The per-store leg ORDER is NOT here: it is the store's "leg list" (see §4.5 阶段 4) because leg
|
|
7855
|
+
* index is part of the contract — `hybrid.ts`'s `unionLegs` merges the original and the
|
|
7856
|
+
* self-reference rewrite runs BY INDEX, so a store that reordered its legs would silently re-map
|
|
7857
|
+
* evidence. What this runner guarantees is that every leg, whichever order it lands in, gets the
|
|
7858
|
+
* same wrapper, the same floor order and the same down-semantic fallback.
|
|
7859
|
+
*/
|
|
7860
|
+
function makeLegRunner(ctx) {
|
|
7861
|
+
const cap = ctx.legCap;
|
|
7862
|
+
return {
|
|
7863
|
+
cap,
|
|
7864
|
+
plain(scores, weight) {
|
|
7865
|
+
return {
|
|
7866
|
+
weight,
|
|
7867
|
+
scores,
|
|
7868
|
+
capped: wasCapped(scores, cap)
|
|
7869
|
+
};
|
|
7870
|
+
},
|
|
7871
|
+
scored(name, raw, weight, floor) {
|
|
7872
|
+
const floored = applyScoreFloor(raw, floor);
|
|
7873
|
+
return {
|
|
7874
|
+
weight,
|
|
7875
|
+
scores: floored.scores,
|
|
7876
|
+
capped: wasCapped(raw, cap),
|
|
7877
|
+
leg: name,
|
|
7878
|
+
droppedByFloor: floored.dropped
|
|
7879
|
+
};
|
|
7880
|
+
},
|
|
7881
|
+
term(name, raw, texts, query, weight, floor) {
|
|
7882
|
+
const floored = applyTermFloor(raw, texts, query, floor);
|
|
7883
|
+
return {
|
|
7884
|
+
weight,
|
|
7885
|
+
scores: floored.scores,
|
|
7886
|
+
capped: wasCapped(raw, cap),
|
|
7887
|
+
leg: name,
|
|
7888
|
+
droppedByFloor: floored.dropped
|
|
7889
|
+
};
|
|
7890
|
+
},
|
|
7891
|
+
semantic(raw, weight, floor) {
|
|
7892
|
+
const floored = applyScoreFloor(raw, floor);
|
|
7893
|
+
return {
|
|
7894
|
+
weight,
|
|
7895
|
+
scores: floored.scores,
|
|
7896
|
+
leg: "semantic",
|
|
7897
|
+
droppedByFloor: floored.dropped
|
|
7898
|
+
};
|
|
7899
|
+
},
|
|
7900
|
+
semanticOff(weight) {
|
|
7901
|
+
return {
|
|
7902
|
+
weight,
|
|
7903
|
+
scores: /* @__PURE__ */ new Map(),
|
|
7904
|
+
leg: "semantic",
|
|
7905
|
+
droppedByFloor: 0
|
|
7906
|
+
};
|
|
7907
|
+
}
|
|
7908
|
+
};
|
|
7909
|
+
}
|
|
7910
|
+
/**
|
|
7911
|
+
* The FTS leg, shared by `MemoryStore.ftsPath` and `KnowledgeStore.ftsPath` (§4.6.1).
|
|
7912
|
+
*
|
|
7913
|
+
* The two were line-for-line the same flow — `buildFtsQuery` → the `MATCH` statement with NEGATED
|
|
7914
|
+
* bm25, else `substringTerms` → the LIKE statement with `rank` — differing only in the DAO and the
|
|
7915
|
+
* order its scope parameters bind. Both conventions are load-bearing and preserved verbatim:
|
|
7916
|
+
*
|
|
7917
|
+
* - **bm25 is negative** (more negative = better match), so the `MATCH` branch negates it and higher
|
|
7918
|
+
* stays better everywhere downstream;
|
|
7919
|
+
* - **the LIKE `rank`** already counts contained terms, so it is used as-is, keeping the two branches
|
|
7920
|
+
* on the same "higher = better" scale;
|
|
7921
|
+
* - **an empty term list returns the empty map** (`substringTerms` yields none for a query with no
|
|
7922
|
+
* 3-char run) — the leg is empty by construction, not by a query that matched nothing.
|
|
7923
|
+
*
|
|
7924
|
+
* THE SHORT-QUERY FALLBACK. `buildFtsQuery` returning `null` means the query carries no term the
|
|
7925
|
+
* trigram index can express (measured: every term in the FTS table is 3 characters), which used to
|
|
7926
|
+
* leave this leg empty by construction — the 2-char CJK shape. `substringTerms` is the one finer
|
|
7927
|
+
* predicate the FTS5 trigram TABLE still answers (`LIKE '%…%'`), bounded by `cap`; the accuracy guard
|
|
7928
|
+
* is the per-row term floor in `applyTermFloor`, which requires the row to CONTAIN the run.
|
|
7929
|
+
*/
|
|
7930
|
+
function ftsLeg(opts) {
|
|
7931
|
+
const ftsQuery = buildFtsQuery(opts.query, opts.tokenizer);
|
|
7932
|
+
if (ftsQuery) {
|
|
7933
|
+
const rows = opts.search(ftsQuery, opts.scope, opts.cap);
|
|
7934
|
+
return new Map(rows.map((r) => [r.id, -r.rank]));
|
|
7935
|
+
}
|
|
7936
|
+
const terms = substringTerms(opts.query);
|
|
7937
|
+
if (terms.length === 0) return /* @__PURE__ */ new Map();
|
|
7938
|
+
const rows = opts.substringSearch(terms, opts.scope, opts.cap);
|
|
7939
|
+
return new Map(rows.map((r) => [r.id, r.rank]));
|
|
7940
|
+
}
|
|
7941
|
+
/**
|
|
7942
|
+
* The semantic leg, shared by `MemoryStore.semanticPath` and `KnowledgeStore.semanticPath` (§4.6.2).
|
|
7943
|
+
*
|
|
7944
|
+
* The flow: encode (unless the caller supplied the vector) → dimension check → publish → over-fetch
|
|
7945
|
+
* `max(50, k)` from the vstore → empty pool early-returns → filter → map id→score. Both stores used
|
|
7946
|
+
* to spell all of it out; only the filter differs, and that difference lives in `filterTopk`.
|
|
7947
|
+
*
|
|
7948
|
+
* Preserved verbatim: the `max(50, k)` over-fetch (the pool must leave room for the filter to remove
|
|
7949
|
+
* rows without shrinking the leg below `k`), the `RetrievalInputError` and its exact message, the
|
|
7950
|
+
* empty-pool early return, and the returned map's insertion order (the vstore's pool order).
|
|
7951
|
+
*/
|
|
7952
|
+
async function semanticLeg(opts) {
|
|
7953
|
+
const vec = opts.queryVector ?? await opts.encode(opts.query);
|
|
7954
|
+
if (vec.length !== opts.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${opts.dim}`);
|
|
7955
|
+
opts.onVector?.(vec);
|
|
7956
|
+
const topk = opts.topk(vec, Math.max(50, opts.k));
|
|
7957
|
+
if (!topk.length) return /* @__PURE__ */ new Map();
|
|
7958
|
+
const allowed = opts.filterTopk(topk.map((t) => t.id));
|
|
7959
|
+
const out = /* @__PURE__ */ new Map();
|
|
7960
|
+
for (const t of topk) if (allowed === void 0 || allowed.has(t.id)) out.set(t.id, t.score);
|
|
7961
|
+
return out;
|
|
7962
|
+
}
|
|
7963
|
+
//#endregion
|
|
7964
|
+
//#region ../core/lib/store/entity_sweep.js
|
|
7965
|
+
/**
|
|
7966
|
+
* The ONE entity-version sweep both stores run.
|
|
7967
|
+
*
|
|
7968
|
+
* `entities_version` is the stamp that says "these entity rows were produced by the CURRENT
|
|
7969
|
+
* extraction rules". A rule change makes every row carrying an older stamp stale, and the only way
|
|
7970
|
+
* to close that gap is to re-extract those rows' text — for memory the entity links, the triples and
|
|
7971
|
+
* the HRR bundle derived from them; for knowledge the `chunk_entities` rows. That SKELETON was
|
|
7972
|
+
* written twice and drifted apart in exactly the ways a copy drifts: memory had the in-flight guard,
|
|
7973
|
+
* the budget clamp and the `{rebuilt, deferred, skipped}` report, while knowledge had the same
|
|
7974
|
+
* selection inlined in `reindex` with no guard at all. This module is the one copy; the stores
|
|
7975
|
+
* contribute a per-aggregate adapter (which rows, how to extract, what to write) and nothing else.
|
|
7976
|
+
*
|
|
7977
|
+
* Layering, deliberate: this is an ORCHESTRATION rule, not SQL, so it lives beside `common.ts` and
|
|
7978
|
+
* `legs.ts` in `store/` — the DAO layer stays split by aggregate and keeps its own statements
|
|
7979
|
+
* (`docs/vector-repair-shared-flow.md` §2 / §7).
|
|
7980
|
+
*
|
|
7981
|
+
* What must NOT move here: the extractor, the rows a store writes, and that store's side effects.
|
|
7982
|
+
* Memory's transaction also replaces triples, moves the HRR bundle and re-queues the conflict check
|
|
7983
|
+
* (both conflict legs read what the sweep rebuilt); knowledge replaces `chunk_entities` under
|
|
7984
|
+
* `MAX_ENTITIES_PER_CHUNK`. Those are domain semantics, and folding them in would trade one
|
|
7985
|
+
* duplicated loop for two conditional branches.
|
|
7986
|
+
*
|
|
7987
|
+
* @module store/entity_sweep
|
|
7988
|
+
*/
|
|
7989
|
+
/**
|
|
7990
|
+
* Rows one sweep pass may rebuild when the caller names no budget.
|
|
7991
|
+
*
|
|
7992
|
+
* `MemoryStore.reindexEntities` has always defaulted to this, and the CLI passes it explicitly; it
|
|
7993
|
+
* lives here so the shared loop and that public re-export cannot drift. Bounds MEMORY, not only
|
|
7994
|
+
* mission: a pass selects every chosen row's text before rebuilding the first one.
|
|
7995
|
+
*/
|
|
7996
|
+
const ENTITY_SWEEP_BATCH = 2e3;
|
|
7997
|
+
/**
|
|
7998
|
+
* The module-level in-flight guard.
|
|
7999
|
+
*
|
|
8000
|
+
* Held here, not in the stores, because it protects THIS loop: two callers can overlap (the plugin's
|
|
8001
|
+
* heartbeat and `maintenance` are both periodic), and a limited ORDERED selection makes two
|
|
8002
|
+
* concurrent passes pick the same rows and re-tag them twice. Overlap is not corruption — each row is
|
|
8003
|
+
* written in its own transaction — it is duplicated mission in a single-threaded process, and the
|
|
8004
|
+
* answer is one pass reporting `skipped`. A `WeakSet` keyed by the caller's guard object keeps the
|
|
8005
|
+
* state in this module without adding a mutable field to either hub store, and without a module-level
|
|
8006
|
+
* boolean that would make two stores' sweeps block each other.
|
|
8007
|
+
*
|
|
8008
|
+
* The `guard` argument is therefore an IDENTITY, not a token: a store must pass the SAME object to
|
|
8009
|
+
* every call for its whole lifetime (both stores hold one as a private field). Two different objects
|
|
8010
|
+
* are two different sweeps and run concurrently — that per-store granularity is the point.
|
|
8011
|
+
*/
|
|
8012
|
+
const SWEEPING = /* @__PURE__ */ new WeakSet();
|
|
8013
|
+
/**
|
|
8014
|
+
* Re-extract the entity rows written by older extraction rules, ONE bounded pass.
|
|
8015
|
+
*
|
|
8016
|
+
* The contract, in order: take the guard (an overlapping pass reports `skipped` and does nothing);
|
|
8017
|
+
* read the scope's stale count and at most `maxPerPass` rows; re-extract each row through the
|
|
8018
|
+
* adapter, isolating failures; report `{ rebuilt, deferred, skipped }`.
|
|
8019
|
+
*
|
|
8020
|
+
* Not an unbounded drain, on purpose: this is reachable from a settings-page button and a periodic
|
|
8021
|
+
* heartbeat, so it must not block on a whole stale corpus. Callers that want the corpus current loop
|
|
8022
|
+
* until `deferred === 0` (the CLI does).
|
|
8023
|
+
*
|
|
8024
|
+
* A row whose extraction or write throws is skipped, not fatal: the transaction that stamps the
|
|
8025
|
+
* version never ran, so the row is still stale and the NEXT pass retries it — the same retry contract
|
|
8026
|
+
* the knowledge store used per chunk. Letting the throw escape would instead abandon the rest of the
|
|
8027
|
+
* batch with no report of what was done.
|
|
8028
|
+
*
|
|
8029
|
+
* @param guard the store's stable in-flight identity (see {@link SWEEPING})
|
|
8030
|
+
*/
|
|
8031
|
+
async function sweepVersionedEntities(target, guard, opts = {}) {
|
|
8032
|
+
const maxPerPass = Math.max(0, Math.floor(opts.budget ?? 2e3));
|
|
8033
|
+
if (SWEEPING.has(guard)) return {
|
|
8034
|
+
rebuilt: 0,
|
|
8035
|
+
deferred: target.staleEntityCount?.() ?? target.staleEntityBatch(0).total,
|
|
8036
|
+
skipped: true
|
|
8037
|
+
};
|
|
8038
|
+
SWEEPING.add(guard);
|
|
8039
|
+
try {
|
|
8040
|
+
const { total, rows } = target.staleEntityBatch(maxPerPass);
|
|
8041
|
+
let rebuilt = 0;
|
|
8042
|
+
for (const row of rows) {
|
|
8043
|
+
if (opts.shouldStop?.() === true) break;
|
|
8044
|
+
if (rebuilt >= maxPerPass) break;
|
|
8045
|
+
try {
|
|
8046
|
+
await target.extractAndWriteEntities(row);
|
|
8047
|
+
} catch (error) {
|
|
8048
|
+
target.onRowError(row, error);
|
|
8049
|
+
continue;
|
|
8050
|
+
}
|
|
8051
|
+
rebuilt += 1;
|
|
8052
|
+
}
|
|
8053
|
+
return {
|
|
8054
|
+
rebuilt,
|
|
8055
|
+
deferred: Math.max(0, total - rebuilt),
|
|
8056
|
+
skipped: false
|
|
8057
|
+
};
|
|
8058
|
+
} finally {
|
|
8059
|
+
SWEEPING.delete(guard);
|
|
8060
|
+
}
|
|
8061
|
+
}
|
|
8062
|
+
//#endregion
|
|
7499
8063
|
//#region ../core/lib/db/vectors.js
|
|
7500
8064
|
/** Little-endian Float32 ↔ BLOB helpers for persisted vectors (cross-platform stable). */
|
|
7501
8065
|
function float32ToBytes(vec) {
|
|
@@ -7687,15 +8251,289 @@ async function forEachYielding(items, visit, every = 200) {
|
|
|
7687
8251
|
if ((index + 1) % every === 0) await yieldToEventLoop();
|
|
7688
8252
|
}
|
|
7689
8253
|
}
|
|
7690
|
-
//#endregion
|
|
7691
|
-
//#region ../core/lib/store/secret_guard.js
|
|
7692
8254
|
/**
|
|
7693
|
-
*
|
|
8255
|
+
* Rebuild one store's live vstore from its persisted rows, count what is unusable in the current
|
|
8256
|
+
* space, and say so once — the shared "open for business" step both stores used to write twice.
|
|
7694
8257
|
*
|
|
7695
|
-
* The
|
|
7696
|
-
*
|
|
7697
|
-
*
|
|
7698
|
-
*
|
|
8258
|
+
* The owning store's ONLY half is the row read (`facts.activeVectorRows()` /
|
|
8259
|
+
* `chunks.vectorRows()`); the reload, the count and the loud warning are one implementation, so a
|
|
8260
|
+
* change to what counts as stale cannot land on one store only (the incident this exists for: the
|
|
8261
|
+
* memory store checked the SPACE while knowledge checked only the WIDTH, so a model swap mixed two
|
|
8262
|
+
* spaces in one index until someone happened to run `kb_reindex`).
|
|
8263
|
+
*/
|
|
8264
|
+
function loadVectorIndex(opts) {
|
|
8265
|
+
reloadVectorIndex(opts.vstore, opts.rows, opts.store);
|
|
8266
|
+
const expectedBytes = opts.vstore.dim * 4;
|
|
8267
|
+
let stale = 0;
|
|
8268
|
+
let spaceStale = 0;
|
|
8269
|
+
let legacy = 0;
|
|
8270
|
+
for (const row of opts.rows) {
|
|
8271
|
+
if (row.vec === null || row.vec.byteLength !== expectedBytes) {
|
|
8272
|
+
stale++;
|
|
8273
|
+
continue;
|
|
8274
|
+
}
|
|
8275
|
+
if (row.embedding_model !== opts.space) {
|
|
8276
|
+
spaceStale++;
|
|
8277
|
+
if (isPreFingerprintSpace(row.embedding_model)) legacy++;
|
|
8278
|
+
}
|
|
8279
|
+
}
|
|
8280
|
+
reportStaleVectors(opts.store, opts.space, {
|
|
8281
|
+
stale,
|
|
8282
|
+
space_stale: spaceStale,
|
|
8283
|
+
legacy
|
|
8284
|
+
}, opts.manualEntry);
|
|
8285
|
+
return {
|
|
8286
|
+
stale,
|
|
8287
|
+
space_stale: spaceStale,
|
|
8288
|
+
legacy
|
|
8289
|
+
};
|
|
8290
|
+
}
|
|
8291
|
+
/**
|
|
8292
|
+
* Encode rows one at a time, add each to the live index, and persist them in groups.
|
|
8293
|
+
*
|
|
8294
|
+
* The shared skeleton: serial encode (a MEASURED decision for chunks — a padded batched forward pass
|
|
8295
|
+
* is 1.79x slower on real 53–550 character chunks), `add` right after a successful encode, one
|
|
8296
|
+
* `write` call per `writeBatch` rows, per-row isolation of encode failures, per-group isolation of
|
|
8297
|
+
* write failures, and the `{ encoded, failed }` count.
|
|
8298
|
+
*
|
|
8299
|
+
* What stays with the store: the backend call, the live index, the write statement, the replay of a
|
|
8300
|
+
* failed group, and the summary log — the memory store logs per row and keeps a half-written index
|
|
8301
|
+
* (documented as the benign direction for it), the knowledge store evicts the group so both sides
|
|
8302
|
+
* agree that the vectors are missing.
|
|
8303
|
+
*/
|
|
8304
|
+
async function encodeAndPersist(rows, target) {
|
|
8305
|
+
const every = Math.max(1, Math.floor(target.writeBatch ?? 1));
|
|
8306
|
+
let encoded = 0;
|
|
8307
|
+
let failed = 0;
|
|
8308
|
+
for (let i = 0; i < rows.length; i += every) {
|
|
8309
|
+
const group = rows.slice(i, i + every);
|
|
8310
|
+
const writes = [];
|
|
8311
|
+
for (const row of group) try {
|
|
8312
|
+
const vec = await target.encode(row.text);
|
|
8313
|
+
target.add(row.id, vec);
|
|
8314
|
+
writes.push({
|
|
8315
|
+
id: row.id,
|
|
8316
|
+
bytes: float32ToBytes(vec)
|
|
8317
|
+
});
|
|
8318
|
+
} catch (error) {
|
|
8319
|
+
failed += 1;
|
|
8320
|
+
target.onRowError?.(row, error);
|
|
8321
|
+
}
|
|
8322
|
+
if (!writes.length) continue;
|
|
8323
|
+
try {
|
|
8324
|
+
target.write(writes);
|
|
8325
|
+
encoded += writes.length;
|
|
8326
|
+
} catch (error) {
|
|
8327
|
+
failed += writes.length;
|
|
8328
|
+
target.onWriteError?.(writes, error);
|
|
8329
|
+
}
|
|
8330
|
+
}
|
|
8331
|
+
return {
|
|
8332
|
+
encoded,
|
|
8333
|
+
failed
|
|
8334
|
+
};
|
|
8335
|
+
}
|
|
8336
|
+
/**
|
|
8337
|
+
* ONE bounded slice of the repair: re-add unusable-but-readable vectors that the live index lost,
|
|
8338
|
+
* drop the rows a re-encode has to replace, then encode up to `limit` missing/foreign rows.
|
|
8339
|
+
*
|
|
8340
|
+
* `limit` is what makes the SAME logic usable as a background migration: one call is a bounded unit
|
|
8341
|
+
* of work the caller can yield between. The explicit fix passes no limit (the operator action drains
|
|
8342
|
+
* everything); the background drive passes its batch size.
|
|
8343
|
+
*
|
|
8344
|
+
* The ORDER is an invariant: unusable bytes leave the table and the live index first, so a
|
|
8345
|
+
* re-encode can pick them up and a process that dies mid-migration leaves at most one batch
|
|
8346
|
+
* vector-less — exactly the `missing` rows the next run resumes on.
|
|
8347
|
+
*/
|
|
8348
|
+
async function repairVectorSlice(target, opts) {
|
|
8349
|
+
const limit = opts.limit ?? Number.POSITIVE_INFINITY;
|
|
8350
|
+
const c = target.classifyVectors(limit);
|
|
8351
|
+
const loadedNow = target.semanticAvailable();
|
|
8352
|
+
let semAvailable = loadedNow;
|
|
8353
|
+
if (!semAvailable && !opts.dryRun && target.warmup !== void 0) semAvailable = await target.warmup();
|
|
8354
|
+
const wouldWarm = !loadedNow;
|
|
8355
|
+
const remainingOf = (encoded) => Math.max(0, c.stale + c.space_stale + c.missing - encoded);
|
|
8356
|
+
const base = {
|
|
8357
|
+
stale: c.stale,
|
|
8358
|
+
space_stale: c.space_stale,
|
|
8359
|
+
missing: c.missing,
|
|
8360
|
+
semantic_available: semAvailable,
|
|
8361
|
+
would_warm: wouldWarm
|
|
8362
|
+
};
|
|
8363
|
+
if (opts.dryRun) return {
|
|
8364
|
+
report: {
|
|
8365
|
+
...base,
|
|
8366
|
+
...c.unindexed === void 0 ? {} : {
|
|
8367
|
+
unindexed: c.unindexed.length,
|
|
8368
|
+
reindexed: 0
|
|
8369
|
+
},
|
|
8370
|
+
dropped: 0,
|
|
8371
|
+
encoded: 0,
|
|
8372
|
+
failed: 0
|
|
8373
|
+
},
|
|
8374
|
+
remaining: remainingOf(0)
|
|
8375
|
+
};
|
|
8376
|
+
let reindexed = 0;
|
|
8377
|
+
if (c.unindexed !== void 0) reindexed = target.reindexUsable?.(c.unindexed.slice(0, limit)) ?? 0;
|
|
8378
|
+
let dropped = 0;
|
|
8379
|
+
if (semAvailable && c.dropIds.length > 0) dropped = target.dropVectors(c.dropIds);
|
|
8380
|
+
const { encoded, failed } = semAvailable ? await target.reencodeVectors(limit) : {
|
|
8381
|
+
encoded: 0,
|
|
8382
|
+
failed: 0
|
|
8383
|
+
};
|
|
8384
|
+
return {
|
|
8385
|
+
report: {
|
|
8386
|
+
...base,
|
|
8387
|
+
...c.unindexed === void 0 ? {} : {
|
|
8388
|
+
unindexed: c.unindexed.length,
|
|
8389
|
+
reindexed
|
|
8390
|
+
},
|
|
8391
|
+
dropped,
|
|
8392
|
+
encoded,
|
|
8393
|
+
failed
|
|
8394
|
+
},
|
|
8395
|
+
remaining: remainingOf(encoded)
|
|
8396
|
+
};
|
|
8397
|
+
}
|
|
8398
|
+
/**
|
|
8399
|
+
* ONE bounded slice of the background migration — the unit the drive (and the tests) step.
|
|
8400
|
+
*
|
|
8401
|
+
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
8402
|
+
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
8403
|
+
*/
|
|
8404
|
+
async function migrateVectorSlice(target, batchSize = 16) {
|
|
8405
|
+
const { report, remaining } = await repairVectorSlice(target, {
|
|
8406
|
+
dryRun: false,
|
|
8407
|
+
limit: Math.max(1, Math.floor(batchSize))
|
|
8408
|
+
});
|
|
8409
|
+
return {
|
|
8410
|
+
remaining,
|
|
8411
|
+
migrated: report.encoded,
|
|
8412
|
+
dropped: report.dropped,
|
|
8413
|
+
reindexed: report.reindexed ?? 0,
|
|
8414
|
+
semantic_available: report.semantic_available
|
|
8415
|
+
};
|
|
8416
|
+
}
|
|
8417
|
+
/** ACTIVE rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
8418
|
+
function migrationRemaining(target) {
|
|
8419
|
+
const health = target.vectorSpaceHealth();
|
|
8420
|
+
return health.stale + health.space_stale + target.missingVectorCount();
|
|
8421
|
+
}
|
|
8422
|
+
/** The `auto_migrate: false` warning is one per process per store, like the startup warning. */
|
|
8423
|
+
const disabledWarned = /* @__PURE__ */ new WeakSet();
|
|
8424
|
+
/**
|
|
8425
|
+
* Drive the bounded background migration for one store until it is current, the model is
|
|
8426
|
+
* unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). A batch FAILURE
|
|
8427
|
+
* never rejects: it is logged and reported as `remaining > 0`, which is what makes the next
|
|
8428
|
+
* heartbeat a retry. Never blocks the event loop for longer than one batch.
|
|
8429
|
+
*
|
|
8430
|
+
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
8431
|
+
* on background re-encoding can leave the loud warning and the manual entry as the only path. The
|
|
8432
|
+
* switch is read through the adapter so a new store cannot forget it.
|
|
8433
|
+
*/
|
|
8434
|
+
async function driveVectorRepair(target, opts = {}) {
|
|
8435
|
+
const log = opts.log ?? retrievalLogger();
|
|
8436
|
+
const stopped = () => opts.shouldStop?.() === true;
|
|
8437
|
+
const enabled = target.autoMigrate() !== false;
|
|
8438
|
+
const snapshot = () => ({
|
|
8439
|
+
enabled,
|
|
8440
|
+
remaining: migrationRemaining(target),
|
|
8441
|
+
migrated: 0,
|
|
8442
|
+
dropped: 0,
|
|
8443
|
+
reindexed: 0,
|
|
8444
|
+
semantic_available: target.semanticAvailable()
|
|
8445
|
+
});
|
|
8446
|
+
const initial = snapshot();
|
|
8447
|
+
if (stopped()) return initial;
|
|
8448
|
+
const health = target.vectorSpaceHealth();
|
|
8449
|
+
const isSpaceChange = health.stale + health.space_stale > 0;
|
|
8450
|
+
if (!enabled) {
|
|
8451
|
+
if (initial.remaining > 0 && !disabledWarned.has(target)) {
|
|
8452
|
+
disabledWarned.add(target);
|
|
8453
|
+
log.warn(`${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) ${isSpaceChange ? "belong to an older embedding space" : "have no vector at all"} and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run ${target.manualEntry} or set \`semantic.auto_migrate: true\``);
|
|
8454
|
+
}
|
|
8455
|
+
return initial;
|
|
8456
|
+
}
|
|
8457
|
+
if (initial.remaining === 0) return initial;
|
|
8458
|
+
const startedAt = Date.now();
|
|
8459
|
+
log.info(isSpaceChange ? `${target.store} vector migration: ${String(initial.remaining)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)` : `${target.store} vector migration: ${String(initial.remaining)} ACTIVE row(s) have no vector at all — encoding them in the background`);
|
|
8460
|
+
let migrated = 0;
|
|
8461
|
+
let dropped = 0;
|
|
8462
|
+
let reindexed = 0;
|
|
8463
|
+
let last = initial;
|
|
8464
|
+
for (;;) {
|
|
8465
|
+
if (stopped()) break;
|
|
8466
|
+
let step;
|
|
8467
|
+
try {
|
|
8468
|
+
step = await migrateVectorSlice(target, opts.batchSize);
|
|
8469
|
+
} catch (error) {
|
|
8470
|
+
log.warn(`${target.store} vector migration: batch failed (${describeError$2(error)}) — ${String(migrationRemaining(target))} vector(s) still cannot be used by the semantic leg; the next pass retries`);
|
|
8471
|
+
break;
|
|
8472
|
+
}
|
|
8473
|
+
migrated += step.migrated;
|
|
8474
|
+
dropped += step.dropped;
|
|
8475
|
+
reindexed += step.reindexed;
|
|
8476
|
+
last = {
|
|
8477
|
+
...step,
|
|
8478
|
+
enabled,
|
|
8479
|
+
migrated,
|
|
8480
|
+
dropped,
|
|
8481
|
+
reindexed
|
|
8482
|
+
};
|
|
8483
|
+
opts.onProgress?.(last);
|
|
8484
|
+
log.info(`${target.store} vector migration: re-encoded ${String(migrated)}/${String(initial.remaining)} (${String(step.remaining)} left)`);
|
|
8485
|
+
if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
|
|
8486
|
+
if (last.remaining === 0) break;
|
|
8487
|
+
await yieldToEventLoop();
|
|
8488
|
+
}
|
|
8489
|
+
if (last.remaining === 0) log.info(`${target.store} vector migration: complete — ${String(migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
|
|
8490
|
+
else if (!stopped()) log.warn(`${target.store} vector migration: ${String(last.remaining)} vector(s) still cannot be used by the semantic leg (the embedding model may be unavailable) — the next heartbeat retries; manual entry ${target.manualEntry}`);
|
|
8491
|
+
return last;
|
|
8492
|
+
}
|
|
8493
|
+
/**
|
|
8494
|
+
* Drive every store in order. A stop request keeps the drive from starting another store, but each
|
|
8495
|
+
* store still answers with its snapshot (so the caller can report what is left everywhere).
|
|
8496
|
+
*/
|
|
8497
|
+
async function driveVectorRepairs(targets, opts = {}) {
|
|
8498
|
+
const outcomes = [];
|
|
8499
|
+
for (const target of targets) outcomes.push(await driveVectorRepair(target, opts));
|
|
8500
|
+
return outcomes;
|
|
8501
|
+
}
|
|
8502
|
+
/**
|
|
8503
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`,
|
|
8504
|
+
* covering one or both stores.
|
|
8505
|
+
*
|
|
8506
|
+
* `store` selects a single store (omitted = both); `dry_run` previews every count without writing —
|
|
8507
|
+
* and without loading the model (so a preview also reports `would_warm`). The top-level
|
|
8508
|
+
* `semantic_available` is AND over the selected stores: `false` means at least one store's repair
|
|
8509
|
+
* could not encode anything.
|
|
8510
|
+
*/
|
|
8511
|
+
async function vectorsFix(targets, opts) {
|
|
8512
|
+
const stores = {};
|
|
8513
|
+
let semanticAvailable = true;
|
|
8514
|
+
for (const kind of ["memory", "knowledge"]) {
|
|
8515
|
+
if (opts.store !== void 0 && opts.store !== kind) continue;
|
|
8516
|
+
const target = targets[kind];
|
|
8517
|
+
if (target === void 0) continue;
|
|
8518
|
+
const { report } = await repairVectorSlice(target, { dryRun: opts.dryRun });
|
|
8519
|
+
stores[kind] = report;
|
|
8520
|
+
semanticAvailable = semanticAvailable && report.semantic_available;
|
|
8521
|
+
}
|
|
8522
|
+
return {
|
|
8523
|
+
stores,
|
|
8524
|
+
semantic_available: semanticAvailable,
|
|
8525
|
+
dry_run: opts.dryRun
|
|
8526
|
+
};
|
|
8527
|
+
}
|
|
8528
|
+
//#endregion
|
|
8529
|
+
//#region ../core/lib/store/secret_guard.js
|
|
8530
|
+
/**
|
|
8531
|
+
* Write-side secret / PII shape guard (P-03) — a PURE function, no store, no IO.
|
|
8532
|
+
*
|
|
8533
|
+
* The memory store is a long-lived, backed-up, sometimes-remote copy of what the model chose to
|
|
8534
|
+
* write down, so a plaintext credential that reaches it is hard to retract. This guard refuses a
|
|
8535
|
+
* fact whose TEXT matches a HIGH-CONFIDENCE credential/PII shape. It is deliberately narrow: only
|
|
8536
|
+
* shapes that a false positive is unlikely to produce, because a guard that refuses legitimate
|
|
7699
8537
|
* notes is worse than no guard — the model learns to stop recording.
|
|
7700
8538
|
*
|
|
7701
8539
|
* What it does NOT do, by design:
|
|
@@ -8460,6 +9298,124 @@ function inList(values) {
|
|
|
8460
9298
|
};
|
|
8461
9299
|
}
|
|
8462
9300
|
//#endregion
|
|
9301
|
+
//#region ../core/lib/db/dao/shared.js
|
|
9302
|
+
/**
|
|
9303
|
+
* The entity-candidate leg's ORDER contract — the tail every candidate statement ends with.
|
|
9304
|
+
*
|
|
9305
|
+
* The formula is the leg's own score (anchored Jaccard with a saturating union,
|
|
9306
|
+
* `store/entity_leg.ts`): `shared / (queryWidth + min(total - shared, widthCap))`. Ordering by it is
|
|
9307
|
+
* load-bearing, not cosmetic: `fuse` scales each leg by its maximum, and a survivor's scaled value
|
|
9308
|
+
* is unchanged only when the ordering — and therefore the maximum — survives the trim. A narrower
|
|
9309
|
+
* row sharing one entity can therefore outrank a wide one sharing two, and the cap keeps it.
|
|
9310
|
+
* `total ASC` is only a deterministic tie-break among equal ratios, and the id makes the order total.
|
|
9311
|
+
*
|
|
9312
|
+
* `idColumn` is the aggregate's own id (`fa.fact_id` / `ce.chunk_id`); that, the `total` subquery's
|
|
9313
|
+
* table and the filter predicates are the ONLY differences between the two statements.
|
|
9314
|
+
*/
|
|
9315
|
+
function entityCandidateTail(idColumn) {
|
|
9316
|
+
return `GROUP BY ${idColumn}
|
|
9317
|
+
ORDER BY (CAST(shared AS REAL) / (? + MIN(total - shared, ?))) DESC, total ASC, ${idColumn} ASC
|
|
9318
|
+
LIMIT ?`;
|
|
9319
|
+
}
|
|
9320
|
+
/**
|
|
9321
|
+
* Run an entity-candidate query: bind order, batching and union are shared; the SQL is the caller's.
|
|
9322
|
+
*
|
|
9323
|
+
* Bind order (identical in both aggregates): `<names …>, <filterParams …>, queryWidth, widthCap, limit`.
|
|
9324
|
+
*/
|
|
9325
|
+
function entityCandidates(db, q) {
|
|
9326
|
+
if (!q.names.length || q.limit <= 0) return [];
|
|
9327
|
+
const out = [];
|
|
9328
|
+
const seen = q.batchNames ? /* @__PURE__ */ new Set() : null;
|
|
9329
|
+
for (const batch of q.batchNames ? batches(q.names) : [q.names]) {
|
|
9330
|
+
const { placeholders, values } = inList(batch);
|
|
9331
|
+
const rows = db.prepare(q.sql(placeholders)).all(...values, ...q.filterParams, q.queryWidth, q.widthCap, q.limit);
|
|
9332
|
+
for (const row of rows) {
|
|
9333
|
+
if (seen !== null) {
|
|
9334
|
+
if (seen.has(row.id)) continue;
|
|
9335
|
+
seen.add(row.id);
|
|
9336
|
+
}
|
|
9337
|
+
out.push(row.id);
|
|
9338
|
+
}
|
|
9339
|
+
}
|
|
9340
|
+
return out;
|
|
9341
|
+
}
|
|
9342
|
+
/**
|
|
9343
|
+
* The batched `UPDATE … SET entities_version = ?` shared by `FactsDao` and `ChunksDao`.
|
|
9344
|
+
*
|
|
9345
|
+
* Only the aggregate's `table` / `idColumn` differ. Returns the summed `changes`; the knowledge
|
|
9346
|
+
* caller ignores it (its method's signature returns `void`), the memory caller's `number` is
|
|
9347
|
+
* unchanged.
|
|
9348
|
+
*/
|
|
9349
|
+
function setEntitiesVersionBatch(db, table, idColumn, ids, version) {
|
|
9350
|
+
let changes = 0;
|
|
9351
|
+
for (const batch of batches(ids)) {
|
|
9352
|
+
const { placeholders, values } = inList(batch);
|
|
9353
|
+
changes += db.prepare(`UPDATE ${table} SET entities_version = ? WHERE ${idColumn} IN (${placeholders})`).run(version, ...values).changes;
|
|
9354
|
+
}
|
|
9355
|
+
return changes;
|
|
9356
|
+
}
|
|
9357
|
+
/**
|
|
9358
|
+
* Document frequency — "how many rows carry each name".
|
|
9359
|
+
*
|
|
9360
|
+
* Shared by `EntitiesDao.activeDocFrequency` and `ChunksDao.docFrequency`; the only difference is
|
|
9361
|
+
* the population the caller's `where`/`from` describe (memory counts ACTIVE facts, knowledge has no
|
|
9362
|
+
* status column and counts chunks). Only the caller's QUERY names are looked up, so this is one
|
|
9363
|
+
* indexed COUNT per name rather than a corpus scan. A name absent from the map has frequency 0.
|
|
9364
|
+
*/
|
|
9365
|
+
function queryDocFrequency(db, names, q) {
|
|
9366
|
+
const out = /* @__PURE__ */ new Map();
|
|
9367
|
+
if (!names.length) return out;
|
|
9368
|
+
const { placeholders, values } = inList(names);
|
|
9369
|
+
const rows = db.prepare(`SELECT ${q.nameExpr} AS name, COUNT(*) AS df${q.from}
|
|
9370
|
+
WHERE ${q.nameExpr} IN (${placeholders})${q.where === void 0 ? "" : ` AND ${q.where}`}
|
|
9371
|
+
GROUP BY ${q.nameExpr}`).all(...values);
|
|
9372
|
+
for (const row of rows) out.set(row.name, Number(row.df));
|
|
9373
|
+
return out;
|
|
9374
|
+
}
|
|
9375
|
+
/**
|
|
9376
|
+
* The SHORT-QUERY LIKE fallback shared by `FactsDao.ftsSubstringSearch` and
|
|
9377
|
+
* `ChunksDao.ftsSubstringSearch`.
|
|
9378
|
+
*
|
|
9379
|
+
* The predicate text itself already comes from `db/tokenizer.ts#likeSubstring`; this owns the
|
|
9380
|
+
* protocol AROUND it, which both copies spelled out:
|
|
9381
|
+
*
|
|
9382
|
+
* 1. build the fragments — empty terms yield no patterns, which means "match nothing" ⇒ `[]`;
|
|
9383
|
+
* 2. `count` lands in the SELECT and `any` in the WHERE, so the SAME pattern list is bound TWICE:
|
|
9384
|
+
* once with the SELECT's placeholders, then — after the caller's filter params — with the
|
|
9385
|
+
* WHERE's;
|
|
9386
|
+
* 3. `LIMIT ?` last, defaulting to `-1` (unbounded; the store bounds the leg).
|
|
9387
|
+
*
|
|
9388
|
+
* `ORDER BY rank DESC, <id> ASC` stays in the caller's SQL: the id column is aggregate-specific and
|
|
9389
|
+
* `rank` must read the SELECT alias, so the fragments and the patterns are built once and used twice.
|
|
9390
|
+
*/
|
|
9391
|
+
function likeSubstringLeg(db, column, terms, buildSql, filterParams = [], limit) {
|
|
9392
|
+
const fragments = likeSubstring(column, terms);
|
|
9393
|
+
if (fragments.params.length === 0) return [];
|
|
9394
|
+
return db.prepare(buildSql(fragments)).all(...fragments.params, ...filterParams, ...fragments.params, limit ?? -1);
|
|
9395
|
+
}
|
|
9396
|
+
/**
|
|
9397
|
+
* Batched entity-name bags for a row-id set: `Map<rowId, names>`.
|
|
9398
|
+
*
|
|
9399
|
+
* Shared by `EntitiesDao.bagsForFacts` and `ChunksDao.entityBags` (§4.6.3): one `IN (…)` statement
|
|
9400
|
+
* per `batches()` slice, rows folded into the map in first-seen order. The SQL SHAPE is the only
|
|
9401
|
+
* aggregate-specific part (`table` / `keyColumn` / `nameColumn` / `joinEntities`); the crucial
|
|
9402
|
+
* `seedEmpty` difference above is the caller's to declare and stays observable.
|
|
9403
|
+
*/
|
|
9404
|
+
function entityBags(db, ids, q) {
|
|
9405
|
+
const out = /* @__PURE__ */ new Map();
|
|
9406
|
+
if (q.seedEmpty) for (const id of ids) out.set(id, []);
|
|
9407
|
+
for (const batch of batches(ids)) {
|
|
9408
|
+
const { placeholders, values } = inList(batch);
|
|
9409
|
+
const rows = db.prepare(`SELECT ${q.keyColumn} AS id, ${q.nameColumn} AS name FROM ${q.table}${q.joinEntities === void 0 ? "" : ` ${q.joinEntities}`} WHERE ${q.keyColumn} IN (${placeholders})`).all(...values);
|
|
9410
|
+
for (const row of rows) {
|
|
9411
|
+
const list = out.get(row.id);
|
|
9412
|
+
if (list) list.push(row.name);
|
|
9413
|
+
else out.set(row.id, [row.name]);
|
|
9414
|
+
}
|
|
9415
|
+
}
|
|
9416
|
+
return out;
|
|
9417
|
+
}
|
|
9418
|
+
//#endregion
|
|
8463
9419
|
//#region ../core/lib/db/dao/entities.js
|
|
8464
9420
|
var EntitiesDao = class {
|
|
8465
9421
|
db;
|
|
@@ -8506,17 +9462,13 @@ var EntitiesDao = class {
|
|
|
8506
9462
|
}
|
|
8507
9463
|
/** Batched entity bags for a candidate set: `Map<factId, names>` (ids with none get `[]`). */
|
|
8508
9464
|
bagsForFacts(ids) {
|
|
8509
|
-
|
|
8510
|
-
|
|
8511
|
-
|
|
8512
|
-
|
|
8513
|
-
|
|
8514
|
-
|
|
8515
|
-
|
|
8516
|
-
WHERE fe.fact_id IN (${placeholders})`).all(...values);
|
|
8517
|
-
for (const row of rows) out.get(row.fact_id)?.push(row.name);
|
|
8518
|
-
}
|
|
8519
|
-
return out;
|
|
9465
|
+
return entityBags(this.db, ids, {
|
|
9466
|
+
table: "fact_entities fe",
|
|
9467
|
+
keyColumn: "fe.fact_id",
|
|
9468
|
+
nameColumn: "e.name",
|
|
9469
|
+
joinEntities: "JOIN entities e ON e.entity_id = fe.entity_id",
|
|
9470
|
+
seedEmpty: true
|
|
9471
|
+
});
|
|
8520
9472
|
}
|
|
8521
9473
|
/** How many entities a fact is linked to — the scorer's `|A|`. */
|
|
8522
9474
|
countForFact(factId) {
|
|
@@ -8533,17 +9485,14 @@ var EntitiesDao = class {
|
|
|
8533
9485
|
* map has frequency 0, i.e. no active fact carries it and it can never be shared.
|
|
8534
9486
|
*/
|
|
8535
9487
|
activeDocFrequency(names) {
|
|
8536
|
-
|
|
8537
|
-
|
|
8538
|
-
const { placeholders, values } = inList(names);
|
|
8539
|
-
const rows = this.db.prepare(`SELECT e.name AS name, COUNT(*) AS df
|
|
9488
|
+
return queryDocFrequency(this.db, names, {
|
|
9489
|
+
from: `
|
|
8540
9490
|
FROM entities e
|
|
8541
9491
|
JOIN fact_entities fe ON fe.entity_id = e.entity_id
|
|
8542
|
-
JOIN facts fa ON fa.fact_id = fe.fact_id
|
|
8543
|
-
|
|
8544
|
-
|
|
8545
|
-
|
|
8546
|
-
return out;
|
|
9492
|
+
JOIN facts fa ON fa.fact_id = fe.fact_id`,
|
|
9493
|
+
nameExpr: "e.name",
|
|
9494
|
+
where: "fa.status = 'active'"
|
|
9495
|
+
});
|
|
8547
9496
|
}
|
|
8548
9497
|
/**
|
|
8549
9498
|
* Candidate facts for the embedding leg, narrowed by the scorer's NECESSARY conditions
|
|
@@ -8602,9 +9551,19 @@ var EntitiesDao = class {
|
|
|
8602
9551
|
* survivors, so one formula stays in charge.
|
|
8603
9552
|
*/
|
|
8604
9553
|
candidateFactsForAnyEntity(names, category, limit, queryWidth, widthCap, source) {
|
|
8605
|
-
|
|
8606
|
-
|
|
8607
|
-
|
|
9554
|
+
return entityCandidates(this.db, {
|
|
9555
|
+
names,
|
|
9556
|
+
filterParams: [
|
|
9557
|
+
category ?? null,
|
|
9558
|
+
category ?? null,
|
|
9559
|
+
source ?? null,
|
|
9560
|
+
source ?? null
|
|
9561
|
+
],
|
|
9562
|
+
limit,
|
|
9563
|
+
queryWidth,
|
|
9564
|
+
widthCap,
|
|
9565
|
+
batchNames: false,
|
|
9566
|
+
sql: (placeholders) => `SELECT fa.fact_id AS id, COUNT(*) AS shared,
|
|
8608
9567
|
(SELECT COUNT(*) FROM fact_entities x WHERE x.fact_id = fa.fact_id) AS total
|
|
8609
9568
|
FROM facts fa
|
|
8610
9569
|
JOIN fact_entities fe ON fe.fact_id = fa.fact_id
|
|
@@ -8613,9 +9572,8 @@ var EntitiesDao = class {
|
|
|
8613
9572
|
AND (? IS NULL OR EXISTS (
|
|
8614
9573
|
SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?
|
|
8615
9574
|
))
|
|
8616
|
-
|
|
8617
|
-
|
|
8618
|
-
LIMIT ?`).all(...values, category ?? null, category ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit).map((row) => row.id);
|
|
9575
|
+
${entityCandidateTail("fa.fact_id")}`
|
|
9576
|
+
});
|
|
8619
9577
|
}
|
|
8620
9578
|
};
|
|
8621
9579
|
//#endregion
|
|
@@ -8967,12 +9925,7 @@ var FactsDao = class FactsDao {
|
|
|
8967
9925
|
}
|
|
8968
9926
|
/** Stamp the rules that produced a batch of rows' entity/triple data. */
|
|
8969
9927
|
setEntitiesVersion(ids, version) {
|
|
8970
|
-
|
|
8971
|
-
for (const batch of batches(ids)) {
|
|
8972
|
-
const { placeholders, values } = inList(batch);
|
|
8973
|
-
changes += this.db.prepare(`UPDATE facts SET entities_version = ? WHERE fact_id IN (${placeholders})`).run(version, ...values).changes;
|
|
8974
|
-
}
|
|
8975
|
-
return changes;
|
|
9928
|
+
return setEntitiesVersionBatch(this.db, "facts", "fact_id", ids, version);
|
|
8976
9929
|
}
|
|
8977
9930
|
/**
|
|
8978
9931
|
* Replace a fact's HRR vector.
|
|
@@ -9352,14 +10305,16 @@ var FactsDao = class FactsDao {
|
|
|
9352
10305
|
* alias, so the LIKE expressions are built once and bound twice — same fragment, same order.
|
|
9353
10306
|
*/
|
|
9354
10307
|
ftsSubstringSearch(terms, category, limit, source) {
|
|
9355
|
-
const { any, count, params } = likeSubstring("fa.content", terms);
|
|
9356
|
-
if (params.length === 0) return [];
|
|
9357
10308
|
const sourceClause = source === void 0 ? "" : " AND EXISTS (SELECT 1 FROM fact_sources fs WHERE fs.fact_id = fa.fact_id AND fs.ref = ?)";
|
|
9358
|
-
return this.db.
|
|
10309
|
+
return likeSubstringLeg(this.db, "fa.content", terms, ({ any, count }) => `SELECT fa.fact_id AS id, ${count} AS rank
|
|
9359
10310
|
FROM facts fa
|
|
9360
10311
|
WHERE fa.status = 'active' AND (? IS NULL OR fa.category = ?)${sourceClause} AND (${any})
|
|
9361
10312
|
ORDER BY rank DESC, fa.fact_id ASC
|
|
9362
|
-
LIMIT
|
|
10313
|
+
LIMIT ?`, source === void 0 ? [category ?? null, category ?? null] : [
|
|
10314
|
+
category ?? null,
|
|
10315
|
+
category ?? null,
|
|
10316
|
+
source
|
|
10317
|
+
], limit);
|
|
9363
10318
|
}
|
|
9364
10319
|
/**
|
|
9365
10320
|
* ①'s predicate: the active-day counter crossed an integer boundary since the last settle.
|
|
@@ -10441,15 +11396,6 @@ const NO_CHECK = {
|
|
|
10441
11396
|
complete: false
|
|
10442
11397
|
};
|
|
10443
11398
|
/**
|
|
10444
|
-
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
10445
|
-
*
|
|
10446
|
-
* Bounds MEMORY, not only mission: the pass selects `content` for every row it takes, so an
|
|
10447
|
-
* "unbounded" pass (the CLI used to ask for `Number.MAX_SAFE_INTEGER`) builds an array of every
|
|
10448
|
-
* stale fact's text before rebuilding the first one. Callers that want the corpus drained loop
|
|
10449
|
-
* until `deferred === 0` instead.
|
|
10450
|
-
*/
|
|
10451
|
-
const ENTITY_SWEEP_BATCH = 2e3;
|
|
10452
|
-
/**
|
|
10453
11399
|
* `avantf_stats` key holding the retrieval-health snapshot.
|
|
10454
11400
|
*
|
|
10455
11401
|
* The counters are process-wide, but the MEMORY store owns the side table, so it is the single
|
|
@@ -10495,8 +11441,14 @@ var MemoryStore = class {
|
|
|
10495
11441
|
* and building MATCH expressions for the wrong one is silent zero recall (see `db/tokenizer.ts`).
|
|
10496
11442
|
*/
|
|
10497
11443
|
ftsTokenizer;
|
|
10498
|
-
/**
|
|
10499
|
-
|
|
11444
|
+
/**
|
|
11445
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
11446
|
+
*
|
|
11447
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It keeps
|
|
11448
|
+
* the rule where the loop is (the periodic callers can overlap), without a mutable field that two
|
|
11449
|
+
* stores would each have to remember to hold.
|
|
11450
|
+
*/
|
|
11451
|
+
entitySweepGuard = {};
|
|
10500
11452
|
/**
|
|
10501
11453
|
* Where the next conflict drain resumes (see `checkContradictions`). A scheduling cursor, not
|
|
10502
11454
|
* derived state: nothing is lost when it resets, and the queue itself is in the database.
|
|
@@ -10536,7 +11488,7 @@ var MemoryStore = class {
|
|
|
10536
11488
|
clock,
|
|
10537
11489
|
budget: opts?.budget ?? trust.tick_max_facts
|
|
10538
11490
|
});
|
|
10539
|
-
this.
|
|
11491
|
+
evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
|
|
10540
11492
|
this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
|
|
10541
11493
|
this.db.pragma("optimize");
|
|
10542
11494
|
return result;
|
|
@@ -10550,22 +11502,32 @@ var MemoryStore = class {
|
|
|
10550
11502
|
return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
|
|
10551
11503
|
}
|
|
10552
11504
|
/**
|
|
10553
|
-
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs
|
|
10554
|
-
*
|
|
10555
|
-
* (the DB is the authoritative vector store; the vstore is a derived read model).
|
|
11505
|
+
* Rebuild the in-memory vstore from persisted `facts.semantic_vector` BLOBs, and say loudly what
|
|
11506
|
+
* belongs to another vector space.
|
|
10556
11507
|
*
|
|
10557
|
-
*
|
|
10558
|
-
*
|
|
10559
|
-
*
|
|
11508
|
+
* Without this the semantic path is silently empty after every process restart (the DB is the
|
|
11509
|
+
* authoritative vector store; the vstore is a derived read model). Only ACTIVE facts are indexed:
|
|
11510
|
+
* archived facts are filtered out of every retrieval path anyway, so loading them would only grow
|
|
11511
|
+
* the index forever — `archive()` evicts and `restore()` re-adds, keeping the invariant.
|
|
11512
|
+
*
|
|
11513
|
+
* The reload, the `{ stale, space_stale, legacy }` count and the warning are the SHARED open-time
|
|
11514
|
+
* step (`store/vector_repair.ts`); this store's half is the row read. A dim-valid vector from
|
|
11515
|
+
* another space is still RANKED (the reload cannot tell the difference), so a silent model swap
|
|
11516
|
+
* would mix two spaces in one index — acting on it is the manual fix's job for the operator path
|
|
11517
|
+
* and the bounded background migration's for the automatic one.
|
|
10560
11518
|
*/
|
|
10561
11519
|
reloadIndex() {
|
|
10562
|
-
|
|
10563
|
-
|
|
10564
|
-
|
|
10565
|
-
|
|
10566
|
-
|
|
10567
|
-
|
|
10568
|
-
|
|
11520
|
+
loadVectorIndex({
|
|
11521
|
+
store: "memory",
|
|
11522
|
+
vstore: this.vstore,
|
|
11523
|
+
rows: this.facts.activeVectorRows().map((r) => ({
|
|
11524
|
+
id: r.fact_id,
|
|
11525
|
+
vec: r.semantic_vector,
|
|
11526
|
+
embedding_model: r.embedding_model
|
|
11527
|
+
})),
|
|
11528
|
+
space: this.vectorSpace(),
|
|
11529
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)"
|
|
11530
|
+
});
|
|
10569
11531
|
}
|
|
10570
11532
|
/**
|
|
10571
11533
|
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of ACTIVE
|
|
@@ -10666,21 +11628,45 @@ var MemoryStore = class {
|
|
|
10666
11628
|
};
|
|
10667
11629
|
}
|
|
10668
11630
|
}
|
|
11631
|
+
/**
|
|
11632
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
11633
|
+
* call, the live index and the single-row write. The write records the VECTOR SPACE, not just the
|
|
11634
|
+
* backend name — a same-width model swap is otherwise indistinguishable from "already encoded"
|
|
11635
|
+
* (DESIGN §20) — and `setSemanticVector` re-queues the fact for the conflict check.
|
|
11636
|
+
*/
|
|
11637
|
+
vectorWriteTarget(space) {
|
|
11638
|
+
return {
|
|
11639
|
+
store: "memory",
|
|
11640
|
+
space,
|
|
11641
|
+
encode: (text) => this.semantic.encode(text),
|
|
11642
|
+
add: (id, vec) => {
|
|
11643
|
+
this.vstore.add(id, vec);
|
|
11644
|
+
},
|
|
11645
|
+
write: (rows) => {
|
|
11646
|
+
for (const r of rows) this.facts.setSemanticVector(r.id, r.bytes, space, this.vstore.name);
|
|
11647
|
+
},
|
|
11648
|
+
onRowError: (row, error) => {
|
|
11649
|
+
this.warnIndexFailure(row.id, error);
|
|
11650
|
+
},
|
|
11651
|
+
onWriteError: (rows, error) => {
|
|
11652
|
+
for (const r of rows) this.warnIndexFailure(r.id, error);
|
|
11653
|
+
}
|
|
11654
|
+
};
|
|
11655
|
+
}
|
|
11656
|
+
warnIndexFailure(factId, error) {
|
|
11657
|
+
retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
|
|
11658
|
+
}
|
|
10669
11659
|
/** Best-effort semantic indexing: encode the fact and add it to the vstore. */
|
|
10670
11660
|
async maybeIndexSemantic(factId, content) {
|
|
10671
|
-
|
|
10672
|
-
|
|
10673
|
-
this.semantic.ensureWarm?.();
|
|
10674
|
-
return false;
|
|
10675
|
-
}
|
|
10676
|
-
const vec = await this.semantic.encode(content);
|
|
10677
|
-
this.vstore.add(factId, vec);
|
|
10678
|
-
this.facts.setSemanticVector(factId, float32ToBytes(vec), this.vectorSpace(), this.vstore.name);
|
|
10679
|
-
return true;
|
|
10680
|
-
} catch (error) {
|
|
10681
|
-
retrievalLogger().warn(`memory index: fact ${String(factId)} was not semantically indexed (${describeError$2(error)}) — run mem_admin vectors_diagnose to see the shortfall, vectors_fix to re-encode`);
|
|
11661
|
+
if (!this.semantic.isAvailable()) {
|
|
11662
|
+
this.semantic.ensureWarm?.();
|
|
10682
11663
|
return false;
|
|
10683
11664
|
}
|
|
11665
|
+
const { encoded } = await encodeAndPersist([{
|
|
11666
|
+
id: factId,
|
|
11667
|
+
text: content
|
|
11668
|
+
}], this.vectorWriteTarget(this.vectorSpace()));
|
|
11669
|
+
return encoded === 1;
|
|
10684
11670
|
}
|
|
10685
11671
|
async update(fact_id, content, category, ttlDays, opts) {
|
|
10686
11672
|
if (!this.getRow(fact_id)) throw new Error(`找不到 fact_id=${fact_id}`);
|
|
@@ -10775,7 +11761,7 @@ var MemoryStore = class {
|
|
|
10775
11761
|
this.facts.reinforceOutcome(fact_id, outcome.next, clock, outcome.bonusCount, outcome.windowAt);
|
|
10776
11762
|
return null;
|
|
10777
11763
|
}).immediate();
|
|
10778
|
-
if (evict !== null) this.
|
|
11764
|
+
if (evict !== null) evictVectors(this.vstore, [evict]);
|
|
10779
11765
|
return this.getRow(fact_id)?.helpful_count ?? 0;
|
|
10780
11766
|
}
|
|
10781
11767
|
/** Permanent-memory administration (spec §6). */
|
|
@@ -11070,168 +12056,103 @@ var MemoryStore = class {
|
|
|
11070
12056
|
};
|
|
11071
12057
|
}
|
|
11072
12058
|
/**
|
|
11073
|
-
*
|
|
11074
|
-
*
|
|
11075
|
-
*
|
|
11076
|
-
*
|
|
11077
|
-
* re-encode can replace them (never `add()` a stale vector — the store
|
|
11078
|
-
* throws on dim mismatch),
|
|
11079
|
-
* (c) encode active facts that have no vector yet (needs the semantic backend),
|
|
11080
|
-
* (d) re-encode vectors written in ANOTHER vector space (a model swap, or the one-time
|
|
11081
|
-
* adoption of a store predating the space id).
|
|
11082
|
-
*
|
|
11083
|
-
* `limit` is what makes the SAME logic usable as a background migration: at most that many rows are
|
|
11084
|
-
* dropped and at most that many are re-encoded, so one call is a bounded unit of work the caller can
|
|
11085
|
-
* yield between. `vectorsFix` passes no limit (the explicit operator action drains everything);
|
|
11086
|
-
* `migrateVectorsBatch` passes the migration's batch size.
|
|
12059
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
12060
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
12061
|
+
* `semantic.auto_migrate`, the dry-run split and the drop-before-encode order — lives there once,
|
|
12062
|
+
* for both stores.
|
|
11087
12063
|
*/
|
|
11088
|
-
|
|
11089
|
-
|
|
11090
|
-
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
|
|
11095
|
-
|
|
11096
|
-
|
|
11097
|
-
|
|
11098
|
-
|
|
11099
|
-
|
|
11100
|
-
|
|
11101
|
-
|
|
11102
|
-
reindexed: 0,
|
|
11103
|
-
dropped: 0,
|
|
11104
|
-
fixed: 0,
|
|
11105
|
-
semantic_available: semAvailable,
|
|
11106
|
-
would_warm: wouldWarm,
|
|
11107
|
-
dry_run: true
|
|
11108
|
-
},
|
|
11109
|
-
remaining: remainingOf(0)
|
|
12064
|
+
vectorRepairTarget() {
|
|
12065
|
+
return {
|
|
12066
|
+
store: "memory",
|
|
12067
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`)",
|
|
12068
|
+
vectorSpace: () => this.vectorSpace(),
|
|
12069
|
+
semanticAvailable: () => this.semantic.isAvailable(),
|
|
12070
|
+
autoMigrate: () => this.config.semantic.auto_migrate,
|
|
12071
|
+
vectorSpaceHealth: () => this.vectorSpaceHealth(),
|
|
12072
|
+
missingVectorCount: () => this.facts.vectorCounts().missing,
|
|
12073
|
+
classifyVectors: (limit) => this.classifyRepairVectors(limit),
|
|
12074
|
+
reindexUsable: (rows) => this.reindexUsableVectors(rows),
|
|
12075
|
+
dropVectors: (ids) => this.dropRepairVectors(ids),
|
|
12076
|
+
reencodeVectors: (limit) => this.reencodeMissingVectors(limit),
|
|
12077
|
+
warmup: () => this.warmupSemantic()
|
|
11110
12078
|
};
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
const toDrop = [...c.staleIds, ...c.spaceStaleIds].slice(0, limit);
|
|
11119
|
-
if (toDrop.length) {
|
|
11120
|
-
const spaceStale = new Set(c.spaceStaleIds);
|
|
11121
|
-
dropped = this.facts.clearVectors(toDrop);
|
|
11122
|
-
this.evictVectors(toDrop.filter((id) => spaceStale.has(id)));
|
|
11123
|
-
}
|
|
11124
|
-
}
|
|
11125
|
-
let fixed = 0;
|
|
11126
|
-
if (semAvailable) {
|
|
11127
|
-
const missingRows = this.facts.missingVectorRows().slice(0, limit);
|
|
11128
|
-
for (const m of missingRows) {
|
|
11129
|
-
if (!await this.maybeIndexSemantic(m.fact_id, m.content)) continue;
|
|
11130
|
-
fixed++;
|
|
11131
|
-
}
|
|
11132
|
-
}
|
|
12079
|
+
}
|
|
12080
|
+
/**
|
|
12081
|
+
* {@link classifyVectors} reduced to what the repair flow steps on: the counts, the ids to drop
|
|
12082
|
+
* (bounded by `limit`) and the decoded vectors that can go back into a lost live index.
|
|
12083
|
+
*/
|
|
12084
|
+
classifyRepairVectors(limit) {
|
|
12085
|
+
const c = this.classifyVectors();
|
|
11133
12086
|
return {
|
|
11134
|
-
|
|
11135
|
-
|
|
11136
|
-
|
|
11137
|
-
|
|
11138
|
-
|
|
11139
|
-
|
|
11140
|
-
|
|
11141
|
-
|
|
11142
|
-
|
|
11143
|
-
|
|
11144
|
-
|
|
11145
|
-
|
|
11146
|
-
|
|
12087
|
+
stale: c.staleIds.length,
|
|
12088
|
+
space_stale: c.spaceStaleIds.length,
|
|
12089
|
+
missing: c.missing,
|
|
12090
|
+
dropIds: [...c.staleIds, ...c.spaceStaleIds].slice(0, limit),
|
|
12091
|
+
unindexed: c.unindexed
|
|
12092
|
+
};
|
|
12093
|
+
}
|
|
12094
|
+
/** Put usable persisted vectors missing from the live index back into it (no model needed). */
|
|
12095
|
+
reindexUsableVectors(rows) {
|
|
12096
|
+
for (const u of rows) this.vstore.add(u.id, u.vec);
|
|
12097
|
+
return rows.length;
|
|
12098
|
+
}
|
|
12099
|
+
/**
|
|
12100
|
+
* Unusable bytes (dim mismatch) and foreign-space vectors both have to go before a re-encode can
|
|
12101
|
+
* pick them up, and for the same reason: `add()` would keep ranking them otherwise. Only the slice
|
|
12102
|
+
* the flow is about to re-encode is passed in, so a process that dies mid-migration leaves at most
|
|
12103
|
+
* one batch vector-less — those rows are exactly the `missing` rows the next run resumes on.
|
|
12104
|
+
*
|
|
12105
|
+
* `clearVectors` re-queues each fact for the conflict check (`conflict_checked = 0`): the vector
|
|
12106
|
+
* the embedding leg was checked against is gone, so the check has to run again.
|
|
12107
|
+
*/
|
|
12108
|
+
dropRepairVectors(ids) {
|
|
12109
|
+
if (!ids.length) return 0;
|
|
12110
|
+
const dropped = this.facts.clearVectors(ids);
|
|
12111
|
+
evictVectors(this.vstore, ids);
|
|
12112
|
+
return dropped;
|
|
12113
|
+
}
|
|
12114
|
+
/**
|
|
12115
|
+
* Encode the ACTIVE facts that still have no vector in this space (needs the semantic backend).
|
|
12116
|
+
*
|
|
12117
|
+
* Every row lands through `setSemanticVector`, which re-queues the fact for the conflict check —
|
|
12118
|
+
* the same rule as {@link dropRepairVectors}: a fact whose vector the leg could not see when it
|
|
12119
|
+
* ran has not been checked.
|
|
12120
|
+
*/
|
|
12121
|
+
async reencodeMissingVectors(limit) {
|
|
12122
|
+
const rows = this.facts.missingVectorRows().slice(0, limit).map((m) => ({
|
|
12123
|
+
id: m.fact_id,
|
|
12124
|
+
text: m.content
|
|
12125
|
+
}));
|
|
12126
|
+
if (!rows.length) return {
|
|
12127
|
+
encoded: 0,
|
|
12128
|
+
failed: 0
|
|
11147
12129
|
};
|
|
12130
|
+
return await encodeAndPersist(rows, this.vectorWriteTarget(this.vectorSpace()));
|
|
11148
12131
|
}
|
|
11149
12132
|
/**
|
|
11150
12133
|
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
11151
12134
|
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
11152
|
-
* the preview also reports `would_warm` (see the contract's `
|
|
12135
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
11153
12136
|
*/
|
|
11154
12137
|
async vectorsFix(dryRun = false) {
|
|
11155
|
-
return (await this.
|
|
12138
|
+
return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
|
|
11156
12139
|
}
|
|
11157
12140
|
/**
|
|
11158
12141
|
* ONE bounded slice of the vector-space migration: at most `batchSize` rows are dropped and
|
|
11159
12142
|
* re-encoded, so the caller keeps control of the event loop between slices. This is the unit the
|
|
11160
12143
|
* background migration ({@link migrateVectors}) and its tests drive.
|
|
11161
|
-
*
|
|
11162
|
-
* Resumability is a property of the DATABASE, not of this process: every slice re-derives what is
|
|
11163
|
-
* stale from the persisted width/space, so a restart simply continues where the last slice stopped.
|
|
11164
12144
|
*/
|
|
11165
12145
|
async migrateVectorsBatch(batchSize = 16) {
|
|
11166
|
-
|
|
11167
|
-
const { report, remaining } = await this.repairVectors({
|
|
11168
|
-
dryRun: false,
|
|
11169
|
-
limit: size
|
|
11170
|
-
});
|
|
11171
|
-
return {
|
|
11172
|
-
remaining,
|
|
11173
|
-
migrated: report.fixed,
|
|
11174
|
-
dropped: report.dropped,
|
|
11175
|
-
reindexed: report.reindexed,
|
|
11176
|
-
semantic_available: report.semantic_available
|
|
11177
|
-
};
|
|
12146
|
+
return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
|
|
11178
12147
|
}
|
|
11179
12148
|
/**
|
|
11180
12149
|
* Drive the bounded background migration until the store is current, the model is unavailable, or
|
|
11181
|
-
* `shouldStop()` turns true (plugin unmount / process shutdown).
|
|
11182
|
-
*
|
|
11183
|
-
*
|
|
11184
|
-
* blocks the event loop for longer than one batch.
|
|
11185
|
-
*
|
|
11186
|
-
* Honours `semantic.auto_migrate` (default on) — the switch exists so a host that must not spend CPU
|
|
11187
|
-
* on background re-encoding can leave the loud warning and the manual entry as the only path.
|
|
12150
|
+
* `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the `semantic.auto_migrate`
|
|
12151
|
+
* switch, the no-progress rule and the store-named logging all live in the shared flow
|
|
12152
|
+
* (`store/vector_repair.ts`); this is the one-store entry callers (and its tests) already use.
|
|
11188
12153
|
*/
|
|
11189
12154
|
async migrateVectors(opts = {}) {
|
|
11190
|
-
|
|
11191
|
-
const snapshot = () => ({
|
|
11192
|
-
enabled,
|
|
11193
|
-
remaining: this.migrationRemaining(),
|
|
11194
|
-
migrated: 0,
|
|
11195
|
-
dropped: 0,
|
|
11196
|
-
reindexed: 0,
|
|
11197
|
-
semantic_available: this.semantic.isAvailable()
|
|
11198
|
-
});
|
|
11199
|
-
if (!enabled) return snapshot();
|
|
11200
|
-
let migrated = 0;
|
|
11201
|
-
let dropped = 0;
|
|
11202
|
-
let reindexed = 0;
|
|
11203
|
-
let last = snapshot();
|
|
11204
|
-
if (last.remaining === 0) return last;
|
|
11205
|
-
for (;;) {
|
|
11206
|
-
if (opts.shouldStop?.() === true) break;
|
|
11207
|
-
let step;
|
|
11208
|
-
try {
|
|
11209
|
-
step = await this.migrateVectorsBatch(opts.batchSize);
|
|
11210
|
-
} catch (error) {
|
|
11211
|
-
retrievalLogger().warn(`memory vector migration: batch failed (${describeError$2(error)}) — ${String(this.migrationRemaining())} vector(s) still belong to an older embedding space; the next pass retries`);
|
|
11212
|
-
break;
|
|
11213
|
-
}
|
|
11214
|
-
migrated += step.migrated;
|
|
11215
|
-
dropped += step.dropped;
|
|
11216
|
-
reindexed += step.reindexed;
|
|
11217
|
-
last = {
|
|
11218
|
-
...step,
|
|
11219
|
-
enabled,
|
|
11220
|
-
migrated,
|
|
11221
|
-
dropped,
|
|
11222
|
-
reindexed
|
|
11223
|
-
};
|
|
11224
|
-
opts.onProgress?.(last);
|
|
11225
|
-
if (step.migrated === 0 && step.dropped === 0 && step.reindexed === 0) break;
|
|
11226
|
-
if (last.remaining === 0) break;
|
|
11227
|
-
await yieldToEventLoop();
|
|
11228
|
-
}
|
|
11229
|
-
return last;
|
|
11230
|
-
}
|
|
11231
|
-
/** Active rows the semantic leg still cannot use (old width, old space, or no vector at all). */
|
|
11232
|
-
migrationRemaining() {
|
|
11233
|
-
const health = this.vectorSpaceHealth();
|
|
11234
|
-
return health.stale + health.space_stale + this.facts.vectorCounts().missing;
|
|
12155
|
+
return await driveVectorRepair(this.vectorRepairTarget(), opts);
|
|
11235
12156
|
}
|
|
11236
12157
|
/**
|
|
11237
12158
|
* One lifecycle pass, owned by the store so the DB writes and the live-index
|
|
@@ -11246,7 +12167,7 @@ var MemoryStore = class {
|
|
|
11246
12167
|
*/
|
|
11247
12168
|
async maintenance() {
|
|
11248
12169
|
const result = runMaintenance(this.db, this.config, { budget: 0 });
|
|
11249
|
-
this.
|
|
12170
|
+
evictVectors(this.vstore, [...result.archived_ids, ...result.purged_ids]);
|
|
11250
12171
|
this.retireConflicts([...result.archived_ids, ...result.purged_ids]);
|
|
11251
12172
|
this.vstore.compact?.();
|
|
11252
12173
|
this.db.pragma("optimize");
|
|
@@ -11279,25 +12200,19 @@ var MemoryStore = class {
|
|
|
11279
12200
|
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
11280
12201
|
* every selected row's `content` before rebuilding the first one.
|
|
11281
12202
|
*
|
|
11282
|
-
* The in-flight guard
|
|
11283
|
-
*
|
|
11284
|
-
*
|
|
11285
|
-
*
|
|
11286
|
-
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
11287
|
-
*
|
|
11288
|
-
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
11289
|
-
* caller asking for "-1 rows" would silently get the whole corpus — the opposite of bounded.
|
|
12203
|
+
* The loop, the budget clamp, the in-flight guard and the `{rebuilt, deferred, skipped}` report all
|
|
12204
|
+
* live in `store/entity_sweep.ts` now — this method is memory's adapter for it, so knowledge's
|
|
12205
|
+
* entity leg and this one cannot drift apart again. What stays HERE is the domain half: the stale
|
|
12206
|
+
* selection (`facts.staleEntityRows`, ACTIVE facts only) and the per-fact write below.
|
|
11290
12207
|
*/
|
|
11291
12208
|
async reindexEntities(budget = ENTITY_SWEEP_BATCH) {
|
|
11292
|
-
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11296
|
-
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
const rows = this.facts.staleEntityRows(1, Math.max(0, Math.floor(budget)));
|
|
11300
|
-
for (const row of rows) {
|
|
12209
|
+
return await sweepVersionedEntities({
|
|
12210
|
+
staleEntityBatch: (limit) => ({
|
|
12211
|
+
total: this.facts.countStaleEntities(1),
|
|
12212
|
+
rows: this.facts.staleEntityRows(1, limit)
|
|
12213
|
+
}),
|
|
12214
|
+
staleEntityCount: () => this.facts.countStaleEntities(1),
|
|
12215
|
+
extractAndWriteEntities: async (row) => {
|
|
11301
12216
|
const tokens = await tagText(row.content);
|
|
11302
12217
|
const entityWrites = normalizeEntityWrites(entitiesFromTokens(tokens, row.content));
|
|
11303
12218
|
const entities = entityWrites.map((e) => e.name);
|
|
@@ -11312,15 +12227,11 @@ var MemoryStore = class {
|
|
|
11312
12227
|
this.facts.setEntitiesVersion([row.fact_id], 1);
|
|
11313
12228
|
this.facts.requeueConflictCheck([row.fact_id]);
|
|
11314
12229
|
})();
|
|
12230
|
+
},
|
|
12231
|
+
onRowError: (row, error) => {
|
|
12232
|
+
retrievalLogger().warn(`memory entity sweep: fact ${String(row.fact_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next pass retries it`);
|
|
11315
12233
|
}
|
|
11316
|
-
|
|
11317
|
-
rebuilt: rows.length,
|
|
11318
|
-
deferred: this.facts.countStaleEntities(1),
|
|
11319
|
-
skipped: false
|
|
11320
|
-
};
|
|
11321
|
-
} finally {
|
|
11322
|
-
this.entitySweepInFlight = false;
|
|
11323
|
-
}
|
|
12234
|
+
}, this.entitySweepGuard, { budget });
|
|
11324
12235
|
}
|
|
11325
12236
|
/**
|
|
11326
12237
|
* A fact left the ACTIVE corpus: retire the open conflicts that name it, so
|
|
@@ -11331,10 +12242,6 @@ var MemoryStore = class {
|
|
|
11331
12242
|
retireConflicts(factIds) {
|
|
11332
12243
|
this.contradictions.resolveForFacts(factIds);
|
|
11333
12244
|
}
|
|
11334
|
-
/** Drop vectors of archived/purged facts from the live index (batched). */
|
|
11335
|
-
evictVectors(factIds) {
|
|
11336
|
-
evictVectors(this.vstore, factIds);
|
|
11337
|
-
}
|
|
11338
12245
|
/**
|
|
11339
12246
|
* Entities that co-occur with `entity` across ACTIVE facts, ranked by occurrence
|
|
11340
12247
|
* count. Archived (incl. superseded) facts are excluded — their entities are not
|
|
@@ -11382,17 +12289,7 @@ var MemoryStore = class {
|
|
|
11382
12289
|
}
|
|
11383
12290
|
/** Hybrid search over the memory corpus (semantic + FTS + entity Jaccard [+ HRR probe] via fusion). */
|
|
11384
12291
|
async search(input) {
|
|
11385
|
-
const result = await
|
|
11386
|
-
query: input.query,
|
|
11387
|
-
limit: input.limit,
|
|
11388
|
-
maxTokens: input.maxTokens,
|
|
11389
|
-
queryVector: input.queryVector,
|
|
11390
|
-
recordStats: input.recordStats,
|
|
11391
|
-
floors: input.floors,
|
|
11392
|
-
...input.includeScores === void 0 ? {} : { includeScores: input.includeScores },
|
|
11393
|
-
...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs },
|
|
11394
|
-
...input.rewriteQuery === void 0 ? {} : { rewriteQuery: input.rewriteQuery }
|
|
11395
|
-
});
|
|
12292
|
+
const result = await runHybridSearch(this.hybridDeps(input), input.query, input);
|
|
11396
12293
|
return {
|
|
11397
12294
|
hits: result.hits,
|
|
11398
12295
|
degraded: result.degraded,
|
|
@@ -11407,10 +12304,7 @@ var MemoryStore = class {
|
|
|
11407
12304
|
* fact→hit mapping, and what to do with the hits the caller actually receives.
|
|
11408
12305
|
*/
|
|
11409
12306
|
hybridDeps(input) {
|
|
11410
|
-
return {
|
|
11411
|
-
kind: "memory",
|
|
11412
|
-
config: this.config,
|
|
11413
|
-
semantic: this.semantic,
|
|
12307
|
+
return makeHybridDeps("memory", this.config, this.semantic, {
|
|
11414
12308
|
legs: (ctx) => this.searchLegs(input, ctx),
|
|
11415
12309
|
texts: (ids) => this.loadTexts(ids),
|
|
11416
12310
|
hits: (ranked, texts) => {
|
|
@@ -11420,7 +12314,7 @@ var MemoryStore = class {
|
|
|
11420
12314
|
onReturn: (kept) => {
|
|
11421
12315
|
if (input.track !== false) this.reinforce(kept.map((h) => h.ref_id));
|
|
11422
12316
|
}
|
|
11423
|
-
};
|
|
12317
|
+
});
|
|
11424
12318
|
}
|
|
11425
12319
|
/**
|
|
11426
12320
|
* The memory legs: semantic, FTS, entity Jaccard, and — for `recall.probe` — the HRR probe.
|
|
@@ -11442,49 +12336,18 @@ var MemoryStore = class {
|
|
|
11442
12336
|
const qEntities = Array.from(new Set((await extractEntities(ctx.query)).map((e) => e.name)));
|
|
11443
12337
|
const anchors = qEntities.length > 0 ? selectAnchors(qEntities, this.entities.activeDocFrequency(qEntities), this.facts.countActive()) : [];
|
|
11444
12338
|
const candidates = anchors.length > 0 ? this.entities.candidateFactsForAnyEntity(anchors, input.category, ctx.legCap, qEntities.length, 3, input.source) : [];
|
|
11445
|
-
|
|
11446
|
-
* Wrap one leg's raw scores. `capped` must be measured on the RAW set (before the relevance
|
|
11447
|
-
* floor): the floor removes the tail anyway, and deriving the flag from the floored size would
|
|
11448
|
-
* erase the "this leg was cut at legCap" signal exactly when it bound.
|
|
11449
|
-
*/
|
|
11450
|
-
const leg = (scores, weight, raw) => ({
|
|
11451
|
-
weight,
|
|
11452
|
-
scores,
|
|
11453
|
-
capped: (raw ?? scores).size === ctx.legCap
|
|
11454
|
-
});
|
|
12339
|
+
const runner = makeLegRunner(ctx);
|
|
11455
12340
|
const ftsRaw = this.ftsPath(ctx.query, input.category, ctx.legCap, input.source);
|
|
11456
|
-
const
|
|
12341
|
+
const ftsLeg = runner.term("fts", ftsRaw, this.loadTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts);
|
|
11457
12342
|
const jaccardRaw = this.jaccardPath(anchors, qEntities.length, candidates);
|
|
11458
|
-
const
|
|
11459
|
-
const jaccardLeg = {
|
|
11460
|
-
...leg(jaccardFloored.scores, ctx.weights.jaccard, jaccardRaw),
|
|
11461
|
-
leg: "jaccard",
|
|
11462
|
-
droppedByFloor: jaccardFloored.dropped
|
|
11463
|
-
};
|
|
12343
|
+
const jaccardLeg = runner.scored("jaccard", jaccardRaw, ctx.weights.jaccard, ctx.floors.jaccard);
|
|
11464
12344
|
const legs = [
|
|
11465
|
-
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) =>
|
|
11466
|
-
const floored = applyScoreFloor(raw, ctx.floors.semantic);
|
|
11467
|
-
return {
|
|
11468
|
-
weight: ctx.weights.semantic,
|
|
11469
|
-
scores: floored.scores,
|
|
11470
|
-
leg: "semantic",
|
|
11471
|
-
droppedByFloor: floored.dropped
|
|
11472
|
-
};
|
|
11473
|
-
}) : {
|
|
11474
|
-
weight: ctx.weights.semantic,
|
|
11475
|
-
scores: /* @__PURE__ */ new Map(),
|
|
11476
|
-
leg: "semantic",
|
|
11477
|
-
droppedByFloor: 0
|
|
11478
|
-
},
|
|
12345
|
+
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector, input.source).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
|
|
11479
12346
|
Promise.resolve(jaccardLeg),
|
|
11480
|
-
|
|
11481
|
-
...leg(ftsFloored.scores, ctx.weights.fts, ftsRaw),
|
|
11482
|
-
leg: "fts",
|
|
11483
|
-
droppedByFloor: ftsFloored.dropped
|
|
11484
|
-
}
|
|
12347
|
+
ftsLeg
|
|
11485
12348
|
];
|
|
11486
|
-
if (input.includeHrr) legs.push(Promise.resolve(
|
|
11487
|
-
if (this.config.retriever.time_window) legs.push(Promise.resolve(
|
|
12349
|
+
if (input.includeHrr) legs.push(Promise.resolve(runner.plain(this.hrrPath(ctx.query, qEntities, [...jaccardLeg.scores.keys()], input.category, ctx.legCap, candidates.length === 0, input.source), ctx.weights.jaccard)));
|
|
12350
|
+
if (this.config.retriever.time_window) legs.push(Promise.resolve(runner.plain(this.timeWindowPath(ctx.query, input.category, ctx.legCap, input.source), ctx.weights.jaccard)));
|
|
11488
12351
|
return legs;
|
|
11489
12352
|
}
|
|
11490
12353
|
/**
|
|
@@ -11609,18 +12472,22 @@ var MemoryStore = class {
|
|
|
11609
12472
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
11610
12473
|
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
11611
12474
|
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
11612
|
-
* instead of re-encoding (performance review §7.7 / P8).
|
|
12475
|
+
* instead of re-encoding (performance review §7.7 / P8). The flow is shared
|
|
12476
|
+
* (`store/legs.ts#semanticLeg`); the only memory-specific piece is the filter — the vstore has no
|
|
12477
|
+
* notion of status/category/source, so candidates go through the DB and archived/purged facts,
|
|
12478
|
+
* other categories and other sources never leak into the leg.
|
|
11613
12479
|
*/
|
|
11614
12480
|
async semanticPath(query, category, k, queryVector, onVector, source) {
|
|
11615
|
-
|
|
11616
|
-
|
|
11617
|
-
|
|
11618
|
-
|
|
11619
|
-
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
12481
|
+
return semanticLeg({
|
|
12482
|
+
query,
|
|
12483
|
+
k,
|
|
12484
|
+
queryVector,
|
|
12485
|
+
onVector,
|
|
12486
|
+
encode: (q) => this.semantic.encode(q),
|
|
12487
|
+
dim: this.vstore.dim,
|
|
12488
|
+
topk: (vec, n) => this.vstore.topk(vec, n),
|
|
12489
|
+
filterTopk: (ids) => new Set(this.facts.activeIdsIn(ids, category, source))
|
|
12490
|
+
});
|
|
11624
12491
|
}
|
|
11625
12492
|
/**
|
|
11626
12493
|
* HRR entity-probe path: phase similarity between the query's entity bundle and
|
|
@@ -11689,21 +12556,20 @@ var MemoryStore = class {
|
|
|
11689
12556
|
* semantic, why it must be synchronous, and how the bar itself was measured.
|
|
11690
12557
|
*/
|
|
11691
12558
|
lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
|
|
11692
|
-
return
|
|
11693
|
-
const fts = buildFtsQuery(term, this.ftsTokenizer);
|
|
11694
|
-
return fts !== null && this.facts.ftsSearch(fts, void 0, 1).length > 0;
|
|
11695
|
-
}, stopAt);
|
|
12559
|
+
return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.facts.ftsSearch(fts, void 0, 1).length > 0, stopAt);
|
|
11696
12560
|
}
|
|
11697
12561
|
ftsPath(query, category, cap, source) {
|
|
11698
|
-
|
|
11699
|
-
|
|
11700
|
-
|
|
11701
|
-
|
|
11702
|
-
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
12562
|
+
return ftsLeg({
|
|
12563
|
+
query,
|
|
12564
|
+
cap,
|
|
12565
|
+
scope: {
|
|
12566
|
+
category,
|
|
12567
|
+
source
|
|
12568
|
+
},
|
|
12569
|
+
tokenizer: this.ftsTokenizer,
|
|
12570
|
+
search: (ftsQuery, scope, limit) => this.facts.ftsSearch(ftsQuery, scope.category, limit, scope.source),
|
|
12571
|
+
substringSearch: (terms, scope, limit) => this.facts.ftsSubstringSearch(terms, scope.category, limit, scope.source)
|
|
12572
|
+
});
|
|
11707
12573
|
}
|
|
11708
12574
|
/**
|
|
11709
12575
|
* Entity leg over the precomputed candidate set (see `searchLegs`): the candidates are the facts
|
|
@@ -11810,7 +12676,7 @@ var MemoryStore = class {
|
|
|
11810
12676
|
supersededId
|
|
11811
12677
|
};
|
|
11812
12678
|
})();
|
|
11813
|
-
if (supersededId !== null) this.
|
|
12679
|
+
if (supersededId !== null) evictVectors(this.vstore, [supersededId]);
|
|
11814
12680
|
return result;
|
|
11815
12681
|
}
|
|
11816
12682
|
/**
|
|
@@ -12179,10 +13045,7 @@ var ChunksDao = class {
|
|
|
12179
13045
|
}
|
|
12180
13046
|
/** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
|
|
12181
13047
|
setEntitiesVersion(ids, version) {
|
|
12182
|
-
|
|
12183
|
-
const { placeholders, values } = inList(batch);
|
|
12184
|
-
this.db.prepare(`UPDATE doc_chunks SET entities_version = ? WHERE chunk_id IN (${placeholders})`).run(version, ...values);
|
|
12185
|
-
}
|
|
13048
|
+
setEntitiesVersionBatch(this.db, "doc_chunks", "chunk_id", ids, version);
|
|
12186
13049
|
}
|
|
12187
13050
|
/** Every persisted chunk vector, for rebuilding the in-memory index at open. */
|
|
12188
13051
|
/**
|
|
@@ -12195,14 +13058,68 @@ var ChunksDao = class {
|
|
|
12195
13058
|
vectorRows() {
|
|
12196
13059
|
return this.db.prepare("SELECT chunk_id AS id, semantic_vector AS vec, embedding_model AS embedding_model FROM doc_chunks WHERE semantic_vector IS NOT NULL").all();
|
|
12197
13060
|
}
|
|
13061
|
+
/**
|
|
13062
|
+
* Vector-space health of the WHOLE chunk corpus, by kind, WITHOUT decoding a single blob.
|
|
13063
|
+
*
|
|
13064
|
+
* The same two counts the open-time warning reports (`stale` = wrong width, `space_stale` = same
|
|
13065
|
+
* width but another recorded space — `embedding_model IS NOT` is null-safe, so a row with no
|
|
13066
|
+
* recorded space counts as foreign exactly like the JS comparison in `reloadIndex`) plus `missing`
|
|
13067
|
+
* (no vector at all), which is the third reason the semantic leg cannot use a chunk.
|
|
13068
|
+
*
|
|
13069
|
+
* This is what lets the shared repair flow read "what is stale" cheaply per slice instead of
|
|
13070
|
+
* re-scanning and hashing the corpus (`corpusState` reads every text; a batched migration must
|
|
13071
|
+
* not).
|
|
13072
|
+
*/
|
|
13073
|
+
vectorSpaceCounts(space, expectedBytes) {
|
|
13074
|
+
const row = this.db.prepare(`SELECT
|
|
13075
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NULL THEN 1 ELSE 0 END), 0) AS missing,
|
|
13076
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) != :bytes THEN 1 ELSE 0 END), 0) AS stale,
|
|
13077
|
+
COALESCE(SUM(CASE WHEN semantic_vector IS NOT NULL AND length(semantic_vector) = :bytes AND embedding_model IS NOT :space THEN 1 ELSE 0 END), 0) AS space_stale
|
|
13078
|
+
FROM doc_chunks`).get({
|
|
13079
|
+
bytes: expectedBytes,
|
|
13080
|
+
space
|
|
13081
|
+
});
|
|
13082
|
+
return {
|
|
13083
|
+
stale: Number(row?.stale ?? 0),
|
|
13084
|
+
space_stale: Number(row?.space_stale ?? 0),
|
|
13085
|
+
missing: Number(row?.missing ?? 0)
|
|
13086
|
+
};
|
|
13087
|
+
}
|
|
13088
|
+
/** Chunks with no persisted vector — the repair flow's `missing` term in `remaining`. */
|
|
13089
|
+
countMissingVectors() {
|
|
13090
|
+
return Number(this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks WHERE semantic_vector IS NULL").get()?.n ?? 0);
|
|
13091
|
+
}
|
|
13092
|
+
/**
|
|
13093
|
+
* Chunks the semantic leg cannot use (no vector / wrong width / another space), bounded and
|
|
13094
|
+
* ordered by id — the repair flow's encode source.
|
|
13095
|
+
*
|
|
13096
|
+
* The content-hash check `vectorReusable` adds is deliberately NOT here: a chunk whose vector is
|
|
13097
|
+
* in the current space is usable even if its text was edited behind the store's back, and that
|
|
13098
|
+
* (rarer) case belongs to `kb_reindex`, which re-derives every leg. Repairing the SPACE must not
|
|
13099
|
+
* pay a jieba/hash pass over the corpus to find rows the space change did not touch.
|
|
13100
|
+
*/
|
|
13101
|
+
staleVectorRows(space, expectedBytes, limit) {
|
|
13102
|
+
const bound = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : -1;
|
|
13103
|
+
return this.db.prepare(`SELECT chunk_id AS chunk_id, text AS text FROM doc_chunks
|
|
13104
|
+
WHERE semantic_vector IS NULL OR length(semantic_vector) != :bytes OR embedding_model IS NOT :space
|
|
13105
|
+
ORDER BY chunk_id ASC LIMIT :limit`).all({
|
|
13106
|
+
bytes: expectedBytes,
|
|
13107
|
+
space,
|
|
13108
|
+
limit: bound
|
|
13109
|
+
});
|
|
13110
|
+
}
|
|
12198
13111
|
count() {
|
|
12199
13112
|
return this.db.prepare("SELECT COUNT(*) AS n FROM doc_chunks").get().n;
|
|
12200
13113
|
}
|
|
12201
13114
|
/**
|
|
12202
|
-
* Replace the entity rows of the given chunks in one transaction
|
|
12203
|
-
*
|
|
13115
|
+
* Replace the entity rows of the given chunks in one transaction.
|
|
13116
|
+
*
|
|
13117
|
+
* `replace` is REQUIRED, not defaulted: `true` clears the old rows first (the entity sweep, where
|
|
13118
|
+
* the extractor may have changed), `false` is the ingest path, where the chunk ids are freshly
|
|
13119
|
+
* inserted and an absent old row set. A default would let one caller silently inherit the other's
|
|
13120
|
+
* semantics — an extra `DELETE` per chunk on ingest, or an append-on-top-of-stale sweep.
|
|
12204
13121
|
*/
|
|
12205
|
-
replaceEntities(rows, replace
|
|
13122
|
+
replaceEntities(rows, replace) {
|
|
12206
13123
|
const del = this.db.prepare("DELETE FROM chunk_entities WHERE chunk_id = ?");
|
|
12207
13124
|
const ins = this.db.prepare("INSERT OR IGNORE INTO chunk_entities (chunk_id, name) VALUES (?, ?)");
|
|
12208
13125
|
this.db.transaction(() => {
|
|
@@ -12214,17 +13131,12 @@ var ChunksDao = class {
|
|
|
12214
13131
|
}
|
|
12215
13132
|
/** Batched entity bags: `Map<chunkId, names>` (ids with none are absent). */
|
|
12216
13133
|
entityBags(ids) {
|
|
12217
|
-
|
|
12218
|
-
|
|
12219
|
-
|
|
12220
|
-
|
|
12221
|
-
|
|
12222
|
-
|
|
12223
|
-
if (list) list.push(r.name);
|
|
12224
|
-
else out.set(r.chunk_id, [r.name]);
|
|
12225
|
-
}
|
|
12226
|
-
}
|
|
12227
|
-
return out;
|
|
13134
|
+
return entityBags(this.db, ids, {
|
|
13135
|
+
table: "chunk_entities",
|
|
13136
|
+
keyColumn: "chunk_id",
|
|
13137
|
+
nameColumn: "name",
|
|
13138
|
+
seedEmpty: false
|
|
13139
|
+
});
|
|
12228
13140
|
}
|
|
12229
13141
|
/** Chunk body + owning-document identity for the fused candidate ids. */
|
|
12230
13142
|
hits(ids) {
|
|
@@ -12264,13 +13176,48 @@ var ChunksDao = class {
|
|
|
12264
13176
|
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12265
13177
|
WHERE (? IS NULL OR d.domain = ?)`).all(domain ?? null, domain ?? null);
|
|
12266
13178
|
}
|
|
13179
|
+
/**
|
|
13180
|
+
* Chunk rows whose `chunk_entities` were produced by OLDER extraction rules — the entity sweep's
|
|
13181
|
+
* selection, bounded and ordered by id.
|
|
13182
|
+
*
|
|
13183
|
+
* `COALESCE(entities_version, 0) < :version`, not `entities_version != :version`: the column is
|
|
13184
|
+
* NULLABLE on purpose (a row written before the derived-state columns existed must look stale
|
|
13185
|
+
* exactly once and be adopted by the first rebuild — see `ChunkStateRow`), and a bare `<` would
|
|
13186
|
+
* evaluate to NULL for those rows and let them hide from the sweep forever.
|
|
13187
|
+
*
|
|
13188
|
+
* The predicate is index-served in id order (`doc_chunks` is keyed by `chunk_id`), so the LIMIT
|
|
13189
|
+
* bounds what is READ, not only what is rebuilt — the same property `FactsDao.staleEntityRows`
|
|
13190
|
+
* documents for the memory side.
|
|
13191
|
+
*/
|
|
13192
|
+
chunkEntityRows(version, limit, domain) {
|
|
13193
|
+
return this.db.prepare(`SELECT dc.chunk_id AS chunk_id, dc.text AS text
|
|
13194
|
+
FROM doc_chunks dc
|
|
13195
|
+
JOIN documents d ON d.doc_id = dc.doc_id
|
|
13196
|
+
WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)
|
|
13197
|
+
ORDER BY dc.chunk_id ASC LIMIT :limit`).all({
|
|
13198
|
+
version,
|
|
13199
|
+
limit,
|
|
13200
|
+
domain: domain ?? null
|
|
13201
|
+
});
|
|
13202
|
+
}
|
|
13203
|
+
/** How many chunks still carry entity rows from older rules — the sweep's `deferred` source. */
|
|
13204
|
+
countStaleEntityChunks(version, domain) {
|
|
13205
|
+
return Number(this.db.prepare(`SELECT COUNT(*) AS n
|
|
13206
|
+
FROM doc_chunks dc
|
|
13207
|
+
JOIN documents d ON d.doc_id = dc.doc_id
|
|
13208
|
+
WHERE COALESCE(dc.entities_version, 0) < :version AND (:domain IS NULL OR d.domain = :domain)`).get({
|
|
13209
|
+
version,
|
|
13210
|
+
domain: domain ?? null
|
|
13211
|
+
})?.n ?? 0);
|
|
13212
|
+
}
|
|
12267
13213
|
/** Rebuild the FTS index from the content table (write-path maintenance). */
|
|
12268
13214
|
rebuildFts() {
|
|
12269
13215
|
this.db.exec("INSERT INTO doc_chunks_fts(doc_chunks_fts) VALUES('rebuild')");
|
|
12270
13216
|
}
|
|
12271
13217
|
/** Active chunks sharing at least one entity name with the query (jaccard candidate set). */
|
|
12272
13218
|
/**
|
|
12273
|
-
* Entity-overlap candidates,
|
|
13219
|
+
* Entity-overlap candidates, highest-score first and CAPPED (see the memory store's sibling;
|
|
13220
|
+
* the score/order tail itself lives in `./shared.js#entityCandidateTail`).
|
|
12274
13221
|
*
|
|
12275
13222
|
* `limit` is applied per name-batch and the batches are unioned, so the result is bounded by
|
|
12276
13223
|
* `limit x ceil(names/batch)` — corpus-independent, which is what this query can promise. The
|
|
@@ -12280,11 +13227,19 @@ var ChunksDao = class {
|
|
|
12280
13227
|
* of the truth. (The memory store's sibling does not batch, so its single `LIMIT` is already exact.)
|
|
12281
13228
|
*/
|
|
12282
13229
|
candidatesByEntityNames(names, domain, source, limit, queryWidth, widthCap) {
|
|
12283
|
-
|
|
12284
|
-
|
|
12285
|
-
|
|
12286
|
-
|
|
12287
|
-
|
|
13230
|
+
return entityCandidates(this.db, {
|
|
13231
|
+
names,
|
|
13232
|
+
filterParams: [
|
|
13233
|
+
domain ?? null,
|
|
13234
|
+
domain ?? null,
|
|
13235
|
+
source ?? null,
|
|
13236
|
+
source ?? null
|
|
13237
|
+
],
|
|
13238
|
+
limit,
|
|
13239
|
+
queryWidth,
|
|
13240
|
+
widthCap,
|
|
13241
|
+
batchNames: true,
|
|
13242
|
+
sql: (placeholders) => `SELECT ce.chunk_id AS id, COUNT(*) AS shared,
|
|
12288
13243
|
(SELECT COUNT(*) FROM chunk_entities x WHERE x.chunk_id = ce.chunk_id) AS total
|
|
12289
13244
|
FROM chunk_entities ce
|
|
12290
13245
|
JOIN doc_chunks dc ON dc.chunk_id = ce.chunk_id
|
|
@@ -12295,12 +13250,8 @@ var ChunksDao = class {
|
|
|
12295
13250
|
-- removed document's chunks out of a result is the JOIN itself.
|
|
12296
13251
|
WHERE ce.name IN (${placeholders})
|
|
12297
13252
|
AND (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?)
|
|
12298
|
-
|
|
12299
|
-
|
|
12300
|
-
LIMIT ?`).all(...values, domain ?? null, domain ?? null, source ?? null, source ?? null, queryWidth, widthCap, limit);
|
|
12301
|
-
for (const r of rows) out.add(r.chunk_id);
|
|
12302
|
-
}
|
|
12303
|
-
return [...out];
|
|
13253
|
+
${entityCandidateTail("ce.chunk_id")}`
|
|
13254
|
+
});
|
|
12304
13255
|
}
|
|
12305
13256
|
/**
|
|
12306
13257
|
* How many chunks carry each name — the knowledge side of the entity leg's document frequency.
|
|
@@ -12310,12 +13261,10 @@ var ChunksDao = class {
|
|
|
12310
13261
|
* all chunks, which is the population the leg ranks.
|
|
12311
13262
|
*/
|
|
12312
13263
|
docFrequency(names) {
|
|
12313
|
-
|
|
12314
|
-
|
|
12315
|
-
|
|
12316
|
-
|
|
12317
|
-
for (const row of rows) out.set(row.name, Number(row.df));
|
|
12318
|
-
return out;
|
|
13264
|
+
return queryDocFrequency(this.db, names, {
|
|
13265
|
+
from: " FROM chunk_entities",
|
|
13266
|
+
nameExpr: "name"
|
|
13267
|
+
});
|
|
12319
13268
|
}
|
|
12320
13269
|
/** FTS leg (`bm25` rank; negated by the caller so higher = better). */
|
|
12321
13270
|
ftsSearch(ftsQuery, domain, source, limit) {
|
|
@@ -12335,14 +13284,17 @@ var ChunksDao = class {
|
|
|
12335
13284
|
* the domain/source filter lives.
|
|
12336
13285
|
*/
|
|
12337
13286
|
ftsSubstringSearch(terms, domain, source, limit) {
|
|
12338
|
-
|
|
12339
|
-
if (params.length === 0) return [];
|
|
12340
|
-
return this.db.prepare(`SELECT dc.chunk_id AS id, ${count} AS rank
|
|
13287
|
+
return likeSubstringLeg(this.db, "dc.text", terms, ({ any, count }) => `SELECT dc.chunk_id AS id, ${count} AS rank
|
|
12341
13288
|
FROM doc_chunks dc
|
|
12342
13289
|
JOIN documents d ON d.doc_id = dc.doc_id
|
|
12343
13290
|
WHERE (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?) AND (${any})
|
|
12344
13291
|
ORDER BY rank DESC, dc.chunk_id ASC
|
|
12345
|
-
LIMIT
|
|
13292
|
+
LIMIT ?`, [
|
|
13293
|
+
domain ?? null,
|
|
13294
|
+
domain ?? null,
|
|
13295
|
+
source ?? null,
|
|
13296
|
+
source ?? null
|
|
13297
|
+
], limit);
|
|
12346
13298
|
}
|
|
12347
13299
|
};
|
|
12348
13300
|
//#endregion
|
|
@@ -13649,6 +14601,15 @@ var KnowledgeStore = class {
|
|
|
13649
14601
|
/** > 0 while a batch (directory import) is writing: commits and baseline refreshes wait. */
|
|
13650
14602
|
deferCorpusWrites = 0;
|
|
13651
14603
|
/**
|
|
14604
|
+
* Identity object for the shared entity sweep's in-flight guard (`store/entity_sweep.ts`).
|
|
14605
|
+
*
|
|
14606
|
+
* The guard itself lives in that module; this is only the per-store identity it keys on. It exists
|
|
14607
|
+
* as a field — rather than the adapter object, which is built per call — because the guard must
|
|
14608
|
+
* survive between two overlapping calls: `reindex` can be reached from a tool call and the
|
|
14609
|
+
* settings-page button at once, and both would select the same stale chunks.
|
|
14610
|
+
*/
|
|
14611
|
+
entitySweepGuard = {};
|
|
14612
|
+
/**
|
|
13652
14613
|
* Per-file stamps (`path → mtimeMs:size`) and the file-set they were taken from, so a reconcile
|
|
13653
14614
|
* can ask "what changed?" for the price of one `stat` per document instead of reading and
|
|
13654
14615
|
* hashing the whole corpus. `null` = no baseline yet.
|
|
@@ -13696,37 +14657,27 @@ var KnowledgeStore = class {
|
|
|
13696
14657
|
this.reloadIndex();
|
|
13697
14658
|
}
|
|
13698
14659
|
/**
|
|
13699
|
-
* Rebuild the in-memory vstore from persisted chunk vectors
|
|
14660
|
+
* Rebuild the in-memory vstore from persisted chunk vectors, and say loudly what belongs to
|
|
14661
|
+
* another vector space.
|
|
13700
14662
|
*
|
|
13701
|
-
* Also detects a VECTOR-SPACE change, which the reload itself cannot see:
|
|
13702
|
-
*
|
|
13703
|
-
*
|
|
13704
|
-
*
|
|
13705
|
-
*
|
|
14663
|
+
* Also detects a VECTOR-SPACE change, which the reload itself cannot see: the reload checks the
|
|
14664
|
+
* width, and a different model at the same width produces dim-valid vectors that are then ranked
|
|
14665
|
+
* against the current query's — two spaces mixed in one index, with no error anywhere.
|
|
14666
|
+
*
|
|
14667
|
+
* The reload, the count and the warning are the SHARED open-time step (`store/vector_repair.ts`),
|
|
14668
|
+
* which is what turned the knowledge store from "warns about a swapped model but never repairs it"
|
|
14669
|
+
* into the same automatic path memory has always had; this store's half is the row read. The
|
|
14670
|
+
* manual entry stays `vectors --fix` (the vector-only repair); `kb_reindex` remains the full
|
|
14671
|
+
* FTS + entities + vectors re-entry.
|
|
13706
14672
|
*/
|
|
13707
14673
|
reloadIndex() {
|
|
13708
|
-
|
|
13709
|
-
|
|
13710
|
-
|
|
13711
|
-
|
|
13712
|
-
|
|
13713
|
-
|
|
13714
|
-
|
|
13715
|
-
for (const row of persisted) {
|
|
13716
|
-
if (!row.vec || row.vec.byteLength !== expectedBytes) {
|
|
13717
|
-
stale++;
|
|
13718
|
-
continue;
|
|
13719
|
-
}
|
|
13720
|
-
if (row.embedding_model !== space) {
|
|
13721
|
-
spaceStale++;
|
|
13722
|
-
if (isPreFingerprintSpace(row.embedding_model)) legacy++;
|
|
13723
|
-
}
|
|
13724
|
-
}
|
|
13725
|
-
reportStaleVectors("knowledge", space, {
|
|
13726
|
-
stale,
|
|
13727
|
-
space_stale: spaceStale,
|
|
13728
|
-
legacy
|
|
13729
|
-
}, "`avantf-mem kb reindex` (MCP/plugin: `kb_reindex`)");
|
|
14674
|
+
loadVectorIndex({
|
|
14675
|
+
store: "knowledge",
|
|
14676
|
+
vstore: this.vstore,
|
|
14677
|
+
rows: this.chunks.vectorRows(),
|
|
14678
|
+
space: this.vectorSpace(),
|
|
14679
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)"
|
|
14680
|
+
});
|
|
13730
14681
|
}
|
|
13731
14682
|
/**
|
|
13732
14683
|
* The write-side domain allowlist AND the 知识页 picker's data, in one place: the configured
|
|
@@ -14005,7 +14956,7 @@ var KnowledgeStore = class {
|
|
|
14005
14956
|
reusable
|
|
14006
14957
|
};
|
|
14007
14958
|
}).immediate();
|
|
14008
|
-
this.
|
|
14959
|
+
evictVectors(this.vstore, result.oldChunkIds);
|
|
14009
14960
|
const indexed = await this.indexChunks(result.newChunkIds, result.reusable);
|
|
14010
14961
|
return {
|
|
14011
14962
|
doc_id: result.doc_id,
|
|
@@ -14042,10 +14993,6 @@ var KnowledgeStore = class {
|
|
|
14042
14993
|
return { file_error: error instanceof Error ? error.message : String(error) };
|
|
14043
14994
|
}
|
|
14044
14995
|
}
|
|
14045
|
-
/** Batched eviction from the live vector index (removeMany when the backend supports it). */
|
|
14046
|
-
evictVectors(ids) {
|
|
14047
|
-
evictVectors(this.vstore, ids);
|
|
14048
|
-
}
|
|
14049
14996
|
/** The resolved ingestion boundary (see `ingest_guard.ts`). */
|
|
14050
14997
|
get ingestLimits() {
|
|
14051
14998
|
return this.kbConfig.ingest;
|
|
@@ -14168,7 +15115,8 @@ var KnowledgeStore = class {
|
|
|
14168
15115
|
};
|
|
14169
15116
|
}
|
|
14170
15117
|
/**
|
|
14171
|
-
*
|
|
15118
|
+
* This store's half of the shared encode-and-persist flow (`store/vector_repair.ts`): the backend
|
|
15119
|
+
* call, the live index and the `WRITE_BATCH` transaction this store measured its way into.
|
|
14172
15120
|
*
|
|
14173
15121
|
* The model call is deliberately NOT batched, and that is a MEASURED decision, not an omission:
|
|
14174
15122
|
* a batched forward pass pads each item to the longest in its group, and real chunks span 53–550
|
|
@@ -14179,34 +15127,43 @@ var KnowledgeStore = class {
|
|
|
14179
15127
|
* length-sorted) for callers whose texts suit it.
|
|
14180
15128
|
*
|
|
14181
15129
|
* What DID pay off is the write side: one commit per `WRITE_BATCH` chunks instead of one per
|
|
14182
|
-
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await
|
|
15130
|
+
* chunk (measured 14.2 µs/row vs 2.2 µs/row), and the transaction never spans an `await` — that
|
|
15131
|
+
* is the `writeBatch` below.
|
|
14183
15132
|
*/
|
|
14184
|
-
|
|
14185
|
-
|
|
14186
|
-
|
|
14187
|
-
|
|
14188
|
-
|
|
14189
|
-
|
|
14190
|
-
|
|
14191
|
-
|
|
14192
|
-
|
|
14193
|
-
writes.
|
|
14194
|
-
chunk_id:
|
|
14195
|
-
bytes:
|
|
15133
|
+
encodeWriteTarget(space) {
|
|
15134
|
+
return {
|
|
15135
|
+
store: "knowledge",
|
|
15136
|
+
space,
|
|
15137
|
+
encode: (text) => this.semantic.encode(text),
|
|
15138
|
+
add: (id, vec) => {
|
|
15139
|
+
this.vstore.add(id, vec);
|
|
15140
|
+
},
|
|
15141
|
+
write: (writes) => {
|
|
15142
|
+
this.chunks.setVectors(writes.map((w) => ({
|
|
15143
|
+
chunk_id: w.id,
|
|
15144
|
+
bytes: w.bytes,
|
|
14196
15145
|
embeddingModel: space
|
|
14197
|
-
});
|
|
14198
|
-
}
|
|
14199
|
-
|
|
14200
|
-
|
|
14201
|
-
|
|
14202
|
-
try {
|
|
14203
|
-
this.chunks.setVectors(writes);
|
|
14204
|
-
encoded += writes.length;
|
|
14205
|
-
} catch {
|
|
14206
|
-
failed += writes.length;
|
|
14207
|
-
this.evictVectors(writes.map((w) => w.chunk_id));
|
|
15146
|
+
})));
|
|
15147
|
+
},
|
|
15148
|
+
writeBatch: WRITE_BATCH,
|
|
15149
|
+
onWriteError: (writes) => {
|
|
15150
|
+
evictVectors(this.vstore, writes.map((w) => w.id));
|
|
14208
15151
|
}
|
|
14209
|
-
}
|
|
15152
|
+
};
|
|
15153
|
+
}
|
|
15154
|
+
/**
|
|
15155
|
+
* Encode texts one at a time and persist them in `WRITE_BATCH` transactions.
|
|
15156
|
+
*
|
|
15157
|
+
* A failed chunk is skipped, not thrown: the rows are already committed and their old vectors
|
|
15158
|
+
* evicted by the ingest path, so a throw here would leave the caller with a failed ingest and NO
|
|
15159
|
+
* COUNT of what was skipped. The shortfall travels back to the tool result
|
|
15160
|
+
* (`IngestResult.vectors_failed`) instead of living only in a log line.
|
|
15161
|
+
*/
|
|
15162
|
+
async encodeAndStore(rows, space) {
|
|
15163
|
+
const { encoded, failed } = await encodeAndPersist(rows.map((r) => ({
|
|
15164
|
+
id: r.chunk_id,
|
|
15165
|
+
text: r.text
|
|
15166
|
+
})), this.encodeWriteTarget(space));
|
|
14210
15167
|
if (failed > 0) retrievalLogger().warn(`kb index: ${String(failed)} of ${String(rows.length)} chunk vector(s) were not written — run kb_reindex to retry (the text/entity/FTS indexes are complete; only the semantic vectors are affected)`);
|
|
14211
15168
|
return {
|
|
14212
15169
|
encoded,
|
|
@@ -14223,6 +15180,102 @@ var KnowledgeStore = class {
|
|
|
14223
15180
|
return vectorSpaceOf(this.semantic, this.config.semantic.local_model);
|
|
14224
15181
|
}
|
|
14225
15182
|
/**
|
|
15183
|
+
* Cheapest possible DETECTION of "the embedding space changed under this store": counts of
|
|
15184
|
+
* persisted chunk vectors whose width or recorded space id no longer matches the one this store
|
|
15185
|
+
* writes into. Reads blob LENGTHS only — never decodes — so the startup warning and the background
|
|
15186
|
+
* drive can afford it at any corpus size.
|
|
15187
|
+
*/
|
|
15188
|
+
vectorSpaceHealth() {
|
|
15189
|
+
const c = this.vectorSpaceCounts();
|
|
15190
|
+
return {
|
|
15191
|
+
stale: c.stale,
|
|
15192
|
+
space_stale: c.space_stale
|
|
15193
|
+
};
|
|
15194
|
+
}
|
|
15195
|
+
/** {@link vectorSpaceHealth} plus `missing`, the third term of the repair flow's `remaining`. */
|
|
15196
|
+
vectorSpaceCounts() {
|
|
15197
|
+
return this.chunks.vectorSpaceCounts(this.vectorSpace(), this.vstore.dim * 4);
|
|
15198
|
+
}
|
|
15199
|
+
/** Async bootstrap the semantic backend (the shared flow's explicit-fix warmup). */
|
|
15200
|
+
async warmupSemantic() {
|
|
15201
|
+
await (this.semantic.warmUp?.() ?? Promise.resolve());
|
|
15202
|
+
return this.semantic.isAvailable();
|
|
15203
|
+
}
|
|
15204
|
+
/**
|
|
15205
|
+
* This store's half of the shared repair flow (`store/vector_repair.ts`): the primitives the flow
|
|
15206
|
+
* steps. The flow itself — batch size, event-loop yield, stop / resume / no-progress rules,
|
|
15207
|
+
* `semantic.auto_migrate`, the dry-run split — lives there once, for both stores.
|
|
15208
|
+
*/
|
|
15209
|
+
vectorRepairTarget() {
|
|
15210
|
+
return {
|
|
15211
|
+
store: "knowledge",
|
|
15212
|
+
manualEntry: "`avantf-mem vectors --fix` (MCP/plugin: `mem_admin vectors_fix`; `kb_reindex` re-runs FTS/entities too)",
|
|
15213
|
+
vectorSpace: () => this.vectorSpace(),
|
|
15214
|
+
semanticAvailable: () => this.semantic.isAvailable(),
|
|
15215
|
+
autoMigrate: () => this.config.semantic.auto_migrate,
|
|
15216
|
+
vectorSpaceHealth: () => this.vectorSpaceHealth(),
|
|
15217
|
+
missingVectorCount: () => this.chunks.countMissingVectors(),
|
|
15218
|
+
classifyVectors: () => this.classifyRepairVectors(),
|
|
15219
|
+
dropVectors: () => 0,
|
|
15220
|
+
reencodeVectors: (limit) => this.reencodeStaleChunks(limit),
|
|
15221
|
+
warmup: () => this.warmupSemantic()
|
|
15222
|
+
};
|
|
15223
|
+
}
|
|
15224
|
+
/**
|
|
15225
|
+
* The chunks the semantic leg cannot use (no vector / wrong width / another space). One SQL
|
|
15226
|
+
* aggregate over blob lengths; `dropIds` is empty because knowledge overwrites in place.
|
|
15227
|
+
*/
|
|
15228
|
+
classifyRepairVectors() {
|
|
15229
|
+
const c = this.vectorSpaceCounts();
|
|
15230
|
+
return {
|
|
15231
|
+
stale: c.stale,
|
|
15232
|
+
space_stale: c.space_stale,
|
|
15233
|
+
missing: c.missing,
|
|
15234
|
+
dropIds: []
|
|
15235
|
+
};
|
|
15236
|
+
}
|
|
15237
|
+
/**
|
|
15238
|
+
* Encode + persist up to `limit` chunks the semantic leg cannot use, bounded and ordered by id.
|
|
15239
|
+
*
|
|
15240
|
+
* It does NOT re-extract entities or rebuild FTS: that is `kb_reindex`'s job, and the whole point
|
|
15241
|
+
* of the automatic repair is that the vector half can run on its own — which is exactly the
|
|
15242
|
+
* objection the "known boundary" in DESIGN §20 used to raise against a background run.
|
|
15243
|
+
*/
|
|
15244
|
+
async reencodeStaleChunks(limit) {
|
|
15245
|
+
const space = this.vectorSpace();
|
|
15246
|
+
const rows = this.chunks.staleVectorRows(space, this.vstore.dim * 4, limit);
|
|
15247
|
+
if (!rows.length) return {
|
|
15248
|
+
encoded: 0,
|
|
15249
|
+
failed: 0
|
|
15250
|
+
};
|
|
15251
|
+
return await this.encodeAndStore(rows, space);
|
|
15252
|
+
}
|
|
15253
|
+
/**
|
|
15254
|
+
* The explicit, unbounded repair behind `mem_admin vectors_fix` / `avantf-mem vectors --fix`.
|
|
15255
|
+
* `dry_run` previews every count without writing — and without loading the model, which is why
|
|
15256
|
+
* the preview also reports `would_warm` (see the contract's `VectorsFixStoreReport`).
|
|
15257
|
+
*/
|
|
15258
|
+
async vectorsFix(dryRun = false) {
|
|
15259
|
+
return (await repairVectorSlice(this.vectorRepairTarget(), { dryRun })).report;
|
|
15260
|
+
}
|
|
15261
|
+
/**
|
|
15262
|
+
* ONE bounded slice of the vector-space migration: at most `batchSize` chunks are re-encoded, so
|
|
15263
|
+
* the caller keeps control of the event loop between slices. This is the unit the background
|
|
15264
|
+
* migration ({@link migrateVectors}) and its tests drive.
|
|
15265
|
+
*/
|
|
15266
|
+
async migrateVectorsBatch(batchSize = 16) {
|
|
15267
|
+
return await migrateVectorSlice(this.vectorRepairTarget(), batchSize);
|
|
15268
|
+
}
|
|
15269
|
+
/**
|
|
15270
|
+
* Drive the bounded background migration for this store until it is current, the model is
|
|
15271
|
+
* unavailable, or `shouldStop()` turns true (plugin unmount / process shutdown). The loop, the
|
|
15272
|
+
* `semantic.auto_migrate` switch, the no-progress rule and the store-named logging all live in the
|
|
15273
|
+
* shared flow (`store/vector_repair.ts`); this is the one-store entry.
|
|
15274
|
+
*/
|
|
15275
|
+
async migrateVectors(opts = {}) {
|
|
15276
|
+
return await driveVectorRepair(this.vectorRepairTarget(), opts);
|
|
15277
|
+
}
|
|
15278
|
+
/**
|
|
14226
15279
|
* A stored vector is reusable when it exists, matches the space, and its text is unchanged.
|
|
14227
15280
|
*
|
|
14228
15281
|
* The three checks answer different questions, and the hash is deliberately the RAREST of them
|
|
@@ -14236,17 +15289,43 @@ var KnowledgeStore = class {
|
|
|
14236
15289
|
if (row.embedding_model !== space) return false;
|
|
14237
15290
|
return row.content_hash === contentHash(row.text);
|
|
14238
15291
|
}
|
|
14239
|
-
/**
|
|
14240
|
-
|
|
15292
|
+
/**
|
|
15293
|
+
* Extract one chunk's entity names, bounded by `MAX_ENTITIES_PER_CHUNK`.
|
|
15294
|
+
*
|
|
15295
|
+
* The bound is here, not in either caller: it is a property of what `chunk_entities` may hold for
|
|
15296
|
+
* one chunk, and the ingest path and the sweep must not disagree about it.
|
|
15297
|
+
*/
|
|
15298
|
+
async extractRowEntities(text) {
|
|
15299
|
+
return (await extractEntities(text)).map((e) => e.name).slice(0, MAX_ENTITIES_PER_CHUNK);
|
|
15300
|
+
}
|
|
15301
|
+
/** One transaction replacing the entity rows of the given chunks (the ingest path). */
|
|
15302
|
+
async replaceChunkEntities(rows) {
|
|
14241
15303
|
const extracted = [];
|
|
14242
|
-
for (const r of rows) {
|
|
14243
|
-
|
|
14244
|
-
|
|
14245
|
-
|
|
14246
|
-
|
|
14247
|
-
|
|
14248
|
-
|
|
14249
|
-
|
|
15304
|
+
for (const r of rows) extracted.push({
|
|
15305
|
+
chunk_id: r.chunk_id,
|
|
15306
|
+
names: await this.extractRowEntities(r.text)
|
|
15307
|
+
});
|
|
15308
|
+
this.chunks.replaceEntities(extracted, false);
|
|
15309
|
+
}
|
|
15310
|
+
/**
|
|
15311
|
+
* The per-row primitive of the shared entity sweep (`store/entity_sweep.ts`).
|
|
15312
|
+
*
|
|
15313
|
+
* Replaces ONE chunk's entity rows and stamps that same row with `ENTITY_EXTRACTOR_VERSION`, in one
|
|
15314
|
+
* transaction. Knowledge's domain difference from memory is exactly this: memory's transaction also
|
|
15315
|
+
* moves triples, the HRR bundle and the conflict queue, while a chunk has none of those — its
|
|
15316
|
+
* derived state is the entity names alone.
|
|
15317
|
+
*
|
|
15318
|
+
* Per row, not per batch, because the sweep's contract is that a failure leaves THAT row stale and
|
|
15319
|
+
* retryable: inside a batched transaction one bad extraction would roll back the whole batch, and
|
|
15320
|
+
* rows it never reached would look stamped.
|
|
15321
|
+
*/
|
|
15322
|
+
async replaceRowEntities(chunkId, text) {
|
|
15323
|
+
const names = await this.extractRowEntities(text);
|
|
15324
|
+
this.chunks.replaceEntities([{
|
|
15325
|
+
chunk_id: chunkId,
|
|
15326
|
+
names
|
|
15327
|
+
}], true);
|
|
15328
|
+
this.chunks.setEntitiesVersion([chunkId], 1);
|
|
14250
15329
|
}
|
|
14251
15330
|
/**
|
|
14252
15331
|
* Ingest local files/directories. Every path goes through the SAME boundary as
|
|
@@ -14330,7 +15409,7 @@ var KnowledgeStore = class {
|
|
|
14330
15409
|
const chunkIds = this.chunks.idsForDoc(docId);
|
|
14331
15410
|
const removed = this.docs.remove(docId);
|
|
14332
15411
|
if (removed) this.commitCorpus(`remove: ${doc.domain}/${doc.source}/${doc.title}`);
|
|
14333
|
-
this.
|
|
15412
|
+
evictVectors(this.vstore, chunkIds);
|
|
14334
15413
|
return removed;
|
|
14335
15414
|
}
|
|
14336
15415
|
/**
|
|
@@ -14522,12 +15601,11 @@ var KnowledgeStore = class {
|
|
|
14522
15601
|
async reindex(domain, opts) {
|
|
14523
15602
|
const dryRun = opts?.dryRun === true;
|
|
14524
15603
|
const space = this.vectorSpace();
|
|
14525
|
-
const
|
|
14526
|
-
const
|
|
14527
|
-
const staleVectors = rows.filter((r) => !this.vectorReusable(r, space));
|
|
15604
|
+
const corpus = this.chunks.corpusState(domain);
|
|
15605
|
+
const staleVectors = corpus.filter((r) => !this.vectorReusable(r, space));
|
|
14528
15606
|
const semanticAvailable = this.semantic.isAvailable();
|
|
14529
15607
|
const report = {
|
|
14530
|
-
chunks:
|
|
15608
|
+
chunks: corpus.length,
|
|
14531
15609
|
entities_rebuilt: 0,
|
|
14532
15610
|
vectors_stale: staleVectors.length,
|
|
14533
15611
|
vectors_encoded: 0,
|
|
@@ -14536,11 +15614,19 @@ var KnowledgeStore = class {
|
|
|
14536
15614
|
};
|
|
14537
15615
|
if (dryRun) return report;
|
|
14538
15616
|
this.chunks.rebuildFts();
|
|
14539
|
-
|
|
14540
|
-
|
|
14541
|
-
|
|
14542
|
-
|
|
14543
|
-
|
|
15617
|
+
report.entities_rebuilt = (await sweepVersionedEntities({
|
|
15618
|
+
staleEntityBatch: (limit) => ({
|
|
15619
|
+
total: this.chunks.countStaleEntityChunks(1, domain),
|
|
15620
|
+
rows: this.chunks.chunkEntityRows(1, limit, domain)
|
|
15621
|
+
}),
|
|
15622
|
+
staleEntityCount: () => this.chunks.countStaleEntityChunks(1, domain),
|
|
15623
|
+
extractAndWriteEntities: async (row) => {
|
|
15624
|
+
await this.replaceRowEntities(row.chunk_id, row.text);
|
|
15625
|
+
},
|
|
15626
|
+
onRowError: (row, error) => {
|
|
15627
|
+
retrievalLogger().warn(`kb entity sweep: chunk ${String(row.chunk_id)} could not be rebuilt (${error instanceof Error ? error.message : String(error)}) — it keeps its old stamp; the next reindex retries it`);
|
|
15628
|
+
}
|
|
15629
|
+
}, this.entitySweepGuard, { budget: this.chunks.countStaleEntityChunks(1, domain) })).rebuilt;
|
|
14544
15630
|
if (!semanticAvailable) this.semantic.ensureWarm?.();
|
|
14545
15631
|
else if (staleVectors.length) report.vectors_encoded = (await this.encodeAndStore(staleVectors, space)).encoded;
|
|
14546
15632
|
return report;
|
|
@@ -14556,18 +15642,7 @@ var KnowledgeStore = class {
|
|
|
14556
15642
|
}
|
|
14557
15643
|
/** Chunk search over the knowledge corpus (semantic + FTS + entity, joint fusion). */
|
|
14558
15644
|
async search(query, opts) {
|
|
14559
|
-
const result = await
|
|
14560
|
-
query,
|
|
14561
|
-
limit: opts?.limit,
|
|
14562
|
-
overFetch: opts?.overFetch,
|
|
14563
|
-
maxTokens: opts?.maxTokens,
|
|
14564
|
-
queryVector: opts?.queryVector,
|
|
14565
|
-
recordStats: opts?.recordStats,
|
|
14566
|
-
floors: opts?.floors,
|
|
14567
|
-
...opts?.includeScores === void 0 ? {} : { includeScores: opts.includeScores },
|
|
14568
|
-
...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs },
|
|
14569
|
-
...opts?.rewriteQuery === void 0 ? {} : { rewriteQuery: opts.rewriteQuery }
|
|
14570
|
-
});
|
|
15645
|
+
const result = await runHybridSearch(this.hybridDeps(opts), query, opts);
|
|
14571
15646
|
opts?.onResult?.(result);
|
|
14572
15647
|
return result.hits;
|
|
14573
15648
|
}
|
|
@@ -14583,10 +15658,7 @@ var KnowledgeStore = class {
|
|
|
14583
15658
|
*/
|
|
14584
15659
|
hybridDeps(opts) {
|
|
14585
15660
|
const rowsById = /* @__PURE__ */ new Map();
|
|
14586
|
-
return {
|
|
14587
|
-
kind: "knowledge",
|
|
14588
|
-
config: this.config,
|
|
14589
|
-
semantic: this.semantic,
|
|
15661
|
+
return makeHybridDeps("knowledge", this.config, this.semantic, {
|
|
14590
15662
|
legs: (ctx) => this.searchLegs(opts, ctx),
|
|
14591
15663
|
texts: (ids) => {
|
|
14592
15664
|
rowsById.clear();
|
|
@@ -14612,7 +15684,7 @@ var KnowledgeStore = class {
|
|
|
14612
15684
|
}];
|
|
14613
15685
|
});
|
|
14614
15686
|
}
|
|
14615
|
-
};
|
|
15687
|
+
});
|
|
14616
15688
|
}
|
|
14617
15689
|
/**
|
|
14618
15690
|
* The knowledge legs: semantic, FTS, and entity overlap against the entities extracted AT INGEST
|
|
@@ -14622,40 +15694,12 @@ var KnowledgeStore = class {
|
|
|
14622
15694
|
* matching corpus only makes `fuse` normalize and sort it (see `MemoryStore.searchLegs`).
|
|
14623
15695
|
*/
|
|
14624
15696
|
async searchLegs(opts, ctx) {
|
|
14625
|
-
|
|
14626
|
-
* Wrap one leg's floored scores. `raw` is the PRE-floor set: `capped` must be measured there,
|
|
14627
|
-
* because the relevance floor removes the tail anyway and re-deriving the flag afterwards would
|
|
14628
|
-
* erase the "cut at legCap" signal exactly when it bound.
|
|
14629
|
-
*/
|
|
14630
|
-
const leg = (scores, weight, name, dropped, raw) => ({
|
|
14631
|
-
weight,
|
|
14632
|
-
scores,
|
|
14633
|
-
capped: (raw ?? scores).size === ctx.legCap,
|
|
14634
|
-
leg: name,
|
|
14635
|
-
droppedByFloor: dropped
|
|
14636
|
-
});
|
|
15697
|
+
const runner = makeLegRunner(ctx);
|
|
14637
15698
|
const ftsRaw = this.ftsPath(ctx.query, opts?.domain, opts?.source, ctx.legCap);
|
|
14638
|
-
const ftsFloored = applyTermFloor(ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
|
|
14639
15699
|
return [
|
|
14640
|
-
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) =>
|
|
14641
|
-
|
|
14642
|
-
|
|
14643
|
-
weight: ctx.weights.semantic,
|
|
14644
|
-
scores: floored.scores,
|
|
14645
|
-
leg: "semantic",
|
|
14646
|
-
droppedByFloor: floored.dropped
|
|
14647
|
-
};
|
|
14648
|
-
}) : {
|
|
14649
|
-
weight: ctx.weights.semantic,
|
|
14650
|
-
scores: /* @__PURE__ */ new Map(),
|
|
14651
|
-
leg: "semantic",
|
|
14652
|
-
droppedByFloor: 0
|
|
14653
|
-
},
|
|
14654
|
-
leg(ftsFloored.scores, ctx.weights.fts, "fts", ftsFloored.dropped, ftsRaw),
|
|
14655
|
-
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => {
|
|
14656
|
-
const floored = applyScoreFloor(raw, ctx.floors.jaccard);
|
|
14657
|
-
return leg(floored.scores, ctx.weights.jaccard, "jaccard", floored.dropped, raw);
|
|
14658
|
-
})
|
|
15700
|
+
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) => runner.semantic(raw, ctx.weights.semantic, ctx.floors.semantic)) : runner.semanticOff(ctx.weights.semantic),
|
|
15701
|
+
runner.term("fts", ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.weights.fts, ctx.floors.fts),
|
|
15702
|
+
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => runner.scored("jaccard", raw, ctx.weights.jaccard, ctx.floors.jaccard))
|
|
14659
15703
|
];
|
|
14660
15704
|
}
|
|
14661
15705
|
/** The texts of a candidate set, for the per-row FTS floor (one batched query). */
|
|
@@ -14670,43 +15714,55 @@ var KnowledgeStore = class {
|
|
|
14670
15714
|
* re-encoding (performance review §7.7 / P8).
|
|
14671
15715
|
*/
|
|
14672
15716
|
async semanticPath(query, k, opts, queryVector, onVector) {
|
|
14673
|
-
|
|
14674
|
-
|
|
14675
|
-
|
|
14676
|
-
|
|
14677
|
-
|
|
14678
|
-
|
|
14679
|
-
|
|
14680
|
-
|
|
14681
|
-
|
|
14682
|
-
|
|
15717
|
+
return semanticLeg({
|
|
15718
|
+
query,
|
|
15719
|
+
k,
|
|
15720
|
+
queryVector,
|
|
15721
|
+
onVector,
|
|
15722
|
+
encode: (q) => this.semantic.encode(q),
|
|
15723
|
+
dim: this.vstore.dim,
|
|
15724
|
+
topk: (vec, n) => this.vstore.topk(vec, n),
|
|
15725
|
+
filterTopk: (ids) => this.scopeFilter(ids, opts)
|
|
15726
|
+
});
|
|
15727
|
+
}
|
|
15728
|
+
/**
|
|
15729
|
+
* The semantic leg's scope filter, as a `filterTopk` primitive: `undefined` when NO domain/source
|
|
15730
|
+
* was asked means "keep the whole pool" — the short-circuit is preserved, so no `chunks.meta`
|
|
15731
|
+
* lookup runs at all. Otherwise one batched lookup, and rows whose domain/source do not match (or
|
|
15732
|
+
* whose chunk is gone) are dropped.
|
|
15733
|
+
*/
|
|
15734
|
+
scopeFilter(ids, opts) {
|
|
15735
|
+
if (!opts?.domain && !opts?.source) return void 0;
|
|
15736
|
+
const meta = new Map(this.chunks.meta(ids).map((r) => [r.chunk_id, r]));
|
|
15737
|
+
const keep = /* @__PURE__ */ new Set();
|
|
15738
|
+
for (const id of ids) {
|
|
15739
|
+
const r = meta.get(id);
|
|
14683
15740
|
if (!r) continue;
|
|
14684
15741
|
if (opts.domain && r.domain !== opts.domain) continue;
|
|
14685
15742
|
if (opts.source && r.source !== opts.source) continue;
|
|
14686
|
-
|
|
15743
|
+
keep.add(id);
|
|
14687
15744
|
}
|
|
14688
|
-
return
|
|
15745
|
+
return keep;
|
|
14689
15746
|
}
|
|
14690
15747
|
/**
|
|
14691
15748
|
* Sync relevance probe for the plugin's conditional hint (DESIGN §12). Same contract as
|
|
14692
15749
|
* `MemoryStore.lexicalProbe` — see `store/lexical.ts`.
|
|
14693
15750
|
*/
|
|
14694
15751
|
lexicalProbe(text, stopAt = Number.POSITIVE_INFINITY) {
|
|
14695
|
-
return
|
|
14696
|
-
const fts = buildFtsQuery(term, this.ftsTokenizer);
|
|
14697
|
-
return fts !== null && this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0;
|
|
14698
|
-
}, stopAt);
|
|
15752
|
+
return probeStoreTerms(text, this.ftsTokenizer, (fts) => this.chunks.ftsSearch(fts, void 0, void 0, 1).length > 0, stopAt);
|
|
14699
15753
|
}
|
|
14700
15754
|
ftsPath(query, domain, source, cap) {
|
|
14701
|
-
|
|
14702
|
-
|
|
14703
|
-
|
|
14704
|
-
|
|
14705
|
-
|
|
14706
|
-
|
|
14707
|
-
|
|
14708
|
-
|
|
14709
|
-
|
|
15755
|
+
return ftsLeg({
|
|
15756
|
+
query,
|
|
15757
|
+
cap,
|
|
15758
|
+
scope: {
|
|
15759
|
+
domain,
|
|
15760
|
+
source
|
|
15761
|
+
},
|
|
15762
|
+
tokenizer: this.ftsTokenizer,
|
|
15763
|
+
search: (ftsQuery, scope, limit) => this.chunks.ftsSearch(ftsQuery, scope.domain, scope.source, limit),
|
|
15764
|
+
substringSearch: (terms, scope, limit) => this.chunks.ftsSubstringSearch(terms, scope.domain, scope.source, limit)
|
|
15765
|
+
});
|
|
14710
15766
|
}
|
|
14711
15767
|
/**
|
|
14712
15768
|
* Entity overlap against entities extracted AT INGEST time (chunk_entities) — no corpus scan.
|
|
@@ -15079,7 +16135,13 @@ function buildRuntime(opts) {
|
|
|
15079
16135
|
case "contradict_resolve": return this.memory.resolveContradiction(req.contradiction_id, req.resolution, req.loser_fact_id);
|
|
15080
16136
|
case "maintenance": return this.memory.maintenance();
|
|
15081
16137
|
case "vectors_diagnose": return this.memory.vectorsDiagnose();
|
|
15082
|
-
case "vectors_fix": return
|
|
16138
|
+
case "vectors_fix": return vectorsFix({
|
|
16139
|
+
memory: this.memory.vectorRepairTarget(),
|
|
16140
|
+
knowledge: this.knowledge.vectorRepairTarget()
|
|
16141
|
+
}, {
|
|
16142
|
+
dryRun: req.dry_run ?? false,
|
|
16143
|
+
...req.store === void 0 ? {} : { store: req.store }
|
|
16144
|
+
});
|
|
15083
16145
|
default: return { error: `admin.${req.action} not implemented` };
|
|
15084
16146
|
}
|
|
15085
16147
|
},
|
|
@@ -15331,6 +16393,7 @@ const descriptors = [
|
|
|
15331
16393
|
offset: count.optional(),
|
|
15332
16394
|
reason: optionalText.optional(),
|
|
15333
16395
|
dry_run: bool.optional(),
|
|
16396
|
+
store: optionalText.optional(),
|
|
15334
16397
|
contradiction_id: count.optional(),
|
|
15335
16398
|
resolution: optionalText.optional(),
|
|
15336
16399
|
loser_fact_id: count.optional()
|
|
@@ -15593,6 +16656,12 @@ function cannotDetermine(error, log, prefix) {
|
|
|
15593
16656
|
* Run the gate. Call this FIRST in `apply()`: on a refusal nothing has been registered yet, so
|
|
15594
16657
|
* "do not load" costs nothing to unwind. Never throws — not even with a throwing logger, because
|
|
15595
16658
|
* "cannot tell" is not "incompatible".
|
|
16659
|
+
*
|
|
16660
|
+
* The check itself is the BASE's `provision` (gather + verdict + logging, one call). The local
|
|
16661
|
+
* fallback stays on purpose (option A): the interface-generation axis cannot gate this dependency
|
|
16662
|
+
* (same-generation bases never read as `incompatible`), and raising the peer floor would degrade
|
|
16663
|
+
* every host on a base that predates the fix — so a `provision` that cannot be performed still
|
|
16664
|
+
* leaves the plugin loading.
|
|
15596
16665
|
* @param ctx - the plugin's context.
|
|
15597
16666
|
* @param log - the plugin's logger.
|
|
15598
16667
|
* @param compat - the loaded gate runtime (see `src/envinit.ts`).
|
|
@@ -15601,22 +16670,11 @@ function cannotDetermine(error, log, prefix) {
|
|
|
15601
16670
|
* @returns the verdict `apply` branches on.
|
|
15602
16671
|
*/
|
|
15603
16672
|
function provision(ctx, log, compat, spec = compat.spec) {
|
|
15604
|
-
let evidence;
|
|
15605
|
-
try {
|
|
15606
|
-
evidence = compat.module.gatherEvidence(ctx, spec);
|
|
15607
|
-
} catch (error) {
|
|
15608
|
-
return cannotDetermine(error, log, compat.prefix);
|
|
15609
|
-
}
|
|
15610
|
-
let verdict;
|
|
15611
16673
|
try {
|
|
15612
|
-
verdict
|
|
16674
|
+
return { verdict: compat.module.provision(ctx, log, spec) };
|
|
15613
16675
|
} catch (error) {
|
|
15614
16676
|
return cannotDetermine(error, log, compat.prefix);
|
|
15615
16677
|
}
|
|
15616
|
-
for (const line of verdict.lines) try {
|
|
15617
|
-
log[line.level](line.message);
|
|
15618
|
-
} catch {}
|
|
15619
|
-
return { verdict };
|
|
15620
16678
|
}
|
|
15621
16679
|
/**
|
|
15622
16680
|
* Keep one `/mem` command that reports a refusal.
|
|
@@ -16062,83 +17120,6 @@ function rangeOf(value) {
|
|
|
16062
17120
|
return typeof range === "string" && range.trim() !== "" ? range : void 0;
|
|
16063
17121
|
}
|
|
16064
17122
|
//#endregion
|
|
16065
|
-
//#region lib/types/interface_gate.js
|
|
16066
|
-
/** Read one member without letting a hostile module's getter escape. */
|
|
16067
|
-
function member(module, name) {
|
|
16068
|
-
try {
|
|
16069
|
-
return module?.[name];
|
|
16070
|
-
} catch {
|
|
16071
|
-
return;
|
|
16072
|
-
}
|
|
16073
|
-
}
|
|
16074
|
-
function cannotTell(reason) {
|
|
16075
|
-
return {
|
|
16076
|
-
status: "cannot-tell",
|
|
16077
|
-
reason
|
|
16078
|
-
};
|
|
16079
|
-
}
|
|
16080
|
-
/**
|
|
16081
|
-
* Whether the base's shared capabilities may be used under `verdict`.
|
|
16082
|
-
*
|
|
16083
|
-
* The ONE mapping from the gate's verdict to the loader's decision, exported so it is asserted
|
|
16084
|
-
* directly and the loader and its test cannot drift: only `incompatible` withholds the base;
|
|
16085
|
-
* `cannot-tell` does not, because it is not a negative answer. A `false` here is still a DEGRADATION
|
|
16086
|
-
* (own prompt defaults, gate skipped, legacy provisioning), never a refused mount.
|
|
16087
|
-
*/
|
|
16088
|
-
function baseIsUsable(verdict) {
|
|
16089
|
-
return verdict.status !== "incompatible";
|
|
16090
|
-
}
|
|
16091
|
-
/**
|
|
16092
|
-
* Where the build's bake record sits when no explicit URL is given: beside the built ENTRY
|
|
16093
|
-
* (`lib/index.js`), which is where `scripts/link-envinit.mjs` writes it. A per-file `tsc` emit of
|
|
16094
|
-
* this module lives one directory below that entry and looks one level up; the first readable record
|
|
16095
|
-
* wins. Under `vitest` (running from `src/`) neither exists, which is the correct "not baked".
|
|
16096
|
-
*/
|
|
16097
|
-
const BAKED_CANDIDATES = ["./interface-version.json", "../interface-version.json"];
|
|
16098
|
-
/**
|
|
16099
|
-
* Read this artifact's baked requirement through the loaded base's own reader.
|
|
16100
|
-
* @param read - the base's `readInterfaceRequirement`, taken off the loaded module.
|
|
16101
|
-
* @param bakedUrl - test seam: read EXACTLY this record instead of walking {@link BAKED_CANDIDATES}.
|
|
16102
|
-
*/
|
|
16103
|
-
function requirementOf(read, bakedUrl) {
|
|
16104
|
-
const candidates = bakedUrl === void 0 ? BAKED_CANDIDATES.map((relative) => new URL(relative, import.meta.url)) : [bakedUrl];
|
|
16105
|
-
for (const url of candidates) {
|
|
16106
|
-
let requirement;
|
|
16107
|
-
try {
|
|
16108
|
-
requirement = read(url);
|
|
16109
|
-
} catch {
|
|
16110
|
-
continue;
|
|
16111
|
-
}
|
|
16112
|
-
const version = requirement?.interfaceVersion;
|
|
16113
|
-
if (typeof version === "number" && Number.isInteger(version) && version > 0) return { interfaceVersion: version };
|
|
16114
|
-
}
|
|
16115
|
-
}
|
|
16116
|
-
/**
|
|
16117
|
-
* Ask the loaded base whether it implements the interface generation this artifact was built for.
|
|
16118
|
-
*
|
|
16119
|
-
* @param module - the loaded base module (only `checkInterface` / `readInterfaceRequirement` are
|
|
16120
|
-
* touched, each through a guarded read).
|
|
16121
|
-
* @param bakedUrl - test seam: the bake record to read; production leaves it undefined.
|
|
16122
|
-
* @returns a verdict that is never `incompatible` unless BOTH sides were readable and differ, and
|
|
16123
|
-
* never a throw.
|
|
16124
|
-
*/
|
|
16125
|
-
function interfaceVerdict(module, bakedUrl) {
|
|
16126
|
-
const check = member(module, "checkInterface");
|
|
16127
|
-
const read = member(module, "readInterfaceRequirement");
|
|
16128
|
-
if (typeof check !== "function" || typeof read !== "function") return cannotTell("the loaded base has no interface gate (checkInterface / readInterfaceRequirement missing) — it predates the runtime interface contract, so which generation it implements cannot be told");
|
|
16129
|
-
const requirement = requirementOf(read, bakedUrl);
|
|
16130
|
-
if (requirement === void 0) return cannotTell("this build has no baked interface requirement (lib/interface-version.json is missing or malformed), so the runtime interface gate cannot run");
|
|
16131
|
-
let verdict;
|
|
16132
|
-
try {
|
|
16133
|
-
verdict = check(requirement.interfaceVersion, module);
|
|
16134
|
-
} catch {
|
|
16135
|
-
return cannotTell("the loaded base threw while judging the interface generation");
|
|
16136
|
-
}
|
|
16137
|
-
const status = verdict?.status;
|
|
16138
|
-
if (status !== "ok" && status !== "incompatible" && status !== "cannot-tell") return cannotTell("the loaded base returned no usable interface verdict");
|
|
16139
|
-
return verdict;
|
|
16140
|
-
}
|
|
16141
|
-
//#endregion
|
|
16142
17123
|
//#region lib/types/envinit.js
|
|
16143
17124
|
/**
|
|
16144
17125
|
* Environment initialisation for the memory plugin, through the family base
|
|
@@ -16398,8 +17379,22 @@ function modelItem(framework, model) {
|
|
|
16398
17379
|
policy: { timeoutMs: MODEL_REQUEST_TIMEOUT_MS }
|
|
16399
17380
|
};
|
|
16400
17381
|
}
|
|
16401
|
-
|
|
16402
|
-
|
|
17382
|
+
/**
|
|
17383
|
+
* The plugin-specific halves of the shared loader's sentences: mem says the FRAMEWORK is skipped and
|
|
17384
|
+
* names the legacy provisioning path among the withheld capabilities. The mechanism is the shim's
|
|
17385
|
+
* (`interface_gate.ts`), the wording is this plugin's.
|
|
17386
|
+
*/
|
|
17387
|
+
const DEGRADE_WORDS = {
|
|
17388
|
+
prefix: OWN_PREFIX,
|
|
17389
|
+
loadFailure: "the framework is SKIPPED and the plugin will mount anyway",
|
|
17390
|
+
withheld: "own prompt defaults, gate skipped, legacy provisioning"
|
|
17391
|
+
};
|
|
17392
|
+
/**
|
|
17393
|
+
* The loaded runtime, cached once per process through the shared cache discipline: a FAILED load is
|
|
17394
|
+
* NOT cached, so a profile reload retries instead of replaying the first failure for the whole
|
|
17395
|
+
* process lifetime.
|
|
17396
|
+
*/
|
|
17397
|
+
const cache = createLoadCache((options) => loadGuarded(options, loadOnce, options.log, DEGRADE_WORDS));
|
|
16403
17398
|
/**
|
|
16404
17399
|
* Ensure the framework, then the compatibility base, then dispatch the expensive resources.
|
|
16405
17400
|
*
|
|
@@ -16409,32 +17404,11 @@ let loading;
|
|
|
16409
17404
|
* base still returns a runtime, with `compat: undefined`: the mount must continue, and the resource
|
|
16410
17405
|
* items are still worth dispatching.
|
|
16411
17406
|
*
|
|
16412
|
-
* The result is cached per process; a FAILED load is not, so a profile reload retries instead of
|
|
16413
|
-
* replaying the first failure for the whole process lifetime.
|
|
16414
|
-
*
|
|
16415
17407
|
* @param options - logger, family root, download policy and the test seams.
|
|
16416
17408
|
* @returns the loaded runtime, or `undefined` when the framework itself is unavailable.
|
|
16417
17409
|
*/
|
|
16418
|
-
|
|
16419
|
-
|
|
16420
|
-
if (loading !== void 0) return loading;
|
|
16421
|
-
loading = loadGuarded(options).then((runtime) => {
|
|
16422
|
-
if (runtime === void 0) loading = void 0;
|
|
16423
|
-
else loaded = runtime;
|
|
16424
|
-
return runtime;
|
|
16425
|
-
}, () => {
|
|
16426
|
-
loading = void 0;
|
|
16427
|
-
});
|
|
16428
|
-
return loading;
|
|
16429
|
-
}
|
|
16430
|
-
/** The guard that makes "never throws" true: every failure is a warning plus `undefined`. */
|
|
16431
|
-
async function loadGuarded(options) {
|
|
16432
|
-
try {
|
|
16433
|
-
return await loadOnce(options);
|
|
16434
|
-
} catch (error) {
|
|
16435
|
-
warn(options.log, `${OWN_PREFIX} WARNING — environment initialisation failed (${reasonOf(error)}); the framework is SKIPPED and the plugin will mount anyway`);
|
|
16436
|
-
return;
|
|
16437
|
-
}
|
|
17410
|
+
function loadEnvinit(options = {}) {
|
|
17411
|
+
return cache.load(options);
|
|
16438
17412
|
}
|
|
16439
17413
|
/** The body of {@link loadEnvinit}; the cache above makes it run at most once per process. */
|
|
16440
17414
|
async function loadOnce(options) {
|
|
@@ -16454,14 +17428,7 @@ async function loadOnce(options) {
|
|
|
16454
17428
|
warn(options.log, `${OWN_PREFIX} WARNING — ${FRAMEWORK_PACKAGE} could not be made available; the plugin falls back to its own provisioning and mounts anyway`);
|
|
16455
17429
|
return;
|
|
16456
17430
|
}
|
|
16457
|
-
|
|
16458
|
-
const verdict = interfaceVerdict(framework);
|
|
16459
|
-
if (!baseIsUsable(verdict)) {
|
|
16460
|
-
warn(options.log, `${OWN_PREFIX} WARNING — interface: ${verdict.reason ?? "the loaded base implements another interface generation"}; the base's shared capabilities are NOT used (own prompt defaults, gate skipped, legacy provisioning) and the plugin mounts anyway`);
|
|
16461
|
-
return;
|
|
16462
|
-
}
|
|
16463
|
-
if (verdict.status === "cannot-tell") warn(options.log, `${OWN_PREFIX} WARNING — interface: ${verdict.reason ?? "the interface generation cannot be told"}; using the loaded base anyway ("cannot tell" is never "incompatible")`);
|
|
16464
|
-
}
|
|
17431
|
+
if (!gateOrDegrade(framework, options.log, DEGRADE_WORDS)) return void 0;
|
|
16465
17432
|
const disposers = [];
|
|
16466
17433
|
let compat;
|
|
16467
17434
|
try {
|
|
@@ -16521,8 +17488,7 @@ async function loadOnce(options) {
|
|
|
16521
17488
|
for (const instance of disposers.splice(0)) try {
|
|
16522
17489
|
instance.dispose();
|
|
16523
17490
|
} catch {}
|
|
16524
|
-
|
|
16525
|
-
loading = void 0;
|
|
17491
|
+
cache.clear();
|
|
16526
17492
|
}
|
|
16527
17493
|
};
|
|
16528
17494
|
}
|
|
@@ -16772,37 +17738,43 @@ function openDocumentPath(documentPath, target, configured, deps = {}) {
|
|
|
16772
17738
|
}
|
|
16773
17739
|
//#endregion
|
|
16774
17740
|
//#region lib/types/vectorMigration.js
|
|
17741
|
+
/**
|
|
17742
|
+
* The plugin's lifecycle for the bounded, resumable background migration of persisted vectors into
|
|
17743
|
+
* the current embedding space.
|
|
17744
|
+
*
|
|
17745
|
+
* Changing the embedding space (default model, width, pooling, normalization, input window, weights)
|
|
17746
|
+
* is a DATA MIGRATION, not a config tweak: every persisted vector still decodes, but in another
|
|
17747
|
+
* model's coordinates, so the semantic leg must not serve it. Measured on the real library after the
|
|
17748
|
+
* 512→768 default-model swap, 78 of 80 ACTIVE facts fell out of the semantic leg and retrieval
|
|
17749
|
+
* degraded to lexical+entity with no error anywhere — the stores' open-time warning now says so, and
|
|
17750
|
+
* this module is what repairs it without the operator having to know `vectors --fix` exists.
|
|
17751
|
+
*
|
|
17752
|
+
* The FLOW lives in the engine, once, for both stores (`@avantf/mem`'s `store/vector_repair.ts`:
|
|
17753
|
+
* batch size, event-loop yield, stop / resume / no-progress rules, the `semantic.auto_migrate`
|
|
17754
|
+
* decision and the four store-named log kinds). It is driven for BOTH stores — memory and knowledge
|
|
17755
|
+
* — in one pass, because one embedding space is configured for the whole runtime: the knowledge
|
|
17756
|
+
* store used to be left behind, silently, with only a manual `kb_reindex` able to fix it.
|
|
17757
|
+
*
|
|
17758
|
+
* This module owns only the process LIFECYCLE: one start, one stop, and "never an unhandled
|
|
17759
|
+
* rejection". It sits BEHIND the startup gate (`awaitStartupGate`, see index.ts): the model warm and
|
|
17760
|
+
* this first batch must not land in the boot window (DESIGN §20.14).
|
|
17761
|
+
*
|
|
17762
|
+
* Deliberately no state: what still needs migrating is derived from the databases on every pass —
|
|
17763
|
+
* and the adapters are created ONCE so the shared flow's once-per-process `auto_migrate` warning is
|
|
17764
|
+
* genuinely once per process, not once per heartbeat.
|
|
17765
|
+
*/
|
|
16775
17766
|
function createVectorMigration(opts) {
|
|
16776
17767
|
const { rt, logger } = opts;
|
|
17768
|
+
const targets = [rt.memory.vectorRepairTarget(), rt.knowledge.vectorRepairTarget()];
|
|
16777
17769
|
let running = false;
|
|
16778
17770
|
let stopped = false;
|
|
16779
|
-
let warnedDisabled = false;
|
|
16780
17771
|
const run = async () => {
|
|
16781
|
-
const health = rt.memory.vectorSpaceHealth();
|
|
16782
|
-
const pending = health.stale + health.space_stale;
|
|
16783
|
-
if (pending === 0) return;
|
|
16784
17772
|
if (rt.closed || !opts.isActive()) return;
|
|
16785
|
-
|
|
16786
|
-
if (!warnedDisabled) {
|
|
16787
|
-
warnedDisabled = true;
|
|
16788
|
-
logger.warn(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space and \`semantic.auto_migrate\` is off — the semantic leg stays blind to them; run \`avantf-mem vectors --fix\` (MCP/plugin: \`mem_admin vectors_fix\`) or set \`semantic.auto_migrate: true\``);
|
|
16789
|
-
}
|
|
16790
|
-
return;
|
|
16791
|
-
}
|
|
16792
|
-
const startedAt = Date.now();
|
|
16793
|
-
logger.info(`vector migration: ${String(pending)} ACTIVE vector(s) belong to an older embedding space (${String(health.space_stale)} same-width, another representation, ${String(health.stale)} wrong-width) — re-encoding in the background (a representation change — model, pooling, normalization, input window or weights — migrates the whole corpus once)`);
|
|
16794
|
-
const outcome = await rt.memory.migrateVectors({
|
|
17773
|
+
await driveVectorRepairs(targets, {
|
|
16795
17774
|
...opts.batchSize === void 0 ? {} : { batchSize: opts.batchSize },
|
|
16796
17775
|
shouldStop: () => stopped || rt.closed || !opts.isActive(),
|
|
16797
|
-
|
|
16798
|
-
logger.info(`vector migration: re-encoded ${String(progress.migrated)}/${String(pending)} (${String(progress.remaining)} left)`);
|
|
16799
|
-
}
|
|
17776
|
+
log: logger
|
|
16800
17777
|
});
|
|
16801
|
-
if (outcome.remaining === 0) {
|
|
16802
|
-
logger.info(`vector migration: complete — ${String(outcome.migrated)} vector(s) re-encoded in ${String(Date.now() - startedAt)}ms; the semantic leg is current`);
|
|
16803
|
-
return;
|
|
16804
|
-
}
|
|
16805
|
-
if (!stopped && !rt.closed && opts.isActive()) logger.warn(`vector migration: ${String(outcome.remaining)} vector(s) still belong to an older embedding space (the embedding model may be unavailable) — the next heartbeat retries; manual entry \`avantf-mem vectors --fix\``);
|
|
16806
17778
|
};
|
|
16807
17779
|
return {
|
|
16808
17780
|
start() {
|
|
@@ -17067,15 +18039,7 @@ const ENTITY_SWEEP_PER_HEARTBEAT = 2e3;
|
|
|
17067
18039
|
/** Minimum spacing between corpus drift checks (`tools/result` fires for EVERY tool). */
|
|
17068
18040
|
const RECONCILE_MIN_INTERVAL_MS = 2e3;
|
|
17069
18041
|
async function runToolSpec(spec, rt, args) {
|
|
17070
|
-
|
|
17071
|
-
if (!parsed.success) return validationError(spec.name, parsed.error.issues);
|
|
17072
|
-
if (!supportsToolKey(spec.key)) return toolErr(`unknown tool ${spec.key}`);
|
|
17073
|
-
try {
|
|
17074
|
-
const value = await dispatchToolKey(rt, spec.key, parsed.data);
|
|
17075
|
-
return toolOk(modelFacingToolResult(spec.key, parsed.data.action, value));
|
|
17076
|
-
} catch (error) {
|
|
17077
|
-
return toolErr(error);
|
|
17078
|
-
}
|
|
18042
|
+
return await runToolSpec$1(rt, spec, args);
|
|
17079
18043
|
}
|
|
17080
18044
|
function present(spec, args) {
|
|
17081
18045
|
return {
|
|
@@ -17429,19 +18393,6 @@ function hostLogger(ctx) {
|
|
|
17429
18393
|
};
|
|
17430
18394
|
}
|
|
17431
18395
|
/**
|
|
17432
|
-
* Whether this plugin's fiber is still alive after the environment await.
|
|
17433
|
-
*
|
|
17434
|
-
* The environment gate can take the whole startup budget (15 s on a first run) and the framework may
|
|
17435
|
-
* even fetch the gate's base during it. A profile reload or unload landing inside that window clears
|
|
17436
|
-
* the fiber's uid, after which every Cordis call throws `INACTIVE_EFFECT` — the plugin would then
|
|
17437
|
-
* half-register (the Remote face cannot be rolled back) instead of stopping cleanly. An unknown
|
|
17438
|
-
* shape is "cannot tell", so only an explicitly cleared uid counts as disposed.
|
|
17439
|
-
*/
|
|
17440
|
-
function stillActive(ctx) {
|
|
17441
|
-
const fiber = ctx.fiber;
|
|
17442
|
-
return fiber === void 0 || fiber.uid !== null;
|
|
17443
|
-
}
|
|
17444
|
-
/**
|
|
17445
18396
|
* Declare the expensive resources through the family framework, warm the tokenizer and the embedding
|
|
17446
18397
|
* model, and continue initialisation as each item settles.
|
|
17447
18398
|
*
|
|
@@ -17597,19 +18548,18 @@ async function apply(ctx, config) {
|
|
|
17597
18548
|
});
|
|
17598
18549
|
const heartbeatMinutes = rt.config.common.trust.presence.heartbeat_minutes;
|
|
17599
18550
|
if (heartbeatMinutes > 0) ctx.effect(() => {
|
|
17600
|
-
const
|
|
17601
|
-
|
|
17602
|
-
|
|
17603
|
-
|
|
17604
|
-
|
|
17605
|
-
|
|
17606
|
-
|
|
17607
|
-
|
|
17608
|
-
|
|
17609
|
-
|
|
17610
|
-
if (typeof unref === "function") unref.call(timer);
|
|
18551
|
+
const stop = installTrustHeartbeat({
|
|
18552
|
+
memory: rt.memory,
|
|
18553
|
+
logger,
|
|
18554
|
+
heartbeatMinutes,
|
|
18555
|
+
onBeat: () => {
|
|
18556
|
+
sweepEntities();
|
|
18557
|
+
vectorMigration.start();
|
|
18558
|
+
},
|
|
18559
|
+
tickFailed: (message) => `trust tick failed: ${message}`
|
|
18560
|
+
});
|
|
17611
18561
|
logger.info(`trust heartbeat: every ${heartbeatMinutes}min (active-day clock + lifecycle sweep)`);
|
|
17612
|
-
return
|
|
18562
|
+
return stop;
|
|
17613
18563
|
});
|
|
17614
18564
|
const promptDir = join(rt.config.home, "prompts");
|
|
17615
18565
|
const kit = env?.kit;
|