@avantf/dsh-mem 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -28
- package/cordis.patch.yml +17 -0
- package/lib/client.js +155 -14
- package/lib/engine/mem/config/config_files.d.ts +26 -0
- package/lib/engine/mem/config/domains.d.ts +20 -0
- package/lib/engine/mem/config/loader.d.ts +35 -0
- package/lib/engine/mem/config/paths.d.ts +53 -0
- package/lib/engine/mem/db/chunk.d.ts +26 -0
- package/lib/engine/mem/db/conn.d.ts +13 -0
- package/lib/engine/mem/db/dao/chunks.d.ts +150 -0
- package/lib/engine/mem/db/dao/contradictions.d.ts +98 -0
- package/lib/engine/mem/db/dao/documents.d.ts +51 -0
- package/lib/engine/mem/db/dao/entities.d.ts +67 -0
- package/lib/engine/mem/db/dao/facts.d.ts +472 -0
- package/lib/engine/mem/db/dao/stats.d.ts +14 -0
- package/lib/engine/mem/db/dao/triples.d.ts +80 -0
- package/lib/engine/mem/db/hash.d.ts +2 -0
- package/lib/engine/mem/db/knowledge.d.ts +17 -0
- package/lib/engine/mem/db/migrations.d.ts +71 -0
- package/lib/engine/mem/db/port.d.ts +57 -0
- package/lib/engine/mem/db/schema.d.ts +26 -0
- package/lib/engine/mem/db/sqlite.d.ts +89 -0
- package/lib/engine/mem/db/store.d.ts +35 -0
- package/lib/engine/mem/db/tokenizer.d.ts +61 -0
- package/lib/engine/mem/db/vectors.d.ts +50 -0
- package/lib/engine/mem/dispatch.d.ts +51 -0
- package/lib/engine/mem/entities/extract.d.ts +84 -0
- package/lib/engine/mem/eval/loader.d.ts +19 -0
- package/lib/engine/mem/eval/metrics.d.ts +25 -0
- package/lib/engine/mem/eval/runner.d.ts +10 -0
- package/lib/engine/mem/eval/write_metrics.d.ts +88 -0
- package/lib/engine/mem/git.d.ts +73 -0
- package/lib/engine/mem/hrr/algebra.d.ts +30 -0
- package/lib/engine/mem/hrr/atoms.d.ts +22 -0
- package/lib/engine/mem/hrr/encode.d.ts +32 -0
- package/lib/engine/mem/hrr/index.d.ts +3 -0
- package/lib/engine/mem/index.d.ts +27 -0
- package/lib/engine/mem/lifecycle/contradiction.d.ts +156 -0
- package/lib/engine/mem/lifecycle/index.d.ts +5 -0
- package/lib/engine/mem/lifecycle/maintenance.d.ts +48 -0
- package/lib/engine/mem/lifecycle/presence.d.ts +29 -0
- package/lib/engine/mem/lifecycle/tick.d.ts +43 -0
- package/lib/engine/mem/lifecycle/trust.d.ts +90 -0
- package/lib/engine/mem/modelBootstrap.d.ts +207 -0
- package/lib/engine/mem/router.d.ts +51 -0
- package/lib/engine/mem/runtime.d.ts +232 -0
- package/lib/engine/mem/store/common.d.ts +82 -0
- package/lib/engine/mem/store/doc_files.d.ts +125 -0
- package/lib/engine/mem/store/document_text.d.ts +47 -0
- package/lib/engine/mem/store/floors.d.ts +191 -0
- package/lib/engine/mem/store/hybrid.d.ts +232 -0
- package/lib/engine/mem/store/ingest_guard.d.ts +25 -0
- package/lib/engine/mem/store/knowledge.d.ts +483 -0
- package/lib/engine/mem/store/lexical.d.ts +67 -0
- package/lib/engine/mem/store/memory.d.ts +543 -0
- package/lib/engine/mem/store/source_picker.d.ts +43 -0
- package/lib/engine/mem-contract/config.d.ts +129 -0
- package/lib/engine/mem-contract/env.d.ts +31 -0
- package/lib/engine/mem-contract/family.d.ts +25 -0
- package/lib/engine/mem-contract/index.d.ts +8 -0
- package/lib/engine/mem-contract/log.d.ts +29 -0
- package/lib/engine/mem-contract/remote.d.ts +80 -0
- package/lib/engine/mem-contract/text.d.ts +92 -0
- package/lib/engine/mem-contract/tools.d.ts +284 -0
- package/lib/engine/mem-contract/types.d.ts +615 -0
- package/lib/engine/mem-retrieval/adapters/auto_vstore.d.ts +47 -0
- package/lib/engine/mem-retrieval/adapters/hnswlib.d.ts +174 -0
- package/lib/engine/mem-retrieval/adapters/local_bge.d.ts +121 -0
- package/lib/engine/mem-retrieval/adapters/local_numpy.d.ts +28 -0
- package/lib/engine/mem-retrieval/adapters/local_reranker.d.ts +52 -0
- package/lib/engine/mem-retrieval/adapters/none_reranker.d.ts +10 -0
- package/lib/engine/mem-retrieval/adapters/warm_gate.d.ts +46 -0
- package/lib/engine/mem-retrieval/budget.d.ts +30 -0
- package/lib/engine/mem-retrieval/fusion.d.ts +53 -0
- package/lib/engine/mem-retrieval/index.d.ts +14 -0
- package/lib/engine/mem-retrieval/interfaces.d.ts +82 -0
- package/lib/engine/mem-retrieval/log.d.ts +11 -0
- package/lib/engine/mem-retrieval/registry.d.ts +23 -0
- package/lib/engine/mem-retrieval/rerank.d.ts +13 -0
- package/lib/engine/mem-retrieval/stats.d.ts +64 -0
- package/lib/engine/mem-retrieval/text_budget.d.ts +52 -0
- package/lib/index.js +1773 -473
- package/lib/interface-version.json +2 -2
- package/lib/types/client/domains.d.ts +5 -4
- package/lib/types/client/domains.d.ts.map +1 -1
- package/lib/types/client/index.d.ts.map +1 -1
- package/lib/types/client/ingest.d.ts +1 -1
- package/lib/types/client/wire.d.ts +36 -0
- package/lib/types/client/wire.d.ts.map +1 -0
- package/lib/types/envinit-bootstrap.d.ts +3 -2
- package/lib/types/envinit.d.ts +2 -19
- package/lib/types/envinit.d.ts.map +1 -1
- package/lib/types/hints.d.ts +1 -1
- package/lib/types/index.d.ts +8 -3
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/open.d.ts +61 -11
- package/lib/types/open.d.ts.map +1 -1
- package/lib/types/prompt.d.ts +19 -1
- package/lib/types/prompt.d.ts.map +1 -1
- package/lib/types/provision.d.ts +1 -1
- package/lib/types/provision.d.ts.map +1 -1
- package/lib/types/reconcile.d.ts +64 -0
- package/lib/types/reconcile.d.ts.map +1 -0
- package/lib/types/remote.d.ts +53 -0
- package/lib/types/remote.d.ts.map +1 -1
- package/lib/types/render.d.ts +35 -0
- package/lib/types/render.d.ts.map +1 -0
- package/lib/types/tool_schema.d.ts +3 -13
- package/lib/types/tool_schema.d.ts.map +1 -1
- package/package.json +24 -15
- package/lib/client.js.map +0 -1
- package/lib/types/client/domains.js +0 -25
- package/lib/types/client/domains.js.map +0 -1
- package/lib/types/client/index.js +0 -1371
- package/lib/types/client/index.js.map +0 -1
- package/lib/types/client/ingest.js +0 -30
- package/lib/types/client/ingest.js.map +0 -1
- package/lib/types/client/paging.js +0 -36
- package/lib/types/client/paging.js.map +0 -1
- package/lib/types/client/summarize.js +0 -79
- package/lib/types/client/summarize.js.map +0 -1
- package/lib/types/data_home.js +0 -26
- package/lib/types/data_home.js.map +0 -1
- package/lib/types/envinit-bootstrap.js +0 -333
- package/lib/types/envinit.js +0 -402
- package/lib/types/envinit.js.map +0 -1
- package/lib/types/hints.js +0 -53
- package/lib/types/hints.js.map +0 -1
- package/lib/types/index.js +0 -755
- package/lib/types/index.js.map +0 -1
- package/lib/types/inject.js +0 -14
- package/lib/types/inject.js.map +0 -1
- package/lib/types/interface_gate.js +0 -84
- package/lib/types/interface_gate.js.map +0 -1
- package/lib/types/open.js +0 -133
- package/lib/types/open.js.map +0 -1
- package/lib/types/prompt.js +0 -148
- package/lib/types/prompt.js.map +0 -1
- package/lib/types/provision.js +0 -177
- package/lib/types/provision.js.map +0 -1
- package/lib/types/remote.js +0 -247
- package/lib/types/remote.js.map +0 -1
- package/lib/types/tool_schema.js +0 -149
- package/lib/types/tool_schema.js.map +0 -1
package/lib/index.js
CHANGED
|
@@ -12,7 +12,7 @@ import { Readable } from "node:stream";
|
|
|
12
12
|
import { pipeline } from "node:stream/promises";
|
|
13
13
|
import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
14
14
|
import { createInflateRaw } from "node:zlib";
|
|
15
|
-
import
|
|
15
|
+
import { readdir } from "node:fs/promises";
|
|
16
16
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
17
17
|
//#region ../contract/lib/types.js
|
|
18
18
|
/** Shared domain types for avantf-mem. Single source of truth in @avantf/mem-contract. */
|
|
@@ -66,6 +66,27 @@ function withoutRetentionDiagnostics(value) {
|
|
|
66
66
|
return project(value);
|
|
67
67
|
}
|
|
68
68
|
/**
|
|
69
|
+
* The floor profiles a retrieval CALL may ask for. There is deliberately no numeric per-call
|
|
70
|
+
* override: the two profiles are named policies, and their values live in ONE place
|
|
71
|
+
* (`@avantf/mem-retrieval`'s `store/floors.ts`), so a caller cannot invent a third calibration.
|
|
72
|
+
*
|
|
73
|
+
* - **omitted** — the default policy: the configured (strict) floors, and if they empty the result
|
|
74
|
+
* while having dropped candidates, ONE relaxed pass over the absolute bottom line before
|
|
75
|
+
* answering. `RecallResult.relaxed` marks that second pass.
|
|
76
|
+
* - **`strict`** — the configured floors with NO fallback. This is what the UI's 严格 mode sends, so
|
|
77
|
+
* the panel can show the honest strict outcome plus how many candidates the floors removed.
|
|
78
|
+
* - **`loose`** — the relaxed floors outright (never "no floors": it still answers "nothing
|
|
79
|
+
* relevant" for an unrelated question).
|
|
80
|
+
*
|
|
81
|
+
* Why a second pass exists at all. The floors are calibrated for paraphrase-style relevance, and
|
|
82
|
+
* there is a band where no threshold separates "answers the question" from "unrelated": measured on
|
|
83
|
+
* the live store, the fact answering 「我是谁」 scored cosine 0.444 while four unrelated queries
|
|
84
|
+
* topped out at 0.384 — a 0.06 margin, and the answering fact was the query's own top-1. A relaxed
|
|
85
|
+
* PASS is bounded (it runs only on an empty strict result and keeps its own floor); a lower default,
|
|
86
|
+
* or exempting the user's `pinned` archive from the floors, trades precision on every query.
|
|
87
|
+
*/
|
|
88
|
+
const FLOOR_PROFILES = ["strict", "loose"];
|
|
89
|
+
/**
|
|
69
90
|
* Weights reported when the semantic leg is unavailable (FTS+entity rebalance).
|
|
70
91
|
* Single source for both stores and the cross-retrieval router — previously the
|
|
71
92
|
* same literal lived in three files and could silently desync.
|
|
@@ -115,13 +136,7 @@ const vectorStoreSchema = z.object({
|
|
|
115
136
|
* sub-millisecond. At 8000 recall falls to 0.89, so for a larger corpus raise
|
|
116
137
|
* `hnswlib_ef_search` (or accept the loss) instead of moving this threshold.
|
|
117
138
|
*/
|
|
118
|
-
auto_thresholds: z.object({
|
|
119
|
-
hnswlib: z.number().int().positive().default(2e3),
|
|
120
|
-
faiss: z.number().int().positive().default(1e5)
|
|
121
|
-
}).default({
|
|
122
|
-
hnswlib: 2e3,
|
|
123
|
-
faiss: 1e5
|
|
124
|
-
}),
|
|
139
|
+
auto_thresholds: z.object({ hnswlib: z.number().int().positive().default(2e3) }).default({ hnswlib: 2e3 }),
|
|
125
140
|
/**
|
|
126
141
|
* ANN search beam width (`ef`). This is the recall/speed knob of the hnswlib index, and it is
|
|
127
142
|
* NOT optional in practice: the library's own default (10) returned 0.45 of the true top-10 in
|
|
@@ -144,6 +159,33 @@ const retrieverSchema = z.object({
|
|
|
144
159
|
max_output_tokens: z.number().int().nonnegative().default(8e3),
|
|
145
160
|
over_fetch_factor: z.number().int().positive().default(5),
|
|
146
161
|
/**
|
|
162
|
+
* Relevance floor for the SEMANTIC leg, in cosine units (`0` = off).
|
|
163
|
+
*
|
|
164
|
+
* Applied to the leg's RAW score BEFORE `fuse()`: fusion scales each leg by its own maximum, so
|
|
165
|
+
* the fused number is relative to the query and cannot carry an absolute cutoff (DESIGN §7 /
|
|
166
|
+
* §20.19). A score EQUAL to the floor is kept — only strictly-lower candidates are dropped.
|
|
167
|
+
*
|
|
168
|
+
* The default is calibrated for the shipped embedder (`Xenova/bge-small-zh-v1.5`, dim 512,
|
|
169
|
+
* mean-pooled AND normalized). A different model is a different cosine scale: re-measure before
|
|
170
|
+
* keeping 0.5 (see DESIGN §20.19).
|
|
171
|
+
*/
|
|
172
|
+
min_semantic_similarity: z.number().min(0).max(1).default(.5),
|
|
173
|
+
/**
|
|
174
|
+
* Relevance floor for the FTS leg: the minimum number of DISTINCT query terms a row must hit
|
|
175
|
+
* (`0` = off). Judged PER ROW — the terms are `store/lexical.ts`'s `relevanceTerms()` (latin words
|
|
176
|
+
* ≥ 5 chars + every CJK 3-gram), and a row that hits fewer is dropped.
|
|
177
|
+
*
|
|
178
|
+
* `bm25` itself is unbounded and query-relative, so it cannot host an absolute floor. With the
|
|
179
|
+
* semantic leg unavailable this value is relaxed to 1 by the orchestration (FTS + entity are then
|
|
180
|
+
* the only evidence a short query has); a configured `0` stays off either way.
|
|
181
|
+
*/
|
|
182
|
+
min_fts_terms: z.number().int().nonnegative().default(2),
|
|
183
|
+
/**
|
|
184
|
+
* Relevance floor for the entity-overlap (Jaccard) leg, as a ratio of the union (`0` = off).
|
|
185
|
+
* A score EQUAL to the floor is kept.
|
|
186
|
+
*/
|
|
187
|
+
min_jaccard: z.number().min(0).max(1).default(.2),
|
|
188
|
+
/**
|
|
147
189
|
* Rows one non-semantic retrieval leg may hand to fusion (`0` = derived: `max(200, 4×overFetch)`).
|
|
148
190
|
*
|
|
149
191
|
* The legs used to return the whole matching corpus — a common phrase matched every fact — and
|
|
@@ -236,7 +278,7 @@ const toolsSchema = z.object({
|
|
|
236
278
|
});
|
|
237
279
|
/** The merged runtime config shared by memory and knowledge stores. */
|
|
238
280
|
const ConfigSchema = z.object({
|
|
239
|
-
dataHome: z.string().default("~/.avantf"),
|
|
281
|
+
dataHome: z.string().default("~/.avantf").describe("数据根的配置层载体(解析次序的第 ② 层):只承载 profile 传进来的配置值——`resolveDataHome` 在合并 `configs/common.yaml` 之前就要定根,所以这个文件里的同一键**不参与**解析。属历史遗留键,保留是为了让既有配置继续可用(见 AGENTS.md「边界与路径」)。"),
|
|
240
282
|
semantic: semanticSchema.prefault({}),
|
|
241
283
|
rerank: rerankSchema.prefault({}),
|
|
242
284
|
vectorStore: vectorStoreSchema.prefault({}),
|
|
@@ -324,6 +366,56 @@ const KnowledgeConfigSchema = z.object({
|
|
|
324
366
|
ingest: ingestSchema.prefault({})
|
|
325
367
|
});
|
|
326
368
|
//#endregion
|
|
369
|
+
//#region ../contract/lib/env.js
|
|
370
|
+
/**
|
|
371
|
+
* Tiny env/path helpers shared by every package — the single source of truth for
|
|
372
|
+
* `~/` expansion and boolean env parsing (previously duplicated in core paths,
|
|
373
|
+
* both model adapters, and the config loader).
|
|
374
|
+
*/
|
|
375
|
+
/** Expand `~/…` (and a BARE `~`) against the USER home directory.
|
|
376
|
+
*
|
|
377
|
+
* The bare form matters: `AVANTF_HOME=~` used to come back verbatim, i.e. as the RELATIVE path `~`,
|
|
378
|
+
* and everything built on it was written into the current working directory. The base kit's own
|
|
379
|
+
* `expandHome` always expanded it, so the two copies disagreed on exactly the input a shell makes
|
|
380
|
+
* easy to type. */
|
|
381
|
+
function expandHome$1(p) {
|
|
382
|
+
if (p === "~") return homedir();
|
|
383
|
+
return p.startsWith("~/") ? join(homedir(), p.slice(2)) : p;
|
|
384
|
+
}
|
|
385
|
+
/** Parse a boolean env var: `0`/`false` → false, unset/empty → undefined, else true. */
|
|
386
|
+
function envFlag(name) {
|
|
387
|
+
const raw = process.env[name];
|
|
388
|
+
if (raw === void 0 || raw === "") return void 0;
|
|
389
|
+
return raw !== "0" && raw.toLowerCase() !== "false";
|
|
390
|
+
}
|
|
391
|
+
/**
|
|
392
|
+
* The global model-download kill switch.
|
|
393
|
+
*
|
|
394
|
+
* TWO switches, either of which turns downloads off:
|
|
395
|
+
*
|
|
396
|
+
* - `AVANTF_MEM_AUTO_DOWNLOAD` — this project's switch (CLI, MCP server, tests);
|
|
397
|
+
* - `AVANTF_ENVINIT_AUTO_DOWNLOAD` — the family framework's switch, a MASTER gate: the operator
|
|
398
|
+
* turns the whole family's provisioning off in one place, so the engine must stop fetching too
|
|
399
|
+
* (otherwise the framework skips its item while the runtime silently downloads the same model).
|
|
400
|
+
*
|
|
401
|
+
* Deliberately honored in the config loader (env layer ④, so `rt.config` reflects reality) and in
|
|
402
|
+
* the model adapters themselves, so even a directly constructed backend (tests, third-party
|
|
403
|
+
* embedders) can never reach the network. As an ops/test kill switch it intentionally overrides the
|
|
404
|
+
* configured/passed `autoDownload`.
|
|
405
|
+
*
|
|
406
|
+
* Three-valued on purpose: either switch `0`/`false` → `false`; the project switch `1`/`true` →
|
|
407
|
+
* `true`; otherwise (including the family switch merely being `1`, which only means "the family did
|
|
408
|
+
* not disable anything") → `undefined`, so the configured value stands. The family switch is a
|
|
409
|
+
* master GATE, not a force-on: it must be able to stop the runtime, never to override
|
|
410
|
+
* `semantic.auto_download: false`.
|
|
411
|
+
*/
|
|
412
|
+
function envAutoDownload() {
|
|
413
|
+
const project = envFlag("AVANTF_MEM_AUTO_DOWNLOAD");
|
|
414
|
+
const family = envFlag("AVANTF_ENVINIT_AUTO_DOWNLOAD");
|
|
415
|
+
if (project === false || family === false) return false;
|
|
416
|
+
return project;
|
|
417
|
+
}
|
|
418
|
+
//#endregion
|
|
327
419
|
//#region ../contract/lib/family.js
|
|
328
420
|
/**
|
|
329
421
|
* The family root and the two managed directories under it — ONE definition of a convention that
|
|
@@ -334,16 +426,20 @@ const KnowledgeConfigSchema = z.object({
|
|
|
334
426
|
* (pandoc), `models` the downloaded embedding/rerank weights in the flat layout the transformers.js
|
|
335
427
|
* cache reads. The engine's built-in defaults point HERE, not at the pre-framework
|
|
336
428
|
* `~/.avantf/{tools,models}`: those legacy directories are no longer a fallback anywhere, so a
|
|
337
|
-
* machine that never creates them loses nothing, and no process resolves a path that only
|
|
429
|
+
* machine that never creates them loses nothing, and no process resolves a path that only missions
|
|
338
430
|
* because somebody left a compatibility symlink behind.
|
|
339
431
|
*
|
|
340
432
|
* `provision` mirrors {@link familyHome} (it is deliberately dependency-free, so it cannot import
|
|
341
433
|
* this module); `packages/core/test/family_paths.spec.ts` pins the two copies together so the
|
|
342
434
|
* convention cannot drift.
|
|
435
|
+
*
|
|
436
|
+
* The `~/` expansion is NOT a private copy any more: this module and `env.ts` used to carry two
|
|
437
|
+
* literally identical `expandTilde`/`expandHome` bodies in the same package, and only the exported
|
|
438
|
+
* one was pinned — so the un-pinned copy could drift silently. It now calls {@link expandHome}.
|
|
343
439
|
*/
|
|
344
440
|
function familyHome$1(env = process.env) {
|
|
345
441
|
const configured = env["AVANTF_HOME"]?.trim() ?? "";
|
|
346
|
-
return configured !== "" ?
|
|
442
|
+
return configured !== "" ? expandHome$1(configured) : join(homedir(), ".avantf", "env");
|
|
347
443
|
}
|
|
348
444
|
/** The managed external-binary root (`<family root>/tools`). */
|
|
349
445
|
function familyToolsDir$1(env = process.env) {
|
|
@@ -353,11 +449,6 @@ function familyToolsDir$1(env = process.env) {
|
|
|
353
449
|
function familyModelsDir(env = process.env) {
|
|
354
450
|
return join(familyHome$1(env), "models");
|
|
355
451
|
}
|
|
356
|
-
/** `~/x` → `<home>/x`, `~` → `<home>`; anything else is used as given. */
|
|
357
|
-
function expandTilde(path) {
|
|
358
|
-
if (path === "~") return homedir();
|
|
359
|
-
return path.startsWith("~/") ? join(homedir(), path.slice(2)) : path;
|
|
360
|
-
}
|
|
361
452
|
//#endregion
|
|
362
453
|
//#region ../contract/lib/tools.js
|
|
363
454
|
/**
|
|
@@ -384,6 +475,13 @@ const QUERY_KINDS = [
|
|
|
384
475
|
"doc_chunk"
|
|
385
476
|
];
|
|
386
477
|
/**
|
|
478
|
+
* The one description both retrieval entry points use for `floors`.
|
|
479
|
+
*
|
|
480
|
+
* Written once because `mem_recall.search` and `kb_query` accept the same override, and a second
|
|
481
|
+
* copy is how the two surfaces drift (the model then reads two different meanings for one field).
|
|
482
|
+
*/
|
|
483
|
+
const FLOORS_FIELD_DESCRIPTION = "相关性门槛档位(可选)。不传=默认策略:严格门槛,严格门槛一条都没命中而确有条目被门槛丢弃时自动再跑一次宽松门槛;strict=只用严格门槛,无自动放宽;loose=直接用宽松门槛(仍有绝对底线,代词式提问如「我是谁」「我司」属于这一类)。";
|
|
484
|
+
/**
|
|
387
485
|
* The longest retrieval query the contract accepts, in characters.
|
|
388
486
|
*
|
|
389
487
|
* There is no useful query beyond this, and there IS a cost: a CJK query is expanded into one trigram
|
|
@@ -465,7 +563,8 @@ const RecallUnion = z.discriminatedUnion("action", [
|
|
|
465
563
|
query: z.string().min(1).max(MAX_QUERY_CHARS, QUERY_TOO_LONG).describe("检索文本。action=search 时必填。"),
|
|
466
564
|
category: z.string().optional().describe("限定分类;可选。"),
|
|
467
565
|
limit: z.number().int().positive().max(50).optional().describe("返回条数上限(1-50);可选。"),
|
|
468
|
-
max_tokens: z.number().int().nonnegative().optional().describe("本次结果的总 token 上限;0=不限制。缺省用配置 retrieval.max_output_tokens。")
|
|
566
|
+
max_tokens: z.number().int().nonnegative().optional().describe("本次结果的总 token 上限;0=不限制。缺省用配置 retrieval.max_output_tokens。"),
|
|
567
|
+
floors: z.enum(FLOOR_PROFILES).optional().describe(FLOORS_FIELD_DESCRIPTION)
|
|
469
568
|
}),
|
|
470
569
|
z.object({
|
|
471
570
|
action: z.literal("ask"),
|
|
@@ -603,7 +702,8 @@ const QueryUnion = z.object({
|
|
|
603
702
|
max_tokens: z.number().int().nonnegative().optional().describe("本次结果的总 token 上限;0=不限制。缺省用配置 retrieval.max_output_tokens。"),
|
|
604
703
|
domain: z.string().optional().describe("限定知识域;可选。"),
|
|
605
704
|
source: z.string().optional().describe("限定来源;可选。"),
|
|
606
|
-
limit: z.number().int().positive().max(50).default(10).describe("返回条数上限(1-50);可选,默认 10。")
|
|
705
|
+
limit: z.number().int().positive().max(50).default(10).describe("返回条数上限(1-50);可选,默认 10。"),
|
|
706
|
+
floors: z.enum(FLOOR_PROFILES).optional().describe(FLOORS_FIELD_DESCRIPTION)
|
|
607
707
|
});
|
|
608
708
|
const REMEMBER_TOOL = {
|
|
609
709
|
key: "remember",
|
|
@@ -723,6 +823,14 @@ function mergedFieldDescription(base, requiredIn) {
|
|
|
723
823
|
* `unwrapRemoteEnvelope` peels both, in order, and also tolerates a bare payload
|
|
724
824
|
* (a gateway that returns raw values, or a future protocol change), so the pages
|
|
725
825
|
* never render an envelope as if it were data.
|
|
826
|
+
*
|
|
827
|
+
* The application envelope also carries the **wire revision** (`wire`, optional): the host stamps
|
|
828
|
+
* the revision of the Remote FACE it publishes onto every answer (see `@avantf/dsh-mem`'s
|
|
829
|
+
* `remote.ts` `WIRE_VERSION`), and the client half reads it back to tell a rebuilt browser bundle
|
|
830
|
+
* from the host process that `dsh web` loaded before it. It rides the envelope rather than one
|
|
831
|
+
* method's payload because this plugin has no single entry point — each tab makes its own first
|
|
832
|
+
* call — while the envelope is the one object every answer shares. Absent means "a host older
|
|
833
|
+
* than the marker": still readable, but the client may not assume a method it knows about exists.
|
|
726
834
|
*/
|
|
727
835
|
function isRecord(value) {
|
|
728
836
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
@@ -854,54 +962,146 @@ function errorCode(error) {
|
|
|
854
962
|
}
|
|
855
963
|
}
|
|
856
964
|
//#endregion
|
|
857
|
-
//#region ../contract/lib/
|
|
965
|
+
//#region ../contract/lib/text.js
|
|
966
|
+
/**
|
|
967
|
+
* Well-formedness of every string mem stores or shows a model — the mem-side MIRROR.
|
|
968
|
+
*
|
|
969
|
+
* The defect this module exists for is ONE shape: a lone surrogate — a UTF-16 code unit in the
|
|
970
|
+
* `D800–DFFF` range with no partner. A string carrying one is not a valid Unicode scalar sequence,
|
|
971
|
+
* and `JSON.stringify` happily emits it as the escape `"\ud800"`: JavaScript's own `JSON.parse`
|
|
972
|
+
* accepts that, so the defect passes every check inside this repository, while a strict parser
|
|
973
|
+
* (Rust's `serde_json`, JSON Schema validators, most non-JS consumers) rejects the whole document.
|
|
974
|
+
* The same half-code-unit also poisons FTS, embeddings and the UI. `String.prototype.toWellFormed`
|
|
975
|
+
* (Node ≥20, present in this repo's Node ≥22.15) replaces each lone surrogate with U+FFFD, which is
|
|
976
|
+
* exactly the repair wanted.
|
|
977
|
+
*
|
|
978
|
+
* ── THE ORIGINAL IS THE BASE KIT ──────────────────────────────────────────────────────────────
|
|
979
|
+
*
|
|
980
|
+
* `base/plugin-base/src/kit/wellformed.ts` owns `wellFormedText` / `wellFormedDeep` (interface v2)
|
|
981
|
+
* and is the copy ONE base release fixes for the whole family. This module is its **deliberate
|
|
982
|
+
* mirror**, in the same sense as `./family.ts` (whose original is the base's `kit/family.ts`): the
|
|
983
|
+
* engine (`@avantf/mem`) and the CLI / MCP faces have **no DSH host and never load the base**
|
|
984
|
+
* (`AGENTS.md`), so they cannot reach the original and keep a dependency-free copy as their
|
|
985
|
+
* documented degradation path. The cross-tree test
|
|
986
|
+
* `mem/packages/plugin/test/wellformed_pin.spec.ts` takes the REAL linked base and compares it with
|
|
987
|
+
* this mirror on one set of samples (the precedent is `family_pin.spec.ts`); **do not change one
|
|
988
|
+
* side without the other**.
|
|
989
|
+
*
|
|
990
|
+
* ── WHO RUNS: the base when a host installed it, the mirror otherwise ─────────────────────────
|
|
991
|
+
*
|
|
992
|
+
* This module is also the ONE place the DSH plugin can hand the base's implementation in:
|
|
993
|
+
* {@link adoptWellFormed} takes the module the bootstrap loaded and swaps the two functions for the
|
|
994
|
+
* base's, falling back per member when it does not carry them. The switch is deliberately
|
|
995
|
+
* **"does the loaded module HAVE the function"** (a `typeof` read of each member, guarded against a
|
|
996
|
+
* hostile getter) — never an interface-generation check, so a base that predates the v2 helpers
|
|
997
|
+
* (generation v1) simply keeps the mirror and the plugin still mounts completely. Nothing here can
|
|
998
|
+
* throw or refuse a mount.
|
|
999
|
+
*
|
|
1000
|
+
* Two directions, two functions, deliberately NOT one:
|
|
1001
|
+
*
|
|
1002
|
+
* - {@link toWellFormedText} is the WRITE side. It additionally applies `normalize('NFC')`, because
|
|
1003
|
+
* Unicode equivalence is a persistence concern: two spellings of the same text must not become
|
|
1004
|
+
* two rows, two FTS terms or two entities. Every store write and every derived string (entity
|
|
1005
|
+
* names, SPO slots) goes through it BEFORE it is persisted. **NFC is mem's policy, not the
|
|
1006
|
+
* well-formed layer's**: the base's `wellFormedText` does NOT normalize, and when it is adopted
|
|
1007
|
+
* the write wrapper here is exactly `base.wellFormedText(value).normalize('NFC')`.
|
|
1008
|
+
* - {@link toWellFormedDeep} is the READ side. It never changes anything except lone surrogates,
|
|
1009
|
+
* and it walks the result value so that data this process did not just write — a row written by
|
|
1010
|
+
* an older build or a foreign driver, a string the tokenizer derived — still serializes to JSON
|
|
1011
|
+
* every strict parser accepts. Applying NFC here would silently rewrite history for display, so
|
|
1012
|
+
* it does not.
|
|
1013
|
+
*
|
|
1014
|
+
* @module contract/text
|
|
1015
|
+
*/
|
|
1016
|
+
/**
|
|
1017
|
+
* The engine's own well-formed layer, and the default: `String.prototype.toWellFormed()` — the
|
|
1018
|
+
* exact call this module has always made (Node ≥20; the repo floor is ≥22.15).
|
|
1019
|
+
*/
|
|
1020
|
+
const nativeText = (value) => value.toWellFormed();
|
|
1021
|
+
/** The well-formed layer in force. {@link adoptWellFormed} replaces it with the base's. */
|
|
1022
|
+
let repairText = nativeText;
|
|
1023
|
+
/**
|
|
1024
|
+
* The local recursion behind {@link toWellFormedDeep} — the mirror's read side.
|
|
1025
|
+
*
|
|
1026
|
+
* Recursion rules, and they are deliberately narrow: only strings are touched
|
|
1027
|
+
* ({@link repairText} — no NFC, see the module note), arrays are mapped, and objects that carry no
|
|
1028
|
+
* `toJSON` are rebuilt with well-formed keys and values. Everything else — numbers, booleans,
|
|
1029
|
+
* `null`, `undefined`, functions, symbols — is returned as-is, so this can never change a value's
|
|
1030
|
+
* JSON meaning. An object WITH a `toJSON` method (`Date`, `Buffer`, `RegExp`, …) is handed back
|
|
1031
|
+
* untouched, because `JSON.stringify` itself would serialize it through that method; cloning its
|
|
1032
|
+
* enumerable own properties instead would turn a `Date` into `{}`.
|
|
1033
|
+
*
|
|
1034
|
+
* Object KEYS are normalized too: a lone surrogate in a property name is emitted verbatim by
|
|
1035
|
+
* `JSON.stringify` (the replacer form of this fix cannot repair it, which is why this clones),
|
|
1036
|
+
* and it breaks a strict parser just the same.
|
|
1037
|
+
*
|
|
1038
|
+
* It reads {@link repairText} at every call, so a host that installs a well-formed string layer
|
|
1039
|
+
* without a recursive one still gets it used throughout the walk.
|
|
1040
|
+
*/
|
|
1041
|
+
function mirrorDeep(value) {
|
|
1042
|
+
if (typeof value === "string") return repairText(value);
|
|
1043
|
+
if (Array.isArray(value)) return value.map((item) => mirrorDeep(item));
|
|
1044
|
+
if (value !== null && typeof value === "object" && typeof value.toJSON !== "function") {
|
|
1045
|
+
const out = {};
|
|
1046
|
+
for (const [key, item] of Object.entries(value)) out[repairText(key)] = mirrorDeep(item);
|
|
1047
|
+
return out;
|
|
1048
|
+
}
|
|
1049
|
+
return value;
|
|
1050
|
+
}
|
|
1051
|
+
/** The recursive repair in force. {@link adoptWellFormed} replaces it with the base's. */
|
|
1052
|
+
let repairDeep = mirrorDeep;
|
|
858
1053
|
/**
|
|
859
|
-
*
|
|
860
|
-
*
|
|
861
|
-
*
|
|
1054
|
+
* The ONE write-side normalization: well-formed, and NFC.
|
|
1055
|
+
*
|
|
1056
|
+
* Called at the store write entries (never at each individual string field), so a fix here lands
|
|
1057
|
+
* everywhere at once.
|
|
862
1058
|
*/
|
|
863
|
-
|
|
1059
|
+
function toWellFormedText(value) {
|
|
1060
|
+
return repairText(value).normalize("NFC");
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* The read-side twin: a structurally identical value in which every string is well-formed.
|
|
864
1064
|
*
|
|
865
|
-
*
|
|
866
|
-
*
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
if (p === "~") return homedir();
|
|
871
|
-
return p.startsWith("~/") ? join(homedir(), p.slice(2)) : p;
|
|
1065
|
+
* Delegates to {@link repairDeep}: the local {@link mirrorDeep} until a host installs the base's
|
|
1066
|
+
* `wellFormedDeep` through {@link adoptWellFormed}, which is the same walk (see the module note).
|
|
1067
|
+
*/
|
|
1068
|
+
function toWellFormedDeep(value) {
|
|
1069
|
+
return repairDeep(value);
|
|
872
1070
|
}
|
|
873
|
-
/**
|
|
874
|
-
function
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
1071
|
+
/** Read one member without letting a hostile module's getter escape. */
|
|
1072
|
+
function member$1(module, name) {
|
|
1073
|
+
try {
|
|
1074
|
+
return module?.[name];
|
|
1075
|
+
} catch {
|
|
1076
|
+
return;
|
|
1077
|
+
}
|
|
878
1078
|
}
|
|
879
1079
|
/**
|
|
880
|
-
*
|
|
1080
|
+
* Prefer the loaded base's well-formed helpers; fall back to this mirror for whatever it lacks.
|
|
881
1081
|
*
|
|
882
|
-
*
|
|
1082
|
+
* The plugin calls this ONCE at mount with the module the bootstrap loaded (`env?.kit`). The
|
|
1083
|
+
* decision per member is a `typeof` check — the v2 members are an OPTIONAL capability, and a base
|
|
1084
|
+
* that predates them (interface v1) or a module with none of them leaves both repairs on the mirror.
|
|
1085
|
+
* It is total: a hostile module reads as "reports none", nothing throws, and a mount is never
|
|
1086
|
+
* refused or changed by the outcome.
|
|
883
1087
|
*
|
|
884
|
-
*
|
|
885
|
-
*
|
|
886
|
-
* turns the whole family's provisioning off in one place, so the engine must stop fetching too
|
|
887
|
-
* (otherwise the framework skips its item while the runtime silently downloads the same model).
|
|
888
|
-
*
|
|
889
|
-
* Deliberately honored in the config loader (env layer ④, so `rt.config` reflects reality) and in
|
|
890
|
-
* the model adapters themselves, so even a directly constructed backend (tests, third-party
|
|
891
|
-
* embedders) can never reach the network. As an ops/test kill switch it intentionally overrides the
|
|
892
|
-
* configured/passed `autoDownload`.
|
|
1088
|
+
* Calling it AGAIN replaces whatever was installed before, so passing `undefined` restores the
|
|
1089
|
+
* mirror (the CLI / MCP faces never call it at all and keep the mirror for their whole life).
|
|
893
1090
|
*
|
|
894
|
-
*
|
|
895
|
-
*
|
|
896
|
-
* not disable anything") → `undefined`, so the configured value stands. The family switch is a
|
|
897
|
-
* master GATE, not a force-on: it must be able to stop the runtime, never to override
|
|
898
|
-
* `semantic.auto_download: false`.
|
|
1091
|
+
* @param kit - the loaded base module (or anything else; only the two members are read).
|
|
1092
|
+
* @returns which repair each side came from, for the caller's log line.
|
|
899
1093
|
*/
|
|
900
|
-
function
|
|
901
|
-
const
|
|
902
|
-
const
|
|
903
|
-
|
|
904
|
-
|
|
1094
|
+
function adoptWellFormed(kit) {
|
|
1095
|
+
const kitText = member$1(kit, "wellFormedText");
|
|
1096
|
+
const kitDeep = member$1(kit, "wellFormedDeep");
|
|
1097
|
+
const text = typeof kitText === "function";
|
|
1098
|
+
const deep = typeof kitDeep === "function";
|
|
1099
|
+
repairText = text ? kitText : nativeText;
|
|
1100
|
+
repairDeep = deep ? kitDeep : mirrorDeep;
|
|
1101
|
+
return {
|
|
1102
|
+
text,
|
|
1103
|
+
deep
|
|
1104
|
+
};
|
|
905
1105
|
}
|
|
906
1106
|
//#endregion
|
|
907
1107
|
//#region lib/types/prompt.js
|
|
@@ -947,7 +1147,7 @@ function envAutoDownload() {
|
|
|
947
1147
|
const MEMORY_PROMPT_SECTION = {
|
|
948
1148
|
name: "avantf:memory-usage",
|
|
949
1149
|
order: 3e3,
|
|
950
|
-
text: "记忆(`mem_remember
|
|
1150
|
+
text: "记忆(`mem_remember`):只主动记录跨会话稳定、能影响未来决策的信息:用户偏好、长期约束、约定与术语、反复踩的坑、可复用判断。优先写成自包含的一条——事实型写「谁 / 什么范围 / 偏好或要求或术语是什么」,规则型写「当……时,应……,附条件与例外」。留下未来可检索的位置(路径 / 来源 / 名称),不抄全文;同一主题用 `update` 修正,不同事实新增。不要记录临时聊天、一次性任务的中间状态、能从权威来源轻易查到且无需跨会话记住的内容;敏感信息只记存放位置、不记明文。记录前自检:忘掉它,未来同类情境下我会不会做错、重走弯路或再问一遍?离开当前上下文还懂吗?还稳定吗?与已有记忆重复吗?通用判断再问跨任务/领域是否成立,领域记忆只要求同类任务可复用。"
|
|
951
1151
|
};
|
|
952
1152
|
/**
|
|
953
1153
|
* WHEN to consult the document knowledge base. See the module comment for why this is a separate
|
|
@@ -964,10 +1164,15 @@ const KNOWLEDGE_PROMPT_SECTION = {
|
|
|
964
1164
|
* directory (`<data_home>/prompts`, see {@link PromptFiles}).
|
|
965
1165
|
*
|
|
966
1166
|
* Every avantf plugin writes its prompts into that one directory and owns a prefix — `mem-*` here,
|
|
967
|
-
* `
|
|
1167
|
+
* `mission-*` in the mission engine — so a deployment can find and diff all of its model-facing text in
|
|
968
1168
|
* one place without any plugin having to guess which files belong to it. This manifest is that
|
|
969
1169
|
* ownership: a `.md` it does not list is ignored (never read, never written, never deleted).
|
|
970
1170
|
*
|
|
1171
|
+
* Since interface v3 the prefix is machine-checked, not a convention: every `file` below is a bare
|
|
1172
|
+
* name starting with `mem-`, and the loader is built with `namespace: 'mem'`
|
|
1173
|
+
* ({@link PROMPT_NAMESPACE}), so the base refuses — with a warning, without touching disk — any spec
|
|
1174
|
+
* that names a file outside this plugin's prefix.
|
|
1175
|
+
*
|
|
971
1176
|
* The FILE owns the text; the CODE keeps the identity. Mapping a file name to a section HERE is what
|
|
972
1177
|
* makes that true: no edit to the directory can move a section to a different point in the prompt
|
|
973
1178
|
* (order) or change the name the harness dedupes on — and no file another plugin owns can be
|
|
@@ -1211,10 +1416,15 @@ const DEFAULT_COMMON_CONFIG = `# avantf-mem 公共配置(分层第②层)。
|
|
|
1211
1416
|
# hnswlib: 2000
|
|
1212
1417
|
# hnswlib_ef_search: 256 # ANN 的召回/速度旋钮,语料变大先调它
|
|
1213
1418
|
|
|
1214
|
-
# retriever: #
|
|
1419
|
+
# retriever: # 三条检索腿在融合里的权重与"相关性门槛"
|
|
1215
1420
|
# weight_semantic: 0.55
|
|
1216
1421
|
# weight_fts: 0.30
|
|
1217
1422
|
# weight_jaccard: 0.15
|
|
1423
|
+
# # 绝对门槛打在每条腿的原始分上、fuse() 之前(融合分只在一次查询内可比,不能当门槛)。
|
|
1424
|
+
# # 0 = 关闭该门槛;分数等于门槛保留。语义后端不可用时 min_fts_terms 的生效值放宽到 1。
|
|
1425
|
+
# min_semantic_similarity: 0.5 # 语义腿余弦;0.5 只对 bge-small-zh-v1.5/512 标定过,换模型要重标
|
|
1426
|
+
# min_fts_terms: 2 # FTS 腿:这一行命中几个不同的查询词元(拉丁词≥5字符 + CJK 3-gram)
|
|
1427
|
+
# min_jaccard: 0.2 # 实体腿 Jaccard 比值
|
|
1218
1428
|
|
|
1219
1429
|
# lifecycle:
|
|
1220
1430
|
# purge_after_archived_days: 365 # 归档事实的物理清理窗口(活跃日)
|
|
@@ -1271,12 +1481,27 @@ function resolveStorePath(raw, fallback) {
|
|
|
1271
1481
|
if (!raw) return fallback;
|
|
1272
1482
|
return expandHome$1(raw);
|
|
1273
1483
|
}
|
|
1274
|
-
|
|
1484
|
+
/**
|
|
1485
|
+
* Read one config YAML, or `null` when it is absent / unreadable / not a mapping.
|
|
1486
|
+
*
|
|
1487
|
+
* A PARSE FAILURE IS NOT SILENT. Treating a broken file as "no configuration" is the right behaviour
|
|
1488
|
+
* — the process must keep starting — but the settings it silently drops include the security knobs
|
|
1489
|
+
* (`knowledge.ingest.local_roots`, `allow_outside_workspace`, `allow_private_network`, `trust.*`), so
|
|
1490
|
+
* one bad indent used to change the ingestion boundary with no trace anywhere. `warnUnknownKeys`
|
|
1491
|
+
* below calls itself "the one message that tells an operator their config file is partly ignored";
|
|
1492
|
+
* the branch where the WHOLE file is ignored was the one with no message at all.
|
|
1493
|
+
*
|
|
1494
|
+
* Still returns `null` (behaviour unchanged, every value falls back to its default). This is a
|
|
1495
|
+
* warning, deliberately not a fail-closed: refusing to start on a syntax error is a host-behaviour
|
|
1496
|
+
* change that needs a user decision, not a drive-by fix.
|
|
1497
|
+
*/
|
|
1498
|
+
function readYamlOrNull(path, logger) {
|
|
1275
1499
|
try {
|
|
1276
1500
|
if (!existsSync(path)) return null;
|
|
1277
1501
|
const parsed = parse(readFileSync(path, "utf8"));
|
|
1278
1502
|
return parsed && typeof parsed === "object" ? parsed : null;
|
|
1279
|
-
} catch {
|
|
1503
|
+
} catch (error) {
|
|
1504
|
+
logger.warn(`${path}: YAML parse failed (${error instanceof Error ? error.message : String(error)}) — the WHOLE file is ignored and every setting falls back to its default`);
|
|
1280
1505
|
return null;
|
|
1281
1506
|
}
|
|
1282
1507
|
}
|
|
@@ -1478,14 +1703,14 @@ function loadConfig(opts) {
|
|
|
1478
1703
|
});
|
|
1479
1704
|
ensureDataLayout(home);
|
|
1480
1705
|
ensureCommonConfigFile(commonConfigPath(home), logger);
|
|
1481
|
-
const commonYaml = readYamlOrNull(commonConfigPath(home));
|
|
1706
|
+
const commonYaml = readYamlOrNull(commonConfigPath(home), logger);
|
|
1482
1707
|
if (commonYaml) {
|
|
1483
1708
|
stripManagedModelKeys(commonYaml, commonConfigPath(home), logger);
|
|
1484
1709
|
warnUnknownKeys(commonConfigPath(home), commonYaml, zodShape(ConfigSchema) ?? {}, logger);
|
|
1485
1710
|
common = ConfigSchema.parse(mergeConfig(common, commonYaml));
|
|
1486
1711
|
}
|
|
1487
1712
|
common = ConfigSchema.parse(applyEnvLayer(common));
|
|
1488
|
-
const memYaml = readYamlOrNull(memoryConfigPath(home));
|
|
1713
|
+
const memYaml = readYamlOrNull(memoryConfigPath(home), logger);
|
|
1489
1714
|
if (memYaml) warnUnknownKeys(memoryConfigPath(home), memYaml, zodShape(MemoryConfigSchema) ?? {}, logger);
|
|
1490
1715
|
let memory = MemoryConfigSchema.parse(memYaml ?? {});
|
|
1491
1716
|
memory = {
|
|
@@ -1495,7 +1720,7 @@ function loadConfig(opts) {
|
|
|
1495
1720
|
path: resolveStorePath(memory.db.path, memoryDbPath(home))
|
|
1496
1721
|
}
|
|
1497
1722
|
};
|
|
1498
|
-
const kbYaml = readYamlOrNull(knowledgeConfigPath(home));
|
|
1723
|
+
const kbYaml = readYamlOrNull(knowledgeConfigPath(home), logger);
|
|
1499
1724
|
if (kbYaml) warnUnknownKeys(knowledgeConfigPath(home), kbYaml, zodShape(KnowledgeConfigSchema) ?? {}, logger);
|
|
1500
1725
|
let knowledge = KnowledgeConfigSchema.parse(kbYaml ?? {});
|
|
1501
1726
|
knowledge = {
|
|
@@ -1729,7 +1954,7 @@ function safeDestination(root, name, artifact) {
|
|
|
1729
1954
|
if (destination !== root && !destination.startsWith(root + sep)) throw new ProvisionError("extract", artifact, `归档条目超出目标目录:${name}`);
|
|
1730
1955
|
return destination;
|
|
1731
1956
|
}
|
|
1732
|
-
/** Extract one entry's bytes to `destination`, verifying both sizes. */
|
|
1957
|
+
/** Extract one entry's bytes to `destination`, verifying both sizes on BOTH methods. */
|
|
1733
1958
|
async function writeEntry(data, view, entry, destination, artifact) {
|
|
1734
1959
|
const localOffset = entry.localOffset;
|
|
1735
1960
|
if (localOffset + 30 > data.byteLength || view.getUint32(localOffset, true) !== LOCAL_SIGNATURE) throw new ProvisionError("extract", artifact, `条目「${entry.name}」的本地文件头损坏`);
|
|
@@ -1739,19 +1964,29 @@ async function writeEntry(data, view, entry, destination, artifact) {
|
|
|
1739
1964
|
const dataEnd = dataStart + entry.compressedSize;
|
|
1740
1965
|
if (dataEnd > data.byteLength) throw new ProvisionError("extract", artifact, `条目「${entry.name}」的数据被截断`);
|
|
1741
1966
|
const compressed = data.subarray(dataStart, dataEnd);
|
|
1967
|
+
if (entry.method === 0) {
|
|
1968
|
+
if (entry.compressedSize !== entry.uncompressedSize) throw new ProvisionError("extract", artifact, `条目「${entry.name}」声明 STORE 但两个大小不一致`);
|
|
1969
|
+
} else if (entry.method !== 8) throw new ProvisionError("extract", artifact, `条目「${entry.name}」的压缩方式 ${String(entry.method)} 不受支持(只支持 STORE/DEFLATE)`);
|
|
1742
1970
|
mkdirSync(dirname(destination), { recursive: true });
|
|
1743
1971
|
const out = createWriteStream(destination);
|
|
1744
1972
|
if (entry.method === 0) {
|
|
1745
|
-
if (entry.compressedSize !== entry.uncompressedSize) throw new ProvisionError("extract", artifact, `条目「${entry.name}」声明 STORE 但两个大小不一致`);
|
|
1746
1973
|
await pipeline(async function* () {
|
|
1747
1974
|
yield compressed;
|
|
1748
1975
|
}(), out);
|
|
1749
1976
|
return;
|
|
1750
1977
|
}
|
|
1751
|
-
|
|
1978
|
+
const expected = entry.uncompressedSize;
|
|
1979
|
+
let written = 0;
|
|
1752
1980
|
await pipeline(async function* () {
|
|
1753
1981
|
yield compressed;
|
|
1754
|
-
}(), createInflateRaw(),
|
|
1982
|
+
}(), createInflateRaw(), async function* (source) {
|
|
1983
|
+
for await (const chunk of source) {
|
|
1984
|
+
written += chunk.length;
|
|
1985
|
+
if (written > expected) throw new ProvisionError("extract", artifact, `条目「${entry.name}」解压后超过声明大小 ${String(expected)} 字节`);
|
|
1986
|
+
yield chunk;
|
|
1987
|
+
}
|
|
1988
|
+
}, out);
|
|
1989
|
+
if (written !== expected) throw new ProvisionError("extract", artifact, `条目「${entry.name}」解压大小与声明不一致(声明 ${String(expected)},实际 ${String(written)} 字节)`);
|
|
1755
1990
|
}
|
|
1756
1991
|
/** On-disk size of a directory tree, for the "did anything land" check and the install log. */
|
|
1757
1992
|
function treeBytes(dir) {
|
|
@@ -1872,7 +2107,7 @@ async function fetchToFile(url, dest, expected, artifact) {
|
|
|
1872
2107
|
* Download from the first working source into `dest`.
|
|
1873
2108
|
*
|
|
1874
2109
|
* Failures are ACCUMULATED, not swallowed: every candidate is tried because the point of a mirror
|
|
1875
|
-
* list is that one of them may be down, but if none
|
|
2110
|
+
* list is that one of them may be down, but if none missions the error carries the whole list of
|
|
1876
2111
|
* `url → reason` so the operator sees whether it was DNS, a TLS interception, or a 404.
|
|
1877
2112
|
*/
|
|
1878
2113
|
async function downloadAnySource(pack, config, dest, artifact) {
|
|
@@ -1939,6 +2174,13 @@ async function extractArchive(pack, archive, destination, artifact) {
|
|
|
1939
2174
|
* to detect its own stale holders and would have to tell the waiter when the winner is done, which is
|
|
1940
2175
|
* a coordination protocol rather than a rename — out of scope here, and recorded instead of implied.
|
|
1941
2176
|
*
|
|
2177
|
+
* This two-rename publish (over the sha256 verification in {@link fetchToFile}) IS the guarantee
|
|
2178
|
+
* level of `@avantf/mem-provision`: strictly below the base's envinit framework, which adds a
|
|
2179
|
+
* family-root lock with pid authority, slow-hold warnings and cross-device rejection. The CLI, the
|
|
2180
|
+
* MCP server and the plugin's degraded mount run on THIS stack, and base hardening does not reach it
|
|
2181
|
+
* — the full boundary and the reason both stacks coexist are in the module doc
|
|
2182
|
+
* (`@avantf/mem-provision`, `src/index.ts`).
|
|
2183
|
+
*
|
|
1942
2184
|
* A crash between the two renames leaves the old tree under `<target>.old-<pid>-<rand>` beside the
|
|
1943
2185
|
* version directory. Nothing enumerates that parent, so the residue is inert; the next install of the
|
|
1944
2186
|
* same version replaces the target normally.
|
|
@@ -2845,7 +3087,7 @@ const INTERROGATIVES = /* @__PURE__ */ new Set([
|
|
|
2845
3087
|
*
|
|
2846
3088
|
* Exists so one write can drive BOTH extractors from a single `jieba.tag()` call: `add`/`update`
|
|
2847
3089
|
* used to tag the same content twice (once for entities, once for triples), which is pure repeated
|
|
2848
|
-
*
|
|
3090
|
+
* mission — measured ~1.1 ms per 1000 characters per pass.
|
|
2849
3091
|
*/
|
|
2850
3092
|
async function tagText(text) {
|
|
2851
3093
|
const jieba = await loadJieba();
|
|
@@ -3103,7 +3345,7 @@ async function parseQueryPattern(query) {
|
|
|
3103
3345
|
const COMPAT_PREFIX = "compat:";
|
|
3104
3346
|
/** Hard cap on the host-readiness wait (see {@link WarmOptions.waitForCapMs}). */
|
|
3105
3347
|
const WAIT_FOR_HOST_MS = 3e4;
|
|
3106
|
-
/** A timer-based delay, the only await in this module that is not tied to real
|
|
3348
|
+
/** A timer-based delay, the only await in this module that is not tied to real mission. */
|
|
3107
3349
|
function delay(ms) {
|
|
3108
3350
|
return new Promise((resolve) => {
|
|
3109
3351
|
setTimeout(resolve, ms);
|
|
@@ -3155,6 +3397,30 @@ async function warmSemantic(rt, reportFinal = true) {
|
|
|
3155
3397
|
return semanticReady;
|
|
3156
3398
|
}
|
|
3157
3399
|
/**
|
|
3400
|
+
* Wait out the startup gate — the host readiness signal (bounded by {@link WAIT_FOR_HOST_MS}) and,
|
|
3401
|
+
* when asked, one quiet turn of the event loop.
|
|
3402
|
+
*
|
|
3403
|
+
* Extracted so there is exactly ONE definition of "a job that must not run inside the host's boot
|
|
3404
|
+
* window". The tokenizer warm below uses it, and so does the plugin's mount-time entity sweep:
|
|
3405
|
+
* `reindexEntities` tags stale rows through nodejieba, whose dictionary parse is the same ~1.2 s
|
|
3406
|
+
* SYNCHRONOUS mission the warm defers, so a sweep fired at mount (before this) could pull that parse
|
|
3407
|
+
* back into the startup window through a second door (performance review §7.5 / P5).
|
|
3408
|
+
*
|
|
3409
|
+
* The waits are NOT preconditions: a host whose boot failed still resolves them, and a host whose
|
|
3410
|
+
* boot hung is bounded here rather than trusted (see {@link WarmOptions.waitForCapMs}).
|
|
3411
|
+
*
|
|
3412
|
+
* @param options - the same options the tokenizer warm takes (the idle gate is opt-in).
|
|
3413
|
+
* @returns how long the idle gate waited (ms), or `0` when it was not requested.
|
|
3414
|
+
*/
|
|
3415
|
+
async function awaitStartupGate(options = {}) {
|
|
3416
|
+
if (options.waitFor !== void 0) {
|
|
3417
|
+
const cap = options.waitForCapMs ?? WAIT_FOR_HOST_MS;
|
|
3418
|
+
await Promise.race([options.waitFor.catch(() => void 0), delay(cap)]);
|
|
3419
|
+
}
|
|
3420
|
+
if (options.deferTokenizerUntilIdle !== true) return 0;
|
|
3421
|
+
return await whenEventLoopIdle();
|
|
3422
|
+
}
|
|
3423
|
+
/**
|
|
3158
3424
|
* Warm ONLY the tokenizer, after the host-readiness signal and (when asked) an idle event loop.
|
|
3159
3425
|
*
|
|
3160
3426
|
* nodejieba loads its dictionary lazily on the FIRST `tag()` (measured ~950 ms, after which a call
|
|
@@ -3166,15 +3432,9 @@ async function warmSemantic(rt, reportFinal = true) {
|
|
|
3166
3432
|
* @param options - the host-readiness signal and the idle gate (see {@link WarmOptions}).
|
|
3167
3433
|
*/
|
|
3168
3434
|
async function warmTokenizer(rt, options = {}) {
|
|
3169
|
-
if (options.waitFor !== void 0)
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
await Promise.race([options.waitFor.catch(() => void 0), delay(cap)]);
|
|
3173
|
-
}
|
|
3174
|
-
if (options.deferTokenizerUntilIdle === true) {
|
|
3175
|
-
const waited = await whenEventLoopIdle();
|
|
3176
|
-
if (waited >= 200) rt.logger.info(`model bootstrap: tokenizer warm waited ${waited}ms for an idle event loop`);
|
|
3177
|
-
}
|
|
3435
|
+
if (options.waitFor !== void 0) rt.logger.info("model bootstrap: tokenizer warm waits for the host readiness signal");
|
|
3436
|
+
const waited = await awaitStartupGate(options);
|
|
3437
|
+
if (options.deferTokenizerUntilIdle === true && waited >= 200) rt.logger.info(`model bootstrap: tokenizer warm waited ${waited}ms for an idle event loop`);
|
|
3178
3438
|
const jiebaStarted = Date.now();
|
|
3179
3439
|
if (await jiebaAvailable()) rt.logger.info(`model bootstrap: tokenizer ready in ${Date.now() - jiebaStarted}ms (nodejieba)`);
|
|
3180
3440
|
else rt.logger.warn(`model bootstrap: nodejieba unavailable in ${Date.now() - jiebaStarted}ms — entity/triple extraction falls back to the regex path (weaker entity + jaccard legs)`);
|
|
@@ -3213,13 +3473,13 @@ function warmTokenizerAsync(rt, options = {}) {
|
|
|
3213
3473
|
* The ONE startup initialization for this plugin: register every artifact this build needs, then run
|
|
3214
3474
|
* the provisioning sweep (non-blocking).
|
|
3215
3475
|
*
|
|
3216
|
-
* It replaced a
|
|
3476
|
+
* It replaced a one-shot model warm-up that sat beside a separate tools sweep, which meant two
|
|
3217
3477
|
* initialization paths, two logging vocabularies, and no single answer to "is this host fully
|
|
3218
3478
|
* provisioned?". Callers (the DSH plugin mount, the MCP entry) now make one call, and the per-artifact
|
|
3219
3479
|
* result lines in the host log come from the same mechanism that installs pandoc.
|
|
3220
3480
|
*
|
|
3221
3481
|
* The FIRST step is the dsh compatibility gate: an incompatible host means NOTHING is resolved,
|
|
3222
|
-
* installed or warmed — preparing an environment for an API we cannot drive is wasted
|
|
3482
|
+
* installed or warmed — preparing an environment for an API we cannot drive is wasted mission.
|
|
3223
3483
|
*
|
|
3224
3484
|
* @param rt - the runtime, whose resolved `tools` config and logger the sweep uses.
|
|
3225
3485
|
* @param options - the host's readiness signal and the dsh compatibility evidence.
|
|
@@ -3256,7 +3516,7 @@ async function provisionToolchainAsync(rt, options = {}) {
|
|
|
3256
3516
|
});
|
|
3257
3517
|
return { skipped: false };
|
|
3258
3518
|
}
|
|
3259
|
-
/** The artifact id, used in the startup log and in
|
|
3519
|
+
/** The artifact id, used in the startup log and in the model artifact's errors. */
|
|
3260
3520
|
const MODEL_ARTIFACT_ID = "model";
|
|
3261
3521
|
/** Read the runtime out of the per-call env, or fail with an explanation of what is missing. */
|
|
3262
3522
|
function runtimeOf(artifactEnv) {
|
|
@@ -3268,7 +3528,7 @@ function runtimeOf(artifactEnv) {
|
|
|
3268
3528
|
* The embedding model (plus the nodejieba tokenizer it shares its warm-up budget with) as an
|
|
3269
3529
|
* ARTIFACT, so the startup sweep in `@avantf/mem-provision` is the only initialization path.
|
|
3270
3530
|
*
|
|
3271
|
-
* It used to be a parallel
|
|
3531
|
+
* It used to be a parallel model warm-up call beside the tools sweep, which meant two
|
|
3272
3532
|
* startup mechanisms with two logging vocabularies and no single place that answered "is this host
|
|
3273
3533
|
* fully provisioned?". The model has no pinned version and no managed directory of its own ON THIS
|
|
3274
3534
|
* (the flat `<repo>/<file>` layout the transformers.js cache reads), so it
|
|
@@ -3409,6 +3669,7 @@ function empty() {
|
|
|
3409
3669
|
rerank_truncated: 0,
|
|
3410
3670
|
output_truncated: 0,
|
|
3411
3671
|
legs_capped: 0,
|
|
3672
|
+
candidates_dropped_by_floor: 0,
|
|
3412
3673
|
updated_at: null,
|
|
3413
3674
|
by_kind: {}
|
|
3414
3675
|
};
|
|
@@ -3426,6 +3687,11 @@ function recordRetrieval(event) {
|
|
|
3426
3687
|
else active.semantic_degraded += 1;
|
|
3427
3688
|
if (event.rerankUsed) active.rerank_used += 1;
|
|
3428
3689
|
if (event.rerankFallback) active.rerank_fallback += 1;
|
|
3690
|
+
const drops = event.droppedByFloor;
|
|
3691
|
+
if (drops !== void 0) {
|
|
3692
|
+
const total = drops.semantic + drops.fts + drops.jaccard + drops.hrr;
|
|
3693
|
+
if (Number.isFinite(total) && total > 0) active.candidates_dropped_by_floor += total;
|
|
3694
|
+
}
|
|
3429
3695
|
const bucket = active.by_kind[event.kind] ??= {
|
|
3430
3696
|
queries: 0,
|
|
3431
3697
|
zero_results: 0,
|
|
@@ -3485,6 +3751,7 @@ function retrievalHealthSummary() {
|
|
|
3485
3751
|
rerank_truncated: h.rerank_truncated,
|
|
3486
3752
|
output_truncated: h.output_truncated,
|
|
3487
3753
|
legs_capped: h.legs_capped,
|
|
3754
|
+
candidates_dropped_by_floor: h.candidates_dropped_by_floor,
|
|
3488
3755
|
updated_at: h.updated_at,
|
|
3489
3756
|
by_kind: h.by_kind
|
|
3490
3757
|
};
|
|
@@ -4170,7 +4437,8 @@ function blankToFamilyModels(configured) {
|
|
|
4170
4437
|
/**
|
|
4171
4438
|
* `local_numpy` — in-memory brute-force vector store.
|
|
4172
4439
|
* Default / dev backend; persists via the caller (the store DB owns the vectors).
|
|
4173
|
-
*
|
|
4440
|
+
* The fallback every backend without a real adapter resolves to; `hnswlib` is the implemented ANN
|
|
4441
|
+
* store, while `faiss`/`pgvector`/`qdrant` warn and land here.
|
|
4174
4442
|
*/
|
|
4175
4443
|
var LocalNumpyVectorStore = class {
|
|
4176
4444
|
name = "local_numpy";
|
|
@@ -4735,6 +5003,12 @@ function unimplementedVStore(name) {
|
|
|
4735
5003
|
* Pluggable backend registries. The DSH plugin does NOT expose these as Cordis
|
|
4736
5004
|
* services; replacement happens inside avantf-mem by config + registry (DESIGN §5).
|
|
4737
5005
|
* The business flow resolves backends only through these factories.
|
|
5006
|
+
*
|
|
5007
|
+
* The three `register*` functions below are the whole pluggability contract: register a name, put
|
|
5008
|
+
* that name in `config.<semantic|rerank|vectorStore>.backend`, and the resolver picks it up — no
|
|
5009
|
+
* business-flow change. They are re-exported on the PUBLISHED plugin face (`@avantf/dsh-mem`) so a
|
|
5010
|
+
* consumer OUTSIDE this repo can actually use the promise DESIGN §5 makes; see `packages/core/src/index.ts`
|
|
5011
|
+
* and `packages/plugin/src/index.ts`. Resolution stays a registry lookup, never duck-typing.
|
|
4738
5012
|
*/
|
|
4739
5013
|
const semanticRegistry = { local_bge: (cfg, opts) => new LocalBgeBackend(cfg.semantic.local_model, cfg.semantic.dim, {
|
|
4740
5014
|
mirror: cfg.semantic.mirror,
|
|
@@ -4760,6 +5034,15 @@ const vstoreRegistry = {
|
|
|
4760
5034
|
pgvector: unimplementedVStore("pgvector"),
|
|
4761
5035
|
qdrant: unimplementedVStore("qdrant")
|
|
4762
5036
|
};
|
|
5037
|
+
function registerSemanticBackend(name, factory) {
|
|
5038
|
+
semanticRegistry[name] = factory;
|
|
5039
|
+
}
|
|
5040
|
+
function registerReranker(name, factory) {
|
|
5041
|
+
rerankRegistry[name] = factory;
|
|
5042
|
+
}
|
|
5043
|
+
function registerVectorStore(name, factory) {
|
|
5044
|
+
vstoreRegistry[name] = factory;
|
|
5045
|
+
}
|
|
4763
5046
|
function resolveSemantic(cfg, opts) {
|
|
4764
5047
|
const f = semanticRegistry[cfg.semantic.backend];
|
|
4765
5048
|
if (!f) throw new Error(`未知的语义后端:${cfg.semantic.backend}`);
|
|
@@ -4972,7 +5255,7 @@ function zipEntryNames(bytes) {
|
|
|
4972
5255
|
* is the ordinary bomb shape — a few megabytes on the wire declaring gigabytes of sheet XML — before
|
|
4973
5256
|
* a reader library inflates any of it into the host's heap. Catching a lying archive would mean
|
|
4974
5257
|
* inflating it ourselves under a running cap instead of handing it to the library, which is a
|
|
4975
|
-
* different (and much larger) piece of
|
|
5258
|
+
* different (and much larger) piece of mission; see the note at the xlsx call site.
|
|
4976
5259
|
*/
|
|
4977
5260
|
function zipDeclaredCost(bytes) {
|
|
4978
5261
|
const directory = centralDirectory(bytes);
|
|
@@ -5314,7 +5597,7 @@ function pandocClaim(input) {
|
|
|
5314
5597
|
if (byExtension !== void 0) return { read: byExtension };
|
|
5315
5598
|
return looksLikeHtml(input.bytes) ? { read: "html+native_divs" } : void 0;
|
|
5316
5599
|
}
|
|
5317
|
-
/** How long pandoc may run on one document. Generous: a large ODT is a few seconds of
|
|
5600
|
+
/** How long pandoc may run on one document. Generous: a large ODT is a few seconds of mission. */
|
|
5318
5601
|
const CONVERT_TIMEOUT_MS = 12e4;
|
|
5319
5602
|
/** Cap on the Markdown pandoc may emit, so a pathological document cannot exhaust memory. */
|
|
5320
5603
|
const MAX_OUTPUT_BYTES = 64 * 1024 * 1024;
|
|
@@ -5372,8 +5655,9 @@ const pandocConverter = {
|
|
|
5372
5655
|
/**
|
|
5373
5656
|
* A rectangular grid of strings → a GitHub-flavored Markdown table.
|
|
5374
5657
|
*
|
|
5375
|
-
*
|
|
5376
|
-
* header row IS, and how a cell's content survives inside a
|
|
5658
|
+
* The grid renderer behind the XLSX converter (CSV reaches a table through pandoc's own reader).
|
|
5659
|
+
* Two decisions matter: what the header row IS, and how a cell's content survives inside a
|
|
5660
|
+
* `|`-delimited row. GFM tables have no
|
|
5377
5661
|
* "no header" form, so a first row that is plainly data (every cell numeric) is treated as data and
|
|
5378
5662
|
* a synthetic `列1…列N` header is emitted above it — the alternative would be promoting the first
|
|
5379
5663
|
* data row to a header and losing it from the body.
|
|
@@ -5481,9 +5765,9 @@ const xlsxConverter = {
|
|
|
5481
5765
|
return zipEntries(input.bytes)?.includes(WORKBOOK_ENTRY) === true;
|
|
5482
5766
|
},
|
|
5483
5767
|
async convert(input) {
|
|
5484
|
-
const label = input.path ?? "xlsx
|
|
5768
|
+
const label = input.path ?? "xlsx 任务簿";
|
|
5485
5769
|
const declared = zipDeclaredCost(input.bytes);
|
|
5486
|
-
if (declared !== null && declared.uncompressed > MAX_DECLARED_BYTES) throw new Error(`${label}
|
|
5770
|
+
if (declared !== null && declared.uncompressed > MAX_DECLARED_BYTES) throw new Error(`${label}:任务簿声明解压后 ${Math.round(declared.uncompressed / 1024 / 1024)} MB(${String(declared.entries)} 个条目),超过 ${String(MAX_DECLARED_BYTES / 1024 / 1024)} MB 上限,已拒绝转换。请先在原文件里删掉不需要的任务表或行,或导出为 CSV 后入库。`);
|
|
5487
5771
|
const { default: ExcelJS } = await import("exceljs");
|
|
5488
5772
|
const workbook = new ExcelJS.Workbook();
|
|
5489
5773
|
const data = Buffer.from(input.bytes.buffer, input.bytes.byteOffset, input.bytes.byteLength);
|
|
@@ -5494,7 +5778,7 @@ const xlsxConverter = {
|
|
|
5494
5778
|
const rowCount = worksheet.rowCount;
|
|
5495
5779
|
const columnCount = worksheet.columnCount;
|
|
5496
5780
|
if (rowCount === 0 || columnCount === 0) {
|
|
5497
|
-
warnings.push(
|
|
5781
|
+
warnings.push(`任务表「${worksheet.name}」为空,未产出表格`);
|
|
5498
5782
|
continue;
|
|
5499
5783
|
}
|
|
5500
5784
|
const rows = [];
|
|
@@ -5506,13 +5790,13 @@ const xlsxConverter = {
|
|
|
5506
5790
|
for (let c = 1; c <= Math.min(columnCount, MAX_COLS); c++) cells.push(formatCell(values[c]));
|
|
5507
5791
|
rows.push(cells);
|
|
5508
5792
|
}
|
|
5509
|
-
if (rowCount > MAX_ROWS) warnings.push(
|
|
5510
|
-
if (columnCount > MAX_COLS) warnings.push(
|
|
5511
|
-
const table = renderTable(rows,
|
|
5512
|
-
if (table.synthesizedHeader) warnings.push(
|
|
5793
|
+
if (rowCount > MAX_ROWS) warnings.push(`任务表「${worksheet.name}」共 ${String(rowCount)} 行,只转换前 ${String(MAX_ROWS)} 行`);
|
|
5794
|
+
if (columnCount > MAX_COLS) warnings.push(`任务表「${worksheet.name}」共 ${String(columnCount)} 列,只转换前 ${String(MAX_COLS)} 列`);
|
|
5795
|
+
const table = renderTable(rows, `任务表「${worksheet.name}」`);
|
|
5796
|
+
if (table.synthesizedHeader) warnings.push(`任务表「${worksheet.name}」首行是数值,已按数据行处理并合成列名`);
|
|
5513
5797
|
sections.push(`## ${worksheet.name}\n\n${table.markdown}`);
|
|
5514
5798
|
}
|
|
5515
|
-
if (sections.length === 0) throw new Error(`${label}
|
|
5799
|
+
if (sections.length === 0) throw new Error(`${label}:转换后没有可入库的表格(任务簿里没有非空任务表)`);
|
|
5516
5800
|
return {
|
|
5517
5801
|
markdown: sections.join("\n\n"),
|
|
5518
5802
|
converter: ID,
|
|
@@ -5799,49 +6083,210 @@ function migrate(db, migrations, ctx) {
|
|
|
5799
6083
|
//#endregion
|
|
5800
6084
|
//#region ../core/lib/db/sqlite.js
|
|
5801
6085
|
/**
|
|
5802
|
-
* The
|
|
6086
|
+
* The SQLite adapter — the runtime's OWN `node:sqlite`, and the only database implementation this
|
|
6087
|
+
* engine has.
|
|
6088
|
+
*
|
|
6089
|
+
* WHY THE BUILTIN AND NOT A BINDING: `better-sqlite3` is a NAN addon, so its `.node` file is tied to
|
|
6090
|
+
* ONE `NODE_MODULE_VERSION`. A dsh host that runs plugin code inside Electron (DSH Desktop does: the
|
|
6091
|
+
* profile host is `DeepSeek Harness.exe` with `ELECTRON_RUN_AS_NODE=1`, i.e. NODE_MODULE_VERSION 149
|
|
6092
|
+
* on Electron 44) cannot load a binding built for plain Node, and the upstream prebuilds stop well
|
|
6093
|
+
* short of that ABI — `new Database(path)` throws "Could not locate the bindings file", and the whole
|
|
6094
|
+
* memory store used to be unavailable there. The builtin is already inside the runtime that runs the
|
|
6095
|
+
* plugin (unflagged from Node 22.13 / 23.4; Electron 44 carries it), FTS5 included, so ONE
|
|
6096
|
+
* implementation serves every host instead of two code paths whose behaviour must be proven equal.
|
|
6097
|
+
* The parity this file owes the engine is therefore written down as a CONTRACT, not as a comparison:
|
|
6098
|
+
*
|
|
6099
|
+
* - statements are cached per SQL text (compiling one costs ~19x a reuse on this driver: measured
|
|
6100
|
+
* 6.03 µs vs 0.31 µs per `SELECT` in a 20k-iteration loop) — the DAOs prepare on every call;
|
|
6101
|
+
* - `run/get/all` take positional parameters OR one object of named parameters, with or without the
|
|
6102
|
+
* `:`/`@`/`$` prefix, and a named parameter the statement does not use is IGNORED (see
|
|
6103
|
+
* {@link NodeSqliteStatement}: the DAOs hand one params object to statements that use subsets of
|
|
6104
|
+
* it, and `FactsDao.purgeArchived` is the call site that made this load-bearing);
|
|
6105
|
+
* - BLOBs are returned as `Buffer` (the builtin hands back a plain `Uint8Array`) — the DAOs and
|
|
6106
|
+
* `hrr/encode.ts` read vector blobs through `buf.buffer`/`byteOffset`, and `Buffer.isBuffer`
|
|
6107
|
+
* checks exist in callers, so the adapter normalizes rather than teaching every reader two shapes;
|
|
6108
|
+
* - `pragma()` returns the pragma's rows (`[]` for a write), `transaction()` is `BEGIN
|
|
6109
|
+
* DEFERRED/IMMEDIATE/EXCLUSIVE` with `SAVEPOINT` nesting and rollback-on-throw — the presence
|
|
6110
|
+
* clock and the lifecycle tick depend on `immediate()` taking the write lock up front.
|
|
6111
|
+
*
|
|
6112
|
+
* The module is loaded through `createRequire`, never a static `import`: `node:sqlite` does not exist
|
|
6113
|
+
* before Node 22.5, and a static import would make THIS FILE unloadable there, failing the plugin's
|
|
6114
|
+
* loader row instead of degrading. A missing builtin has to be a PROBE failure (the runtime then
|
|
6115
|
+
* mounts DEGRADED and every tool answers with the reason), never a module-load failure.
|
|
6116
|
+
* @module @avantf/mem/db/sqlite
|
|
6117
|
+
*/
|
|
6118
|
+
/** Upper bound on the prepared-statement cache — see {@link NodeSqliteDb.prepare} for its key space. */
|
|
6119
|
+
const MAX_CACHED_STATEMENTS = 2e3;
|
|
6120
|
+
/** One line of an error, for a probe's report: the callers log it, they never rethrow the object. */
|
|
6121
|
+
function messageOf(error) {
|
|
6122
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
6123
|
+
const first = text.split("\n")[0] ?? text;
|
|
6124
|
+
return first.length > 300 ? `${first.slice(0, 297)}…` : first;
|
|
6125
|
+
}
|
|
6126
|
+
/**
|
|
6127
|
+
* Load the builtin synchronously, or explain why this runtime cannot.
|
|
5803
6128
|
*
|
|
5804
|
-
*
|
|
5805
|
-
* `
|
|
5806
|
-
*
|
|
6129
|
+
* `createRequire` (not `import`) because the port is synchronous by design: the stores open inside a
|
|
6130
|
+
* synchronous `buildRuntime`. The specifier is passed to the resolved `require` rather than written
|
|
6131
|
+
* as a `require(...)` literal, so a bundler leaves it as a runtime call instead of trying to resolve
|
|
6132
|
+
* (or inline) a builtin that may not exist on the target.
|
|
5807
6133
|
*/
|
|
5808
|
-
|
|
6134
|
+
function loadNodeSqlite() {
|
|
6135
|
+
try {
|
|
6136
|
+
const loaded = createRequire(import.meta.url)("node:sqlite");
|
|
6137
|
+
if (loaded === null || typeof loaded !== "object" || typeof loaded.DatabaseSync !== "function") return { reason: "the runtime exposes no node:sqlite DatabaseSync" };
|
|
6138
|
+
return { module: loaded };
|
|
6139
|
+
} catch (error) {
|
|
6140
|
+
return { reason: `node:sqlite is unavailable (${messageOf(error)}) — it needs Node >= 22.5, and is unflagged from Node 22.13 / 23.4` };
|
|
6141
|
+
}
|
|
6142
|
+
}
|
|
6143
|
+
/**
|
|
6144
|
+
* Whether this runtime can actually SERVE the stores through `node:sqlite`, or why it cannot.
|
|
6145
|
+
*
|
|
6146
|
+
* Constructing a throwaway `:memory:` handle is only the first half. A runtime can expose the module
|
|
6147
|
+
* and still be unusable for this engine in two ways that would otherwise surface much later, as a DDL
|
|
6148
|
+
* error or as a broken lifecycle tick in production:
|
|
6149
|
+
*
|
|
6150
|
+
* - **no FTS5** — both schemas create `fts5` virtual tables the moment a store is opened, so a build
|
|
6151
|
+
* without the module cannot hold a store at all;
|
|
6152
|
+
* - **no `setAllowUnknownNamedParameters`** (Node v22.15 / v23.11) — `node:sqlite` throws on a named
|
|
6153
|
+
* parameter the statement does not use, while the DAOs deliberately pass one params object to
|
|
6154
|
+
* statements that use subsets of it. Without the switch, `FactsDao.purgeArchived` takes the whole
|
|
6155
|
+
* lifecycle tick down.
|
|
6156
|
+
*
|
|
6157
|
+
* The probe reports the FIRST of these that applies as one line; `undefined` means "this runtime can
|
|
6158
|
+
* serve a store". It is NOT a dry run of the caller's database: a file that is corrupt, locked or
|
|
6159
|
+
* written by a newer schema must still fail loudly in {@link openSqlite} (and mount DEGRADED with its
|
|
6160
|
+
* own reason) rather than being reported as a driver problem.
|
|
6161
|
+
*/
|
|
6162
|
+
function sqliteProbe() {
|
|
6163
|
+
const loaded = loadNodeSqlite();
|
|
6164
|
+
if (loaded.module === void 0) return loaded.reason;
|
|
6165
|
+
return probeModule(loaded.module);
|
|
6166
|
+
}
|
|
6167
|
+
/**
|
|
6168
|
+
* The capability half of {@link sqliteProbe}: what a loaded `node:sqlite` module must be able to do.
|
|
6169
|
+
*
|
|
6170
|
+
* Exported — and taking the module as an argument — so the contract spec can hand it a FAKE and
|
|
6171
|
+
* assert each refusal by name. That is not a driver seam: it is the only way to exercise "this
|
|
6172
|
+
* runtime's builtin cannot serve a store" on a machine whose builtin can, and the failure path is
|
|
6173
|
+
* exactly the one nobody sees until it happens on a user's host.
|
|
6174
|
+
*/
|
|
6175
|
+
function probeModule(module) {
|
|
6176
|
+
let probe;
|
|
6177
|
+
try {
|
|
6178
|
+
probe = new module.DatabaseSync(":memory:");
|
|
6179
|
+
probe.exec("CREATE VIRTUAL TABLE probe_fts USING fts5(body)");
|
|
6180
|
+
const statement = probe.prepare("SELECT :value AS value");
|
|
6181
|
+
if (typeof statement.setAllowUnknownNamedParameters !== "function") return "node:sqlite has no setAllowUnknownNamedParameters (needs Node >= 22.15 / >= 23.11)";
|
|
6182
|
+
statement.setAllowUnknownNamedParameters(true);
|
|
6183
|
+
statement.get({
|
|
6184
|
+
value: 1,
|
|
6185
|
+
unused: 2
|
|
6186
|
+
});
|
|
6187
|
+
return;
|
|
6188
|
+
} catch (error) {
|
|
6189
|
+
return `node:sqlite cannot serve a store (${messageOf(error)})`;
|
|
6190
|
+
} finally {
|
|
6191
|
+
probe?.close();
|
|
6192
|
+
}
|
|
6193
|
+
}
|
|
6194
|
+
/**
|
|
6195
|
+
* This process's verdict on the builtin, decided on first use and then fixed.
|
|
6196
|
+
*
|
|
6197
|
+
* Memoized because it is a property of the RUNTIME, not of a database: re-probing at every store open
|
|
6198
|
+
* would spend an in-memory database per open (memory, knowledge, and the tokenizer self-check) to
|
|
6199
|
+
* re-learn something that cannot change. The first open pays for it; every later open reuses the
|
|
6200
|
+
* answer — including a refusal, which keeps failing with the same reason.
|
|
6201
|
+
*/
|
|
6202
|
+
let serveVerdict;
|
|
6203
|
+
let served = false;
|
|
6204
|
+
function serveReason() {
|
|
6205
|
+
if (!served) {
|
|
6206
|
+
served = true;
|
|
6207
|
+
serveVerdict = sqliteProbe();
|
|
6208
|
+
}
|
|
6209
|
+
return serveVerdict;
|
|
6210
|
+
}
|
|
6211
|
+
/**
|
|
6212
|
+
* A one-line, NEVER-THROWING description for the startup log.
|
|
6213
|
+
*
|
|
6214
|
+
* The runtime logs this before it opens the stores, so it must not be the thing that turns "this
|
|
6215
|
+
* runtime cannot serve the builtin" into a crash before the caller's own error handling runs. The
|
|
6216
|
+
* version is part of the line on purpose: the store files stay per-host (a Desktop HOME and a WSL
|
|
6217
|
+
* HOME are different machines' worth of data), so the SQLite version is the one fact that tells an
|
|
6218
|
+
* operator WHICH build is answering — and the fact to quote when a query behaves differently across
|
|
6219
|
+
* two hosts. A refused runtime is reported as such, so this line and the later DEGRADED mount agree.
|
|
6220
|
+
*/
|
|
6221
|
+
function describeSqlite() {
|
|
6222
|
+
const refused = serveReason();
|
|
6223
|
+
if (refused !== void 0) return `unavailable: ${refused}`;
|
|
6224
|
+
const loaded = loadNodeSqlite();
|
|
6225
|
+
if (loaded.module === void 0) return `unavailable: ${loaded.reason}`;
|
|
6226
|
+
try {
|
|
6227
|
+
const db = new loaded.module.DatabaseSync(":memory:");
|
|
6228
|
+
try {
|
|
6229
|
+
const row = db.prepare("SELECT sqlite_version() AS version").get();
|
|
6230
|
+
return `node:sqlite ${String(row?.version ?? "unknown")}`;
|
|
6231
|
+
} finally {
|
|
6232
|
+
db.close();
|
|
6233
|
+
}
|
|
6234
|
+
} catch (error) {
|
|
6235
|
+
return `unavailable: ${messageOf(error)}`;
|
|
6236
|
+
}
|
|
6237
|
+
}
|
|
6238
|
+
/**
|
|
6239
|
+
* One row as the port promises it: a BLOB is a `Buffer`.
|
|
6240
|
+
*
|
|
6241
|
+
* The copy is a VIEW over the same bytes (`Buffer.from(arrayBuffer, offset, length)`), so it costs no
|
|
6242
|
+
* data movement; the row object is only rebuilt when it actually carries a `Uint8Array`.
|
|
6243
|
+
*/
|
|
6244
|
+
function normalizeRow(row) {
|
|
6245
|
+
if (row === null || typeof row !== "object") return row;
|
|
6246
|
+
const source = row;
|
|
6247
|
+
let mapped;
|
|
6248
|
+
for (const key of Object.keys(source)) {
|
|
6249
|
+
const value = source[key];
|
|
6250
|
+
if (value instanceof Uint8Array && !Buffer.isBuffer(value)) {
|
|
6251
|
+
mapped ??= { ...source };
|
|
6252
|
+
mapped[key] = Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
6253
|
+
}
|
|
6254
|
+
}
|
|
6255
|
+
return mapped ?? source;
|
|
6256
|
+
}
|
|
6257
|
+
var NodeSqliteStatement = class {
|
|
5809
6258
|
stmt;
|
|
5810
6259
|
constructor(stmt) {
|
|
5811
6260
|
this.stmt = stmt;
|
|
6261
|
+
this.stmt.setAllowUnknownNamedParameters?.(true);
|
|
5812
6262
|
}
|
|
5813
6263
|
all(...params) {
|
|
5814
|
-
return this.stmt.all(...params);
|
|
6264
|
+
return this.stmt.all(...params).map((row) => normalizeRow(row));
|
|
5815
6265
|
}
|
|
5816
6266
|
get(...params) {
|
|
5817
|
-
|
|
6267
|
+
const row = this.stmt.get(...params);
|
|
6268
|
+
return row === void 0 ? void 0 : normalizeRow(row);
|
|
5818
6269
|
}
|
|
5819
6270
|
run(...params) {
|
|
5820
6271
|
const result = this.stmt.run(...params);
|
|
5821
6272
|
return {
|
|
5822
|
-
changes: result.changes,
|
|
6273
|
+
changes: Number(result.changes),
|
|
5823
6274
|
lastInsertRowid: result.lastInsertRowid
|
|
5824
6275
|
};
|
|
5825
6276
|
}
|
|
5826
6277
|
};
|
|
5827
|
-
|
|
5828
|
-
const MAX_CACHED_STATEMENTS = 2e3;
|
|
5829
|
-
var SqliteDb = class {
|
|
6278
|
+
var NodeSqliteDb = class {
|
|
5830
6279
|
db;
|
|
5831
6280
|
/**
|
|
5832
|
-
* Prepared-statement cache, keyed by SQL text.
|
|
5833
|
-
*
|
|
5834
|
-
*
|
|
5835
|
-
*
|
|
5836
|
-
*
|
|
5837
|
-
* was a real, per-call tax (~65–95 µs per write, plus the same again per search).
|
|
5838
|
-
*
|
|
5839
|
-
* One cache here covers every DAO without touching them, and the key space is bounded by the
|
|
5840
|
-
* SQL TEXT the code contains (~40 `.prepare(` sites, plus the handful of distinct `IN (...)`
|
|
5841
|
-
* widths `batches()` produces). Statements are reusable and re-entrant for `all`/`get`/`run`
|
|
5842
|
-
* (this port exposes no cursor API), and invalidated wholesale on `close()`.
|
|
6281
|
+
* Prepared-statement cache, keyed by SQL text. See the module doc for the measured cost of not
|
|
6282
|
+
* caching. The key space is NOT just the ~40 literal statements: `batches()` emits a different
|
|
6283
|
+
* `IN (?, …)` text per arity, and the arity follows the DATA (measured: 20 adds opened 17 keys, 15
|
|
6284
|
+
* of them one statement at different widths), so a resident process would otherwise grow this map
|
|
6285
|
+
* indefinitely — hence the same bound, and the same wholesale clear, as everywhere else here.
|
|
5843
6286
|
*/
|
|
5844
6287
|
statements = /* @__PURE__ */ new Map();
|
|
6288
|
+
/** Nesting depth, so a transaction inside a transaction becomes a SAVEPOINT. */
|
|
6289
|
+
depth = 0;
|
|
5845
6290
|
constructor(db) {
|
|
5846
6291
|
this.db = db;
|
|
5847
6292
|
}
|
|
@@ -5849,7 +6294,7 @@ var SqliteDb = class {
|
|
|
5849
6294
|
let cached = this.statements.get(sql);
|
|
5850
6295
|
if (cached === void 0) {
|
|
5851
6296
|
if (this.statements.size >= MAX_CACHED_STATEMENTS) this.statements.clear();
|
|
5852
|
-
cached = new
|
|
6297
|
+
cached = new NodeSqliteStatement(this.db.prepare(sql));
|
|
5853
6298
|
this.statements.set(sql, cached);
|
|
5854
6299
|
}
|
|
5855
6300
|
return cached;
|
|
@@ -5857,20 +6302,77 @@ var SqliteDb = class {
|
|
|
5857
6302
|
exec(sql) {
|
|
5858
6303
|
this.db.exec(sql);
|
|
5859
6304
|
}
|
|
5860
|
-
|
|
5861
|
-
|
|
5862
|
-
|
|
6305
|
+
/**
|
|
6306
|
+
* `PRAGMA` through a prepared statement: `node:sqlite` has no `pragma()` of its own, and preparing
|
|
6307
|
+
* the statement is what returns the rows a reader (`user_version`, `journal_mode`) asks for — a
|
|
6308
|
+
* write pragma simply returns `[]`.
|
|
6309
|
+
*/
|
|
5863
6310
|
pragma(source) {
|
|
5864
|
-
return this.db.
|
|
6311
|
+
return this.db.prepare(`PRAGMA ${source}`).all().map((row) => normalizeRow(row));
|
|
6312
|
+
}
|
|
6313
|
+
transaction(fn) {
|
|
6314
|
+
const runner = (mode) => () => this.atomic(fn, mode);
|
|
6315
|
+
const tx = runner("DEFERRED");
|
|
6316
|
+
tx.immediate = runner("IMMEDIATE");
|
|
6317
|
+
tx.deferred = runner("DEFERRED");
|
|
6318
|
+
tx.exclusive = runner("EXCLUSIVE");
|
|
6319
|
+
return tx;
|
|
5865
6320
|
}
|
|
5866
6321
|
close() {
|
|
5867
6322
|
this.statements.clear();
|
|
5868
6323
|
this.db.close();
|
|
5869
6324
|
}
|
|
6325
|
+
atomic(fn, mode) {
|
|
6326
|
+
if (this.depth > 0) {
|
|
6327
|
+
const savepoint = `avantf_nested_${String(this.depth)}`;
|
|
6328
|
+
this.depth += 1;
|
|
6329
|
+
this.db.exec(`SAVEPOINT ${savepoint}`);
|
|
6330
|
+
try {
|
|
6331
|
+
const value = fn();
|
|
6332
|
+
this.db.exec(`RELEASE ${savepoint}`);
|
|
6333
|
+
return value;
|
|
6334
|
+
} catch (error) {
|
|
6335
|
+
this.db.exec(`ROLLBACK TO ${savepoint}`);
|
|
6336
|
+
this.db.exec(`RELEASE ${savepoint}`);
|
|
6337
|
+
throw error;
|
|
6338
|
+
} finally {
|
|
6339
|
+
this.depth -= 1;
|
|
6340
|
+
}
|
|
6341
|
+
}
|
|
6342
|
+
this.depth = 1;
|
|
6343
|
+
this.db.exec(`BEGIN ${mode}`);
|
|
6344
|
+
try {
|
|
6345
|
+
const value = fn();
|
|
6346
|
+
this.db.exec("COMMIT");
|
|
6347
|
+
return value;
|
|
6348
|
+
} catch (error) {
|
|
6349
|
+
try {
|
|
6350
|
+
this.db.exec("ROLLBACK");
|
|
6351
|
+
} catch {}
|
|
6352
|
+
throw error;
|
|
6353
|
+
} finally {
|
|
6354
|
+
this.depth = 0;
|
|
6355
|
+
}
|
|
6356
|
+
}
|
|
5870
6357
|
};
|
|
5871
|
-
/**
|
|
5872
|
-
|
|
5873
|
-
|
|
6358
|
+
/**
|
|
6359
|
+
* Open (or create) a SQLite database behind the port. `':memory:'` is a valid path.
|
|
6360
|
+
*
|
|
6361
|
+
* The runtime's verdict is checked HERE, not only by the startup log: a runtime whose builtin cannot
|
|
6362
|
+
* serve a store (no FTS5, or no tolerance for unused named parameters — see {@link sqliteProbe}) must
|
|
6363
|
+
* fail this open, so the caller's own error handling degrades the plugin with that reason instead of
|
|
6364
|
+
* the engine discovering it half-way through a schema migration or a lifecycle tick.
|
|
6365
|
+
*
|
|
6366
|
+
* `verdict` is a TEST seam, and deliberately not a driver seam: there is one implementation and it
|
|
6367
|
+
* cannot be replaced at runtime. It exists so the refusal path can be asserted on a machine whose
|
|
6368
|
+
* builtin missions — the path nobody sees until a user hits it.
|
|
6369
|
+
*/
|
|
6370
|
+
function openSqlite(path, verdict = serveReason) {
|
|
6371
|
+
const refused = verdict();
|
|
6372
|
+
if (refused !== void 0) throw new Error(refused);
|
|
6373
|
+
const loaded = loadNodeSqlite();
|
|
6374
|
+
if (loaded.module === void 0) throw new Error(loaded.reason);
|
|
6375
|
+
return new NodeSqliteDb(new loaded.module.DatabaseSync(path));
|
|
5874
6376
|
}
|
|
5875
6377
|
//#endregion
|
|
5876
6378
|
//#region ../core/lib/db/store.js
|
|
@@ -5911,8 +6413,8 @@ function wasUpgraded(migration) {
|
|
|
5911
6413
|
}
|
|
5912
6414
|
/**
|
|
5913
6415
|
* Open (or create) a store database: directory, PRAGMAs, then every migration newer than
|
|
5914
|
-
* the file's `user_version`. A failure closes the handle —
|
|
5915
|
-
*
|
|
6416
|
+
* the file's `user_version`. A failure closes the handle — an open handle keeps the WAL and
|
|
6417
|
+
* its locks alive, and a half-open store would be worse than none.
|
|
5916
6418
|
*/
|
|
5917
6419
|
function openStoreDb(opts) {
|
|
5918
6420
|
mkdirSync(dirname(opts.path), { recursive: true });
|
|
@@ -6231,6 +6733,202 @@ function probeTerms(text, holds, stopAt = Number.POSITIVE_INFINITY) {
|
|
|
6231
6733
|
matched
|
|
6232
6734
|
};
|
|
6233
6735
|
}
|
|
6736
|
+
//#endregion
|
|
6737
|
+
//#region ../core/lib/store/floors.js
|
|
6738
|
+
/**
|
|
6739
|
+
* What the RELAXED pass lowers each floor TO, as absolute values.
|
|
6740
|
+
*
|
|
6741
|
+
* - semantic **0.40**: strictly BETWEEN the measured unrelated-query ceiling (0.384) and the
|
|
6742
|
+
* pronoun-query top-1 (0.444). Both bounds are load-bearing. Above 0.384, so a genuinely
|
|
6743
|
+
* unrelated question still comes back EMPTY from the relaxed pass — the four measured samples
|
|
6744
|
+
* topped out at 0.300/0.384/0.315/0.351, and a 0.35 bar would already admit two of them. Below
|
|
6745
|
+
* 0.444, so 「我是谁」 finds its fact. 0.40 keeps the larger margin on the noise side, which is
|
|
6746
|
+
* where a false positive costs the caller a wrong answer rather than a missing one.
|
|
6747
|
+
* - fts: NOT a constant — {@link looseTermFloor}, half the query's distinct terms with a floor of
|
|
6748
|
+
* 1. `1` here is the "no terms to scale by" baseline (the previously constant value); a query
|
|
6749
|
+
* that cannot produce terms is not graded at all, and one that produces a single term (every
|
|
6750
|
+
* 3-char CJK query) is already graded against that one term by the reachability clamp.
|
|
6751
|
+
* - jaccard **0.15** (positive — the previous `0` was the ungated hole this closes). Measured, not
|
|
6752
|
+
* guessed, on two corpora: (a) the live store, 64 active facts, the common-entity questions a
|
|
6753
|
+
* real user asks about this repo ("插件怎么安装" / "任务怎么拆分" / "知识库在哪里" …) put their
|
|
6754
|
+
* TOP non-answering entity candidate at Jaccard 0.0909 and most at 0.02–0.07, so 0.15 removes that
|
|
6755
|
+
* whole tail; (b) the gold-labelled 35-query eval set, where the LOWEST Jaccard any answering pair
|
|
6756
|
+
* reached is 0.25 (「老王喜欢什么风格」) — a bottom line above 0.25 starts dropping gold pairs,
|
|
6757
|
+
* while 0.15 filters none. The entity leg cannot fully separate the two classes on short queries
|
|
6758
|
+
* (a one-entity query against a one-entity fact is 1.0 by construction, and non-gold pairs reach
|
|
6759
|
+
* 0.667), so this is deliberately a "no incidental small overlap" line, not a relevance classifier.
|
|
6760
|
+
*/
|
|
6761
|
+
const LOOSE_FLOORS = {
|
|
6762
|
+
semantic: .4,
|
|
6763
|
+
fts: 1,
|
|
6764
|
+
jaccard: .15
|
|
6765
|
+
};
|
|
6766
|
+
/**
|
|
6767
|
+
* The FTS bar the RELAXED pass applies to a query carrying `termCount` distinct terms.
|
|
6768
|
+
*
|
|
6769
|
+
* Positive on purpose: "Relaxed" must never mean "ungated", and the old constant of `1` let a single
|
|
6770
|
+
* incidental trigram become a leg's head (per-leg max normalization makes it 1.0) on any query long
|
|
6771
|
+
* enough to produce several terms. Half the query's terms — `ceil(termCount / 2)`, at least 1 — keeps
|
|
6772
|
+
* a real fraction of what the query asked for while still relaxing the configured bar of 2 to 1 for
|
|
6773
|
+
* a two-term query. `termCount` 0/omitted keeps the baseline of 1 (such a query is not graded by
|
|
6774
|
+
* {@link applyTermFloor} at all). The `min(configured, …)` in {@link resolveFloors} means this can
|
|
6775
|
+
* only relax, never raise.
|
|
6776
|
+
*/
|
|
6777
|
+
function looseTermFloor(termCount) {
|
|
6778
|
+
const half = termCount === void 0 || termCount <= 0 ? 0 : Math.ceil(termCount / 2);
|
|
6779
|
+
return Math.max(1, half);
|
|
6780
|
+
}
|
|
6781
|
+
/** The floor every leg's drops start from. */
|
|
6782
|
+
function emptyFloorDrops() {
|
|
6783
|
+
return {
|
|
6784
|
+
semantic: 0,
|
|
6785
|
+
fts: 0,
|
|
6786
|
+
jaccard: 0,
|
|
6787
|
+
hrr: 0
|
|
6788
|
+
};
|
|
6789
|
+
}
|
|
6790
|
+
/**
|
|
6791
|
+
* The FTS bar actually applied to a query carrying `termCount` distinct terms.
|
|
6792
|
+
*
|
|
6793
|
+
* A query cannot be graded against more terms than it has (see the module comment: a 3-char CJK
|
|
6794
|
+
* query has one trigram), so the configured bar is CLAMPED to the reachable maximum. `termCount`
|
|
6795
|
+
* `0`/omitted leaves the value alone — such a query is not graded at all by {@link applyTermFloor},
|
|
6796
|
+
* and changing the REPORTED value there would misdescribe what the caller is reading. `0` stays `0`
|
|
6797
|
+
* (the operator's off switch).
|
|
6798
|
+
*
|
|
6799
|
+
* This is the ONE place the clamp lives; `resolveFloors` (what the result reports) and
|
|
6800
|
+
* `applyTermFloor` (what the leg does) both call it, so the number the caller reads is the number
|
|
6801
|
+
* the leg used.
|
|
6802
|
+
*/
|
|
6803
|
+
function effectiveTermFloor(floor, termCount) {
|
|
6804
|
+
if (!(floor > 0)) return floor;
|
|
6805
|
+
if (termCount === void 0 || termCount <= 0) return floor;
|
|
6806
|
+
return Math.min(floor, termCount);
|
|
6807
|
+
}
|
|
6808
|
+
/**
|
|
6809
|
+
* The legs a pass actually dropped candidates on — the only ones a relaxed pass can help.
|
|
6810
|
+
*
|
|
6811
|
+
* A leg that dropped nothing has no candidate below its strict floor, so lowering that floor admits
|
|
6812
|
+
* nothing new; including it in the relaxed `floors` would just misreport which leg was the problem.
|
|
6813
|
+
* `hrr` maps onto `jaccard` because the probe shares the entity leg's candidate set and floor (see
|
|
6814
|
+
* `HybridLeg.leg`).
|
|
6815
|
+
*/
|
|
6816
|
+
function droppedLegs(drops) {
|
|
6817
|
+
const legs = [];
|
|
6818
|
+
if (drops.semantic > 0) legs.push("semantic");
|
|
6819
|
+
if (drops.fts > 0) legs.push("fts");
|
|
6820
|
+
if (drops.jaccard > 0 || drops.hrr > 0) legs.push("jaccard");
|
|
6821
|
+
return legs;
|
|
6822
|
+
}
|
|
6823
|
+
/**
|
|
6824
|
+
* Resolve the effective floors for one query (see the module comment for the rules).
|
|
6825
|
+
*
|
|
6826
|
+
* Exported so both stores and the tests read the SAME rule rather than re-deriving it — the whole
|
|
6827
|
+
* point of hosting the orchestration in one file was that a per-store copy of a rule drifts.
|
|
6828
|
+
*
|
|
6829
|
+
* The relaxed profile takes `min(configured, relaxed)` per leg: it can only RELAX. A leg the
|
|
6830
|
+
* operator disabled (`0` = off) stays disabled, a floor the operator already set below the relaxed
|
|
6831
|
+
* value is left alone, and {@link FloorResolution.relaxLegs} can only narrow WHICH legs move.
|
|
6832
|
+
* "Relaxed" never means "ungated" — otherwise the pass could not answer "nothing relevant".
|
|
6833
|
+
*/
|
|
6834
|
+
function resolveFloors(retriever, semAvail, resolution = {}) {
|
|
6835
|
+
const configured = {
|
|
6836
|
+
semantic: retriever.min_semantic_similarity,
|
|
6837
|
+
fts: semAvail ? retriever.min_fts_terms : retriever.min_fts_terms > 0 ? 1 : 0,
|
|
6838
|
+
jaccard: retriever.min_jaccard
|
|
6839
|
+
};
|
|
6840
|
+
const mayRelax = (leg) => resolution.profile === "loose" && (resolution.relaxLegs === void 0 || resolution.relaxLegs.includes(leg));
|
|
6841
|
+
const lowered = {
|
|
6842
|
+
semantic: mayRelax("semantic") ? Math.min(configured.semantic, LOOSE_FLOORS.semantic) : configured.semantic,
|
|
6843
|
+
fts: mayRelax("fts") ? Math.min(configured.fts, looseTermFloor(resolution.termCount)) : configured.fts,
|
|
6844
|
+
jaccard: mayRelax("jaccard") ? Math.min(configured.jaccard, LOOSE_FLOORS.jaccard) : configured.jaccard
|
|
6845
|
+
};
|
|
6846
|
+
return {
|
|
6847
|
+
...lowered,
|
|
6848
|
+
fts: effectiveTermFloor(lowered.fts, resolution.termCount)
|
|
6849
|
+
};
|
|
6850
|
+
}
|
|
6851
|
+
/** `0` = off; a score EQUAL to the floor passes (the floor drops strictly-lower candidates only). */
|
|
6852
|
+
function passesFloor(score, floor) {
|
|
6853
|
+
return !(floor > 0) || score >= floor;
|
|
6854
|
+
}
|
|
6855
|
+
/** Drop the entries of one leg's raw score map that fall below `floor`, counting them. */
|
|
6856
|
+
function applyScoreFloor(scores, floor) {
|
|
6857
|
+
if (!(floor > 0) || scores.size === 0) return {
|
|
6858
|
+
scores,
|
|
6859
|
+
dropped: 0
|
|
6860
|
+
};
|
|
6861
|
+
const out = /* @__PURE__ */ new Map();
|
|
6862
|
+
let dropped = 0;
|
|
6863
|
+
for (const [id, score] of scores) if (passesFloor(score, floor)) out.set(id, score);
|
|
6864
|
+
else dropped += 1;
|
|
6865
|
+
return {
|
|
6866
|
+
scores: out,
|
|
6867
|
+
dropped
|
|
6868
|
+
};
|
|
6869
|
+
}
|
|
6870
|
+
/**
|
|
6871
|
+
* How many distinct query terms `text` carries.
|
|
6872
|
+
*
|
|
6873
|
+
* Substring containment, case-insensitively: FTS5's trigram tokenizer matches a quoted phrase by
|
|
6874
|
+
* consecutive trigrams and is ASCII-case-insensitive, which is the same predicate as "the term
|
|
6875
|
+
* occurs in the text". `relevanceTerms` lowercases its latin words, so the comparison has to.
|
|
6876
|
+
*/
|
|
6877
|
+
function countMatchedTerms(text, terms) {
|
|
6878
|
+
if (text === "") return 0;
|
|
6879
|
+
const haystack = text.toLowerCase();
|
|
6880
|
+
let matched = 0;
|
|
6881
|
+
for (const term of terms) if (haystack.includes(term)) matched += 1;
|
|
6882
|
+
return matched;
|
|
6883
|
+
}
|
|
6884
|
+
/**
|
|
6885
|
+
* Apply the per-row FTS floor to one FTS leg's scores.
|
|
6886
|
+
*
|
|
6887
|
+
* The terms come from `relevanceTerms(query)` (latin words ≥ 5 chars + every CJK 3-gram) — the same
|
|
6888
|
+
* tokenisation the conditional-hint probe uses, so "relevant enough" means one thing in this repo.
|
|
6889
|
+
* A query with NO such term is not graded at all: there is nothing to count, and dropping every row
|
|
6890
|
+
* on a technicality would turn a 3–4 char latin query (`buildFtsQuery` still emits it) into silence.
|
|
6891
|
+
*
|
|
6892
|
+
* The bar itself is {@link effectiveTermFloor}: a query that can only produce one term (a 3-char CJK
|
|
6893
|
+
* query is one trigram) is measured against that one term, not against a configured 2 it could never
|
|
6894
|
+
* reach.
|
|
6895
|
+
*/
|
|
6896
|
+
function applyTermFloor(scores, texts, query, floor) {
|
|
6897
|
+
if (!(floor > 0) || scores.size === 0) return {
|
|
6898
|
+
scores,
|
|
6899
|
+
dropped: 0
|
|
6900
|
+
};
|
|
6901
|
+
const terms = relevanceTerms(query);
|
|
6902
|
+
if (terms.length === 0) return {
|
|
6903
|
+
scores,
|
|
6904
|
+
dropped: 0
|
|
6905
|
+
};
|
|
6906
|
+
const effective = effectiveTermFloor(floor, terms.length);
|
|
6907
|
+
const out = /* @__PURE__ */ new Map();
|
|
6908
|
+
let dropped = 0;
|
|
6909
|
+
for (const [id, score] of scores) if (passesFloor(countMatchedTerms(texts.get(id) ?? "", terms), effective)) out.set(id, score);
|
|
6910
|
+
else dropped += 1;
|
|
6911
|
+
return {
|
|
6912
|
+
scores: out,
|
|
6913
|
+
dropped
|
|
6914
|
+
};
|
|
6915
|
+
}
|
|
6916
|
+
/** Sum per-leg drop counts (the cross-store result merges both stores' reports). */
|
|
6917
|
+
function mergeFloorDrops(...parts) {
|
|
6918
|
+
const out = emptyFloorDrops();
|
|
6919
|
+
for (const part of parts) {
|
|
6920
|
+
if (part === void 0) continue;
|
|
6921
|
+
out.semantic += part.semantic;
|
|
6922
|
+
out.fts += part.fts;
|
|
6923
|
+
out.jaccard += part.jaccard;
|
|
6924
|
+
out.hrr += part.hrr;
|
|
6925
|
+
}
|
|
6926
|
+
return out;
|
|
6927
|
+
}
|
|
6928
|
+
/** Sum every leg of one drop report (the retry decision and the health counter both read it). */
|
|
6929
|
+
function totalFloorDrops(drops) {
|
|
6930
|
+
return drops.semantic + drops.fts + drops.jaccard + drops.hrr;
|
|
6931
|
+
}
|
|
6234
6932
|
/**
|
|
6235
6933
|
* A retrieval input the CALLER got wrong, as opposed to a backend that failed at runtime.
|
|
6236
6934
|
*
|
|
@@ -6308,42 +7006,99 @@ async function hybridSearch(deps, plan) {
|
|
|
6308
7006
|
jaccard: retriever.weight_jaccard
|
|
6309
7007
|
} : DEGRADED_WEIGHTS;
|
|
6310
7008
|
const overFetch = Math.max(limit, plan.overFetch ?? limit * (retriever.over_fetch_factor || 5));
|
|
6311
|
-
const
|
|
6312
|
-
|
|
6313
|
-
|
|
6314
|
-
|
|
6315
|
-
|
|
6316
|
-
|
|
6317
|
-
|
|
6318
|
-
|
|
6319
|
-
}
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6323
|
-
|
|
6324
|
-
|
|
6325
|
-
|
|
6326
|
-
|
|
6327
|
-
|
|
6328
|
-
|
|
6329
|
-
|
|
6330
|
-
|
|
7009
|
+
const legCap = legCapFor(deps.config, overFetch);
|
|
7010
|
+
const termCount = relevanceTerms(query).length;
|
|
7011
|
+
/**
|
|
7012
|
+
* The query vector in force, ACROSS both passes.
|
|
7013
|
+
*
|
|
7014
|
+
* The single-store path is handed no vector by its caller (only the cross-store router encodes
|
|
7015
|
+
* once), so before this the relaxed retry re-encoded the same query — ~4 ms on a warm model, paid
|
|
7016
|
+
* every time the strict pass emptied with floor drops. A store publishes what it encoded through
|
|
7017
|
+
* {@link HybridContext.onQueryVector}, and the retry below reuses it. A caller-supplied vector
|
|
7018
|
+
* (cross-store) is already here and is passed through unchanged.
|
|
7019
|
+
*/
|
|
7020
|
+
let passVector = plan.queryVector;
|
|
7021
|
+
/**
|
|
7022
|
+
* One full pass under ONE floor profile: legs → fuse → live filter → rerank → slice → budget.
|
|
7023
|
+
*
|
|
7024
|
+
* `onReturn` (the store's reinforcement) is deliberately NOT called here: the strict pass is a
|
|
7025
|
+
* probe whenever a relaxed pass may follow, and reinforcing text the caller never receives is
|
|
7026
|
+
* exactly the defect `onReturn`'s doc calls out. The chosen pass is delivered by the caller below.
|
|
7027
|
+
*
|
|
7028
|
+
* `passVector` is read at call time and updated through `onQueryVector` while the pass runs, so the
|
|
7029
|
+
* relaxed retry starts with whatever the strict pass encoded (see the variable's own comment).
|
|
7030
|
+
*/
|
|
7031
|
+
const runPass = async (profile, relaxLegs) => {
|
|
7032
|
+
const floors = resolveFloors(retriever, semAvail, {
|
|
7033
|
+
profile,
|
|
7034
|
+
termCount,
|
|
7035
|
+
...relaxLegs === void 0 ? {} : { relaxLegs }
|
|
7036
|
+
});
|
|
7037
|
+
const legs = await runLegs(deps, {
|
|
7038
|
+
query,
|
|
7039
|
+
limit,
|
|
7040
|
+
overFetch,
|
|
7041
|
+
legCap,
|
|
7042
|
+
semAvail,
|
|
7043
|
+
weights,
|
|
7044
|
+
floors,
|
|
7045
|
+
...passVector === void 0 ? {} : { queryVector: passVector },
|
|
7046
|
+
onQueryVector: (vec) => {
|
|
7047
|
+
passVector = vec;
|
|
7048
|
+
}
|
|
7049
|
+
});
|
|
7050
|
+
let capped = 0;
|
|
7051
|
+
for (const leg of legs) if (leg.capped === true) capped += 1;
|
|
7052
|
+
const droppedByFloor = emptyFloorDrops();
|
|
7053
|
+
for (const leg of legs) if (leg.leg !== void 0 && leg.droppedByFloor !== void 0) droppedByFloor[leg.leg] += leg.droppedByFloor;
|
|
7054
|
+
const fused = fuse(legs.map((leg) => ({
|
|
7055
|
+
weight: leg.weight,
|
|
7056
|
+
scores: leg.scores
|
|
7057
|
+
})), overFetch);
|
|
7058
|
+
const texts = deps.texts(fused.map((h) => h.id));
|
|
7059
|
+
const live = fused.filter((h) => texts.has(h.id));
|
|
7060
|
+
const ranked = (await rerankHits(deps.reranker, query, live, (id) => texts.get(id) ?? "")).slice(0, limit);
|
|
7061
|
+
const hits = deps.hits(ranked, texts);
|
|
7062
|
+
const budgeted = applyOutputBudget(deps.config, hits, plan.maxTokens);
|
|
7063
|
+
return {
|
|
7064
|
+
hits: budgeted.kept,
|
|
7065
|
+
used_tokens: budgeted.used_tokens,
|
|
7066
|
+
floors,
|
|
7067
|
+
dropped_by_floor: droppedByFloor,
|
|
7068
|
+
capped
|
|
7069
|
+
};
|
|
7070
|
+
};
|
|
7071
|
+
let chosen = await runPass(plan.floors === "loose" ? "loose" : "strict", plan.relaxLegs);
|
|
7072
|
+
let relaxed = false;
|
|
7073
|
+
if (plan.floors === void 0 && chosen.hits.length === 0 && totalFloorDrops(chosen.dropped_by_floor) > 0) {
|
|
7074
|
+
const loosened = await runPass("loose", droppedLegs(chosen.dropped_by_floor));
|
|
7075
|
+
if (loosened.hits.length > 0) {
|
|
7076
|
+
chosen = loosened;
|
|
7077
|
+
relaxed = true;
|
|
7078
|
+
}
|
|
7079
|
+
}
|
|
7080
|
+
for (let i = 0; i < chosen.capped; i += 1) recordLegCapped();
|
|
7081
|
+
deps.onReturn?.(chosen.hits);
|
|
6331
7082
|
if (plan.recordStats !== false) {
|
|
6332
7083
|
const rerank = rerankState(deps.reranker);
|
|
6333
7084
|
recordRetrieval({
|
|
6334
7085
|
kind: deps.kind,
|
|
6335
|
-
results:
|
|
7086
|
+
results: chosen.hits.length,
|
|
6336
7087
|
latencyMs: Date.now() - startedAt,
|
|
6337
7088
|
semanticLive: semAvail,
|
|
6338
7089
|
rerankUsed: rerank.used,
|
|
6339
|
-
rerankFallback: rerank.fallback
|
|
7090
|
+
rerankFallback: rerank.fallback,
|
|
7091
|
+
droppedByFloor: chosen.dropped_by_floor
|
|
6340
7092
|
});
|
|
6341
7093
|
}
|
|
6342
7094
|
return {
|
|
6343
|
-
hits:
|
|
7095
|
+
hits: chosen.hits,
|
|
6344
7096
|
degraded: !semAvail,
|
|
6345
7097
|
weights,
|
|
6346
|
-
used_tokens:
|
|
7098
|
+
used_tokens: chosen.used_tokens,
|
|
7099
|
+
floors: chosen.floors,
|
|
7100
|
+
dropped_by_floor: chosen.dropped_by_floor,
|
|
7101
|
+
...relaxed ? { relaxed: true } : {}
|
|
6347
7102
|
};
|
|
6348
7103
|
}
|
|
6349
7104
|
/**
|
|
@@ -6458,6 +7213,23 @@ function vectorCachePath(dbPath, space) {
|
|
|
6458
7213
|
//#endregion
|
|
6459
7214
|
//#region ../core/lib/store/common.js
|
|
6460
7215
|
/**
|
|
7216
|
+
* The ONE write-side normalization both stores call before anything reaches the database.
|
|
7217
|
+
*
|
|
7218
|
+
* `@avantf/mem-contract`'s `toWellFormedText` is the implementation (well-formed + NFC); this name
|
|
7219
|
+
* is the store-layer entry, so the write paths say WHAT they are doing ("normalize this for a
|
|
7220
|
+
* write") rather than repeating a Unicode recipe. Every text field a store persists — fact content,
|
|
7221
|
+
* category, archive reason, a document's domain/source/title/uri, entity names and the SPO slots of
|
|
7222
|
+
* extracted triples — goes through here, which is what keeps a lone surrogate (and the strict-JSON
|
|
7223
|
+
* failure it causes downstream) out of the database in the first place.
|
|
7224
|
+
*/
|
|
7225
|
+
function normalizeWrite(value) {
|
|
7226
|
+
return toWellFormedText(value);
|
|
7227
|
+
}
|
|
7228
|
+
/** {@link normalizeWrite} over a batch (entity names, caller paths, …). */
|
|
7229
|
+
function normalizeWrites(values) {
|
|
7230
|
+
return values.map(normalizeWrite);
|
|
7231
|
+
}
|
|
7232
|
+
/**
|
|
6461
7233
|
* Identity of the vector space a store currently writes into.
|
|
6462
7234
|
*
|
|
6463
7235
|
* A change of backend, model or width makes every stored vector stale, and the point of recording
|
|
@@ -6494,6 +7266,36 @@ function reportForeignVectors(kind, space, foreign, fix) {
|
|
|
6494
7266
|
if (foreign <= 0) return;
|
|
6495
7267
|
retrievalLogger().warn(`${kind} index: ${String(foreign)} vector(s) were written in another space than ${space} — they are still ranked; run ${fix} to re-encode them`);
|
|
6496
7268
|
}
|
|
7269
|
+
/**
|
|
7270
|
+
* Hand the event loop back between two synchronous batches.
|
|
7271
|
+
*
|
|
7272
|
+
* The mount-time corpus reconcile reads and hashes EVERY managed document (and then walks the tree
|
|
7273
|
+
* again for orphans), all of it synchronous `node:fs`; at 10k documents that is a >0.6 s block of the
|
|
7274
|
+
* host's event loop (measured — see the performance review §7.5 / P6). `setImmediate` (a macrotask),
|
|
7275
|
+
* not a resolved-promise microtask: yielding to the microtask queue would let other JS continuations
|
|
7276
|
+
* run while timers and I/O queued behind the batch still waited.
|
|
7277
|
+
*/
|
|
7278
|
+
function yieldToEventLoop() {
|
|
7279
|
+
return new Promise((resolve) => {
|
|
7280
|
+
setImmediate(() => {
|
|
7281
|
+
resolve();
|
|
7282
|
+
});
|
|
7283
|
+
});
|
|
7284
|
+
}
|
|
7285
|
+
/**
|
|
7286
|
+
* Visit `items` in order, yielding the event loop every `every` items.
|
|
7287
|
+
*
|
|
7288
|
+
* Use this INSTEAD of a bare `for` loop whenever the per-item mission is synchronous filesystem or
|
|
7289
|
+
* hashing work whose item count follows the corpus (not the request): the caller's whole loop used to
|
|
7290
|
+
* be one uninterrupted block, so the host could not serve a timer or an I/O callback until it was
|
|
7291
|
+
* over. An empty list does nothing (and never yields).
|
|
7292
|
+
*/
|
|
7293
|
+
async function forEachYielding(items, visit, every = 200) {
|
|
7294
|
+
for (let index = 0; index < items.length; index += 1) {
|
|
7295
|
+
visit(items[index], index);
|
|
7296
|
+
if ((index + 1) % every === 0) await yieldToEventLoop();
|
|
7297
|
+
}
|
|
7298
|
+
}
|
|
6497
7299
|
//#endregion
|
|
6498
7300
|
//#region ../core/lib/hrr/atoms.js
|
|
6499
7301
|
/** 2π (float64, cross-platform stable). */
|
|
@@ -6664,7 +7466,8 @@ function hrrToBytes(vec) {
|
|
|
6664
7466
|
*
|
|
6665
7467
|
* Reads through a `DataView` instead of `new Float64Array(buf.buffer, buf.byteOffset, …)`:
|
|
6666
7468
|
* that constructor requires an 8-byte-ALIGNED `byteOffset`, and nothing about a SQLite
|
|
6667
|
-
* BLOB promises one (
|
|
7469
|
+
* BLOB promises one (the driver happens to hand back offset-0 views today, and the adapter passes
|
|
7470
|
+
* that offset through unchanged, but the
|
|
6668
7471
|
* callers here also pass plain `Uint8Array`s — a subarray or a pooled `Buffer` slice
|
|
6669
7472
|
* would throw a RangeError from deep inside retrieval). A trailing partial element is
|
|
6670
7473
|
* ignored, as before.
|
|
@@ -6688,12 +7491,72 @@ function hrrFromBytes(buf) {
|
|
|
6688
7491
|
}
|
|
6689
7492
|
return out;
|
|
6690
7493
|
}
|
|
7494
|
+
//#endregion
|
|
7495
|
+
//#region ../core/lib/db/chunk.js
|
|
7496
|
+
/**
|
|
7497
|
+
* Batch an id list for a SQL `IN (...)` clause.
|
|
7498
|
+
*
|
|
7499
|
+
* SQLite caps the number of bound parameters in one statement (SQLITE_MAX_VARIABLE_NUMBER,
|
|
7500
|
+
* 32 766 in the bundled build). A query whose id list can grow with the CORPUS rather than
|
|
7501
|
+
* with the request — `entityBags` on the write-path contradiction check, `filterActive` on
|
|
7502
|
+
* the catch-up sweep — will therefore hit "too many SQL variables" on a large store. The
|
|
7503
|
+
* write path swallows that as best-effort, so the symptom is a detection feature that
|
|
7504
|
+
* silently stops running instead of an error anyone notices.
|
|
7505
|
+
*/
|
|
7506
|
+
/** Parameters per statement — far below the engine limit, still one round trip per batch. */
|
|
7507
|
+
const SQL_PARAM_BATCH = 500;
|
|
6691
7508
|
/** Split `items` into consecutive batches of at most `size` (an empty list yields none). */
|
|
6692
|
-
function batches(items, size =
|
|
7509
|
+
function batches(items, size = SQL_PARAM_BATCH) {
|
|
6693
7510
|
const out = [];
|
|
6694
7511
|
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|
6695
7512
|
return out;
|
|
6696
7513
|
}
|
|
7514
|
+
/**
|
|
7515
|
+
* The `IN (…)` placeholder LADDER.
|
|
7516
|
+
*
|
|
7517
|
+
* The text of an `IN (?,?,…)` clause is part of the prepared-statement CACHE KEY (see
|
|
7518
|
+
* `db/sqlite.ts`), and the arity follows the DATA: a batched id list's last batch can be any width,
|
|
7519
|
+
* and the unbatched hit-set queries take whatever the caller passes. Left exact, those widths turn the
|
|
7520
|
+
* cache key space into thousands of one-use statements, and the cache's wholesale clear at its bound
|
|
7521
|
+
* then throws away the ~40 hot literal statements along with them (performance review §7.9 / P10).
|
|
7522
|
+
*
|
|
7523
|
+
* A key can only collapse if BOTH the placeholder text and the bound-parameter count are fixed for a
|
|
7524
|
+
* whole rung — padding the SQL with NULL literals would leave the text varying with the real arity.
|
|
7525
|
+
* So {@link inList} fixes the width to the next rung and pads the BOUND VALUES by repeating the last
|
|
7526
|
+
* one. Duplicates are inert in an `IN (…)`: `x IN (1,2,2)` is `x IN (1,2)`. The caller binds
|
|
7527
|
+
* `values`, not its original array.
|
|
7528
|
+
*
|
|
7529
|
+
* Above the last rung the exact width is used: those sites (a hit set, a predicate list) are bounded
|
|
7530
|
+
* by the caller, and padding an already-large list would only widen it.
|
|
7531
|
+
*/
|
|
7532
|
+
const IN_WIDTH_RUNGS = [
|
|
7533
|
+
8,
|
|
7534
|
+
16,
|
|
7535
|
+
32,
|
|
7536
|
+
64,
|
|
7537
|
+
128,
|
|
7538
|
+
256,
|
|
7539
|
+
512
|
|
7540
|
+
];
|
|
7541
|
+
/**
|
|
7542
|
+
* Prepare a value list for an `IN (…)` clause at a ladder-rounded, cache-stable width.
|
|
7543
|
+
*
|
|
7544
|
+
* `count <= 0` yields `NULL` (matches nothing) rather than `IN ()`, a syntax error — callers guard
|
|
7545
|
+
* empty lists anyway; this is belt-and-braces.
|
|
7546
|
+
*/
|
|
7547
|
+
function inList(values) {
|
|
7548
|
+
const count = values.length;
|
|
7549
|
+
if (count === 0) return {
|
|
7550
|
+
placeholders: "NULL",
|
|
7551
|
+
values: []
|
|
7552
|
+
};
|
|
7553
|
+
const rung = IN_WIDTH_RUNGS.find((width) => width >= count) ?? count;
|
|
7554
|
+
const padded = rung === count ? [...values] : [...values, ...Array.from({ length: rung - count }, () => values[count - 1])];
|
|
7555
|
+
return {
|
|
7556
|
+
placeholders: Array.from({ length: rung }, () => "?").join(","),
|
|
7557
|
+
values: padded
|
|
7558
|
+
};
|
|
7559
|
+
}
|
|
6697
7560
|
//#endregion
|
|
6698
7561
|
//#region ../core/lib/db/dao/entities.js
|
|
6699
7562
|
var EntitiesDao = class {
|
|
@@ -6739,10 +7602,10 @@ var EntitiesDao = class {
|
|
|
6739
7602
|
if (!ids.length) return out;
|
|
6740
7603
|
for (const id of ids) out.set(id, []);
|
|
6741
7604
|
for (const batch of batches(ids)) {
|
|
6742
|
-
const placeholders
|
|
7605
|
+
const { placeholders, values } = inList(batch);
|
|
6743
7606
|
const rows = this.db.prepare(`SELECT fe.fact_id AS fact_id, e.name AS name FROM fact_entities fe
|
|
6744
7607
|
JOIN entities e ON e.entity_id = fe.entity_id
|
|
6745
|
-
WHERE fe.fact_id IN (${placeholders})`).all(...
|
|
7608
|
+
WHERE fe.fact_id IN (${placeholders})`).all(...values);
|
|
6746
7609
|
for (const row of rows) out.get(row.fact_id)?.push(row.name);
|
|
6747
7610
|
}
|
|
6748
7611
|
return out;
|
|
@@ -6784,12 +7647,12 @@ var EntitiesDao = class {
|
|
|
6784
7647
|
/** Active facts linked to ALL of `names` (AND-join, `reason` leg). */
|
|
6785
7648
|
activeFactsForAllEntities(names) {
|
|
6786
7649
|
if (!names.length) return [];
|
|
6787
|
-
const placeholders
|
|
7650
|
+
const { placeholders, values } = inList(names);
|
|
6788
7651
|
return this.db.prepare(`SELECT fe.fact_id AS fact_id FROM fact_entities fe
|
|
6789
7652
|
JOIN entities e ON e.entity_id = fe.entity_id
|
|
6790
7653
|
JOIN facts fa ON fa.fact_id = fe.fact_id
|
|
6791
7654
|
WHERE e.name IN (${placeholders}) AND fa.status = 'active'
|
|
6792
|
-
GROUP BY fe.fact_id HAVING COUNT(DISTINCT e.name) = ?`).all(...
|
|
7655
|
+
GROUP BY fe.fact_id HAVING COUNT(DISTINCT e.name) = ?`).all(...values, names.length).map((row) => row.fact_id);
|
|
6793
7656
|
}
|
|
6794
7657
|
/** Active facts sharing ANY of `names`, in one category (`jaccard` leg). */
|
|
6795
7658
|
/**
|
|
@@ -6808,7 +7671,7 @@ var EntitiesDao = class {
|
|
|
6808
7671
|
*/
|
|
6809
7672
|
candidateFactsForAnyEntity(names, category, limit) {
|
|
6810
7673
|
if (!names.length || limit <= 0) return [];
|
|
6811
|
-
const placeholders
|
|
7674
|
+
const { placeholders, values } = inList(names);
|
|
6812
7675
|
return this.db.prepare(`SELECT fa.fact_id AS id, COUNT(*) AS shared,
|
|
6813
7676
|
(SELECT COUNT(*) FROM fact_entities x WHERE x.fact_id = fa.fact_id) AS total
|
|
6814
7677
|
FROM facts fa
|
|
@@ -6817,7 +7680,7 @@ var EntitiesDao = class {
|
|
|
6817
7680
|
WHERE e.name IN (${placeholders}) AND fa.status = 'active' AND (? IS NULL OR fa.category = ?)
|
|
6818
7681
|
GROUP BY fa.fact_id
|
|
6819
7682
|
ORDER BY (CAST(shared AS REAL) / (total + ? - shared)) DESC, fa.fact_id ASC
|
|
6820
|
-
LIMIT ?`).all(...
|
|
7683
|
+
LIMIT ?`).all(...values, category ?? null, category ?? null, names.length, limit).map((row) => row.id);
|
|
6821
7684
|
}
|
|
6822
7685
|
};
|
|
6823
7686
|
//#endregion
|
|
@@ -6857,6 +7720,42 @@ const FACT_COLUMNS_NO_BLOB = `fact_id, content, category, tags, trust_score, set
|
|
|
6857
7720
|
retrieval_count, helpful_count, last_retrieved_at, embedding_model, vector_store, status,
|
|
6858
7721
|
supersedes_id, entities_version, conflict_checked, archived_at, archive_reason, ttl_days,
|
|
6859
7722
|
mirror_source, mirror_target, created_at, updated_at`;
|
|
7723
|
+
/**
|
|
7724
|
+
* The FTS leg's exact SQL, exported so the plan guard (`test/fts_plan.spec.ts`) and the bench
|
|
7725
|
+
* control group EXPLAIN the production text itself instead of a copy that can silently drift.
|
|
7726
|
+
*
|
|
7727
|
+
* `NOT INDEXED` on the `facts` join is LOAD-BEARING, not decoration — this query is the one place
|
|
7728
|
+
* where a healthy-looking covering index (`idx_facts_status_category (status, category)`) is the
|
|
7729
|
+
* WRONG drive. Without planner statistics SQLite cannot know that `MATCH` is the selective access
|
|
7730
|
+
* path, so it drives from `facts` and RE-RUNS the whole external-content MATCH once per row:
|
|
7731
|
+
*
|
|
7732
|
+
* bad : SEARCH fa USING COVERING INDEX idx_facts_status_category (status=?)
|
|
7733
|
+
* | SCAN f VIRTUAL TABLE INDEX 0:=M1 <- the inner `=M1`
|
|
7734
|
+
* good: SCAN f VIRTUAL TABLE INDEX 0:M1
|
|
7735
|
+
* | SEARCH fa USING INTEGER PRIMARY KEY (rowid=?)
|
|
7736
|
+
*
|
|
7737
|
+
* Measured on the synthetic corpus at n=1000 (selective query, ~10 hits) in the fresh-session
|
|
7738
|
+
* shape: 574 ms → 4.6 ms for the leg; the hint's `LIMIT 1` shape is the SAME statement, 568 ms
|
|
7739
|
+
* → 1.9 ms.
|
|
7740
|
+
*
|
|
7741
|
+
* Why the missing statistics are not a startup-only blip: the open tick's `PRAGMA optimize` writes
|
|
7742
|
+
* no stats for an EMPTY database, so on a fresh install the whole first session runs without them
|
|
7743
|
+
* while the corpus grows in it; the next tick (≤60 min heartbeat) or a restart is what writes them.
|
|
7744
|
+
* `bench-memory` used to hide this because it seeds, then REOPENS the runtime (whose open tick
|
|
7745
|
+
* analyzes). Hence "ANALYZE facts at open when `sqlite_stat1` lacks a `facts` row and `facts` has
|
|
7746
|
+
* rows" does NOT close the window either: at open the fresh database has zero rows (the guard is
|
|
7747
|
+
* false), and once rows exist the next optimize is up to an hour away. `NOT INDEXED` pins the
|
|
7748
|
+
* FTS-driven plan independently of `sqlite_stat1`, so there is no window at all. It cannot change
|
|
7749
|
+
* results — only the join order — because the rowid join to `facts` is an INTEGER PRIMARY KEY
|
|
7750
|
+
* lookup (still available under `NOT INDEXED`), and `status`/`category` stay residual filters on
|
|
7751
|
+
* that one joined row, exactly as in the good plan above. Same spirit as §19 `page()`'s
|
|
7752
|
+
* `INDEXED BY`, which pins the plan against `PRAGMA optimize` flipping it back.
|
|
7753
|
+
*/
|
|
7754
|
+
const FTS_SEARCH_SQL = `SELECT f.rowid AS id, bm25(facts_fts) AS rank FROM facts_fts f
|
|
7755
|
+
JOIN facts fa NOT INDEXED ON fa.fact_id = f.rowid
|
|
7756
|
+
WHERE facts_fts MATCH ? AND fa.status = 'active' AND (? IS NULL OR fa.category = ?)
|
|
7757
|
+
ORDER BY rank ASC
|
|
7758
|
+
LIMIT ?`;
|
|
6860
7759
|
var FactsDao = class FactsDao {
|
|
6861
7760
|
db;
|
|
6862
7761
|
constructor(db) {
|
|
@@ -6871,8 +7770,8 @@ var FactsDao = class FactsDao {
|
|
|
6871
7770
|
activeIds(ids) {
|
|
6872
7771
|
const out = [];
|
|
6873
7772
|
for (const batch of batches(ids)) {
|
|
6874
|
-
const placeholders
|
|
6875
|
-
const rows = this.db.prepare(`SELECT fact_id FROM facts WHERE status = 'active' AND fact_id IN (${placeholders})`).all(...
|
|
7773
|
+
const { placeholders, values } = inList(batch);
|
|
7774
|
+
const rows = this.db.prepare(`SELECT fact_id FROM facts WHERE status = 'active' AND fact_id IN (${placeholders})`).all(...values);
|
|
6876
7775
|
for (const row of rows) out.push(row.fact_id);
|
|
6877
7776
|
}
|
|
6878
7777
|
return out;
|
|
@@ -6917,15 +7816,15 @@ var FactsDao = class FactsDao {
|
|
|
6917
7816
|
*/
|
|
6918
7817
|
rowsByIds(ids) {
|
|
6919
7818
|
if (!ids.length) return [];
|
|
6920
|
-
const placeholders
|
|
6921
|
-
return this.db.prepare(`SELECT ${FACT_COLUMNS_NO_BLOB} FROM facts WHERE fact_id IN (${placeholders})`).all(...
|
|
7819
|
+
const { placeholders, values } = inList(ids);
|
|
7820
|
+
return this.db.prepare(`SELECT ${FACT_COLUMNS_NO_BLOB} FROM facts WHERE fact_id IN (${placeholders})`).all(...values);
|
|
6922
7821
|
}
|
|
6923
7822
|
/** `fact_id → content` for a hit set (never N+1). */
|
|
6924
7823
|
textsByIds(ids) {
|
|
6925
7824
|
const out = /* @__PURE__ */ new Map();
|
|
6926
7825
|
if (!ids.length) return out;
|
|
6927
|
-
const placeholders
|
|
6928
|
-
const rows = this.db.prepare(`SELECT fact_id, content FROM facts WHERE fact_id IN (${placeholders})`).all(...
|
|
7826
|
+
const { placeholders, values } = inList(ids);
|
|
7827
|
+
const rows = this.db.prepare(`SELECT fact_id, content FROM facts WHERE fact_id IN (${placeholders})`).all(...values);
|
|
6929
7828
|
for (const row of rows) out.set(row.fact_id, row.content);
|
|
6930
7829
|
return out;
|
|
6931
7830
|
}
|
|
@@ -6933,8 +7832,8 @@ var FactsDao = class FactsDao {
|
|
|
6933
7832
|
timesByIds(ids) {
|
|
6934
7833
|
const out = /* @__PURE__ */ new Map();
|
|
6935
7834
|
if (!ids.length) return out;
|
|
6936
|
-
const placeholders
|
|
6937
|
-
const rows = this.db.prepare(`SELECT fact_id, created_at, updated_at FROM facts WHERE fact_id IN (${placeholders})`).all(...
|
|
7835
|
+
const { placeholders, values } = inList(ids);
|
|
7836
|
+
const rows = this.db.prepare(`SELECT fact_id, created_at, updated_at FROM facts WHERE fact_id IN (${placeholders})`).all(...values);
|
|
6938
7837
|
for (const row of rows) out.set(row.fact_id, {
|
|
6939
7838
|
created_at: row.created_at === null ? "" : String(row.created_at),
|
|
6940
7839
|
updated_at: row.updated_at === null ? null : String(row.updated_at)
|
|
@@ -7007,8 +7906,8 @@ var FactsDao = class FactsDao {
|
|
|
7007
7906
|
setEntitiesVersion(ids, version) {
|
|
7008
7907
|
let changes = 0;
|
|
7009
7908
|
for (const batch of batches(ids)) {
|
|
7010
|
-
const placeholders
|
|
7011
|
-
changes += this.db.prepare(`UPDATE facts SET entities_version = ? WHERE fact_id IN (${placeholders})`).run(version, ...
|
|
7909
|
+
const { placeholders, values } = inList(batch);
|
|
7910
|
+
changes += this.db.prepare(`UPDATE facts SET entities_version = ? WHERE fact_id IN (${placeholders})`).run(version, ...values).changes;
|
|
7012
7911
|
}
|
|
7013
7912
|
return changes;
|
|
7014
7913
|
}
|
|
@@ -7066,8 +7965,8 @@ var FactsDao = class FactsDao {
|
|
|
7066
7965
|
markConflictChecked(ids) {
|
|
7067
7966
|
let changes = 0;
|
|
7068
7967
|
for (const batch of batches(ids)) {
|
|
7069
|
-
const placeholders
|
|
7070
|
-
changes += this.db.prepare(`UPDATE facts SET conflict_checked = 1 WHERE fact_id IN (${placeholders})`).run(...
|
|
7968
|
+
const { placeholders, values } = inList(batch);
|
|
7969
|
+
changes += this.db.prepare(`UPDATE facts SET conflict_checked = 1 WHERE fact_id IN (${placeholders})`).run(...values).changes;
|
|
7071
7970
|
}
|
|
7072
7971
|
return changes;
|
|
7073
7972
|
}
|
|
@@ -7083,8 +7982,8 @@ var FactsDao = class FactsDao {
|
|
|
7083
7982
|
requeueConflictCheck(ids) {
|
|
7084
7983
|
let changes = 0;
|
|
7085
7984
|
for (const batch of batches(ids)) {
|
|
7086
|
-
const placeholders
|
|
7087
|
-
changes += this.db.prepare(`UPDATE facts SET conflict_checked = 0 WHERE fact_id IN (${placeholders})`).run(...
|
|
7985
|
+
const { placeholders, values } = inList(batch);
|
|
7986
|
+
changes += this.db.prepare(`UPDATE facts SET conflict_checked = 0 WHERE fact_id IN (${placeholders})`).run(...values).changes;
|
|
7088
7987
|
}
|
|
7089
7988
|
return changes;
|
|
7090
7989
|
}
|
|
@@ -7228,7 +8127,7 @@ var FactsDao = class FactsDao {
|
|
|
7228
8127
|
/**
|
|
7229
8128
|
* The two "reinforcement quota" numbers of `trust_diagnose`, from ONE statement.
|
|
7230
8129
|
*
|
|
7231
|
-
* They share a predicate and used to run it twice — twice the
|
|
8130
|
+
* They share a predicate and used to run it twice — twice the mission for two columns of the same
|
|
7232
8131
|
* rows. It is now also sargable (`bonus_window_at > datetime('now', '-1 day')`, a plain string
|
|
7233
8132
|
* comparison on the stored UTC-ISO format) and served by `idx_facts_bonus_window`, instead of
|
|
7234
8133
|
* calling `julianday` on every row of `facts` (measured 13.4–16.2 ms at 20k, per query).
|
|
@@ -7276,8 +8175,8 @@ var FactsDao = class FactsDao {
|
|
|
7276
8175
|
clearVectors(ids) {
|
|
7277
8176
|
let changes = 0;
|
|
7278
8177
|
for (const batch of batches(ids)) {
|
|
7279
|
-
const placeholders
|
|
7280
|
-
changes += this.db.prepare(`UPDATE facts SET semantic_vector = NULL, embedding_model = NULL, conflict_checked = 0 WHERE fact_id IN (${placeholders})`).run(...
|
|
8178
|
+
const { placeholders, values } = inList(batch);
|
|
8179
|
+
changes += this.db.prepare(`UPDATE facts SET semantic_vector = NULL, embedding_model = NULL, conflict_checked = 0 WHERE fact_id IN (${placeholders})`).run(...values).changes;
|
|
7281
8180
|
}
|
|
7282
8181
|
return changes;
|
|
7283
8182
|
}
|
|
@@ -7312,24 +8211,24 @@ var FactsDao = class FactsDao {
|
|
|
7312
8211
|
if (!ids.length) return [];
|
|
7313
8212
|
const out = [];
|
|
7314
8213
|
for (const batch of batches(ids)) {
|
|
7315
|
-
const placeholders
|
|
8214
|
+
const { placeholders, values } = inList(batch);
|
|
7316
8215
|
out.push(...this.db.prepare(`SELECT fact_id, hrr_vector FROM facts
|
|
7317
8216
|
WHERE fact_id IN (${placeholders}) AND status = 'active' AND hrr_vector IS NOT NULL
|
|
7318
|
-
AND (? IS NULL OR category = ?)`).all(...
|
|
8217
|
+
AND (? IS NULL OR category = ?)`).all(...values, category ?? null, category ?? null));
|
|
7319
8218
|
}
|
|
7320
8219
|
return out;
|
|
7321
8220
|
}
|
|
7322
8221
|
/** ACTIVE subset of `ids` in one category (`null` = any) — the semantic leg's filter. */
|
|
7323
8222
|
activeIdsIn(ids, category) {
|
|
7324
8223
|
if (!ids.length) return [];
|
|
7325
|
-
const placeholders
|
|
7326
|
-
return this.db.prepare(`SELECT fact_id FROM facts WHERE fact_id IN (${placeholders}) AND status = 'active' AND (? IS NULL OR category = ?)`).all(...
|
|
8224
|
+
const { placeholders, values } = inList(ids);
|
|
8225
|
+
return this.db.prepare(`SELECT fact_id FROM facts WHERE fact_id IN (${placeholders}) AND status = 'active' AND (? IS NULL OR category = ?)`).all(...values, category ?? null, category ?? null).map((row) => row.fact_id);
|
|
7327
8226
|
}
|
|
7328
8227
|
/** `retrieval_count` + the "last used" clock (spec §2.5), batched. */
|
|
7329
8228
|
touchUsage(ids) {
|
|
7330
8229
|
for (const batch of batches(ids)) {
|
|
7331
|
-
const placeholders
|
|
7332
|
-
this.db.prepare(`UPDATE facts SET retrieval_count = retrieval_count + 1, last_retrieved_at = CURRENT_TIMESTAMP WHERE fact_id IN (${placeholders})`).run(...
|
|
8230
|
+
const { placeholders, values } = inList(batch);
|
|
8231
|
+
this.db.prepare(`UPDATE facts SET retrieval_count = retrieval_count + 1, last_retrieved_at = CURRENT_TIMESTAMP WHERE fact_id IN (${placeholders})`).run(...values);
|
|
7333
8232
|
}
|
|
7334
8233
|
}
|
|
7335
8234
|
/** Recall bonus not granted: only the settled trust moved. */
|
|
@@ -7347,14 +8246,12 @@ var FactsDao = class FactsDao {
|
|
|
7347
8246
|
* FTS5 leg over the external-content `facts_fts`.
|
|
7348
8247
|
*
|
|
7349
8248
|
* `bm25()` is negative (more negative = better), so the caller negates; a JOIN back to
|
|
7350
|
-
* `facts` keeps archived rows and other categories out of the leg
|
|
8249
|
+
* `facts` keeps archived rows and other categories out of the leg — and `NOT INDEXED` keeps
|
|
8250
|
+
* that JOIN from becoming the drive (see {@link FTS_SEARCH_SQL}, which carries the plans and
|
|
8251
|
+
* measurements).
|
|
7351
8252
|
*/
|
|
7352
8253
|
ftsSearch(ftsQuery, category, limit) {
|
|
7353
|
-
return this.db.prepare(
|
|
7354
|
-
JOIN facts fa ON fa.fact_id = f.rowid
|
|
7355
|
-
WHERE facts_fts MATCH ? AND fa.status = 'active' AND (? IS NULL OR fa.category = ?)
|
|
7356
|
-
ORDER BY rank ASC
|
|
7357
|
-
LIMIT ?`).all(ftsQuery, category ?? null, category ?? null, limit ?? -1);
|
|
8254
|
+
return this.db.prepare(FTS_SEARCH_SQL).all(ftsQuery, category ?? null, category ?? null, limit ?? -1);
|
|
7358
8255
|
}
|
|
7359
8256
|
/**
|
|
7360
8257
|
* ①'s predicate: the active-day counter crossed an integer boundary since the last settle.
|
|
@@ -7471,17 +8368,24 @@ var FactsDao = class FactsDao {
|
|
|
7471
8368
|
*/
|
|
7472
8369
|
purgeArchived(p) {
|
|
7473
8370
|
const baseParams = FactsDao.purgeParams(p);
|
|
8371
|
+
const bounded = p.budget !== void 0 && p.budget > 0;
|
|
8372
|
+
let remaining = bounded ? Math.floor(p.budget) : 0;
|
|
7474
8373
|
const purged = /* @__PURE__ */ new Set();
|
|
7475
8374
|
for (const branch of FactsDao.purgeBranches(p)) {
|
|
7476
|
-
|
|
8375
|
+
if (bounded && remaining <= 0) break;
|
|
7477
8376
|
const predicate = bounded ? `${branch} AND fact_id IN (SELECT fact_id FROM facts WHERE ${branch} LIMIT :budget)` : branch;
|
|
7478
8377
|
const params = bounded ? {
|
|
7479
8378
|
...baseParams,
|
|
7480
|
-
budget:
|
|
8379
|
+
budget: remaining
|
|
7481
8380
|
} : baseParams;
|
|
7482
8381
|
this.db.prepare(`UPDATE facts SET supersedes_id = NULL
|
|
7483
8382
|
WHERE supersedes_id IN (SELECT fact_id FROM facts WHERE ${predicate})`).run(params);
|
|
7484
|
-
|
|
8383
|
+
let deleted = 0;
|
|
8384
|
+
for (const row of this.db.prepare(`DELETE FROM facts WHERE ${predicate} RETURNING fact_id`).all(params)) {
|
|
8385
|
+
purged.add(row.fact_id);
|
|
8386
|
+
deleted += 1;
|
|
8387
|
+
}
|
|
8388
|
+
remaining -= deleted;
|
|
7485
8389
|
}
|
|
7486
8390
|
return [...purged];
|
|
7487
8391
|
}
|
|
@@ -7584,9 +8488,9 @@ var TriplesDao = class {
|
|
|
7584
8488
|
*/
|
|
7585
8489
|
activeFactsBySubject(subjs, secondPred) {
|
|
7586
8490
|
if (!subjs.length) return [];
|
|
7587
|
-
const placeholders
|
|
8491
|
+
const { placeholders, values } = inList(subjs);
|
|
7588
8492
|
const secondClause = secondPred ? "AND t.pred = ?" : "";
|
|
7589
|
-
const params = secondPred ? [...
|
|
8493
|
+
const params = secondPred ? [...values, secondPred] : [...values];
|
|
7590
8494
|
return this.db.prepare(`SELECT DISTINCT t.fact_id AS fact_id FROM triples t
|
|
7591
8495
|
JOIN facts fa ON fa.fact_id = t.fact_id
|
|
7592
8496
|
WHERE fa.status = 'active' AND t.subj IN (${placeholders}) ${secondClause}`).all(...params).map((row) => row.fact_id);
|
|
@@ -7726,7 +8630,7 @@ var ContradictionsDao = class {
|
|
|
7726
8630
|
* {@link resolveForFact} for a BATCH of ids leaving the corpus — the lifecycle tick path.
|
|
7727
8631
|
*
|
|
7728
8632
|
* The per-id loop cost O(archived × log) plus one statement compile per id: measured 6.3 s for
|
|
7729
|
-
* 999 ids at 99k open pairs. Two statements PER BATCH (one per side) do the same
|
|
8633
|
+
* 999 ids at 99k open pairs. Two statements PER BATCH (one per side) do the same mission: each is
|
|
7730
8634
|
* an index seek on the plain `fact_a`/`fact_b` index, and the side that matched supplies the
|
|
7731
8635
|
* loser (the fact that actually left the corpus), so the recorded verdict is unchanged.
|
|
7732
8636
|
*/
|
|
@@ -7735,15 +8639,15 @@ var ContradictionsDao = class {
|
|
|
7735
8639
|
if (ids.length === 0) return 0;
|
|
7736
8640
|
let changes = 0;
|
|
7737
8641
|
for (const batch of batches(ids)) {
|
|
7738
|
-
const placeholders
|
|
8642
|
+
const { placeholders, values } = inList(batch);
|
|
7739
8643
|
changes += this.db.prepare(`UPDATE contradiction_log
|
|
7740
8644
|
SET resolved = 1, resolution = ?, resolved_at = CURRENT_TIMESTAMP, loser_fact_id = fact_a,
|
|
7741
8645
|
resolved_by = 'auto'
|
|
7742
|
-
WHERE resolved = 0 AND fact_a IN (${placeholders})`).run(resolution, ...
|
|
8646
|
+
WHERE resolved = 0 AND fact_a IN (${placeholders})`).run(resolution, ...values).changes;
|
|
7743
8647
|
changes += this.db.prepare(`UPDATE contradiction_log
|
|
7744
8648
|
SET resolved = 1, resolution = ?, resolved_at = CURRENT_TIMESTAMP, loser_fact_id = fact_b,
|
|
7745
8649
|
resolved_by = 'auto'
|
|
7746
|
-
WHERE resolved = 0 AND fact_b IN (${placeholders})`).run(resolution, ...
|
|
8650
|
+
WHERE resolved = 0 AND fact_b IN (${placeholders})`).run(resolution, ...values).changes;
|
|
7747
8651
|
}
|
|
7748
8652
|
return changes;
|
|
7749
8653
|
}
|
|
@@ -7785,7 +8689,9 @@ var ContradictionsDao = class {
|
|
|
7785
8689
|
const POLARITY_CONFLICT_SCORE = .95;
|
|
7786
8690
|
const OBJECT_CONFLICT_SCORE = .5;
|
|
7787
8691
|
const DEFAULT_CONTRADICT_THRESHOLD = .6;
|
|
8692
|
+
const EMBED_MIN_ENTITIES = 2;
|
|
7788
8693
|
const EMBED_OVERLAP_MIN = .5;
|
|
8694
|
+
const EMBED_SIM_MIN = .75;
|
|
7789
8695
|
const EMBED_SIM_DUP_MAX = .97;
|
|
7790
8696
|
/**
|
|
7791
8697
|
* Embedding-fallback contradiction score: `entity_overlap * semantic_sim`, only
|
|
@@ -7796,10 +8702,10 @@ const EMBED_SIM_DUP_MAX = .97;
|
|
|
7796
8702
|
function detectContradictionEmbedding(entitiesA, entitiesB, vecA, vecB, opts) {
|
|
7797
8703
|
const a = new Set(entitiesA);
|
|
7798
8704
|
const b = new Set(entitiesB);
|
|
7799
|
-
const minEntities = opts?.minEntities ??
|
|
7800
|
-
const overlapMin = opts?.overlapMin ??
|
|
7801
|
-
const simMin = opts?.simMin ??
|
|
7802
|
-
const simDupMax = opts?.simDupMax ??
|
|
8705
|
+
const minEntities = opts?.minEntities ?? EMBED_MIN_ENTITIES;
|
|
8706
|
+
const overlapMin = opts?.overlapMin ?? EMBED_OVERLAP_MIN;
|
|
8707
|
+
const simMin = opts?.simMin ?? EMBED_SIM_MIN;
|
|
8708
|
+
const simDupMax = opts?.simDupMax ?? EMBED_SIM_DUP_MAX;
|
|
7803
8709
|
if (Math.min(a.size, b.size) < minEntities) return 0;
|
|
7804
8710
|
const union = /* @__PURE__ */ new Set([...a, ...b]);
|
|
7805
8711
|
if (union.size === 0) return 0;
|
|
@@ -7986,11 +8892,11 @@ var ContradictDetector = class {
|
|
|
7986
8892
|
*/
|
|
7987
8893
|
findEntitySharing(factId) {
|
|
7988
8894
|
const mineCount = this.entities.countForFact(factId);
|
|
7989
|
-
if (mineCount <
|
|
8895
|
+
if (mineCount < EMBED_MIN_ENTITIES) return [];
|
|
7990
8896
|
const p = Math.max(EMBED_OVERLAP_MIN, this.threshold / EMBED_SIM_DUP_MAX);
|
|
7991
8897
|
const minShared = Math.ceil(p * mineCount);
|
|
7992
8898
|
const maxEntities = Math.floor(mineCount / p);
|
|
7993
|
-
if (minShared > mineCount || maxEntities <
|
|
8899
|
+
if (minShared > mineCount || maxEntities < EMBED_MIN_ENTITIES) return [];
|
|
7994
8900
|
return this.entities.candidateFacts(factId, minShared, maxEntities);
|
|
7995
8901
|
}
|
|
7996
8902
|
/**
|
|
@@ -8032,7 +8938,7 @@ var ContradictDetector = class {
|
|
|
8032
8938
|
if (!vecC) continue;
|
|
8033
8939
|
complete.push(c);
|
|
8034
8940
|
const entC = entFor(c);
|
|
8035
|
-
if (entC.size <
|
|
8941
|
+
if (entC.size < EMBED_MIN_ENTITIES) continue;
|
|
8036
8942
|
for (const t of candidates.get(c) ?? []) {
|
|
8037
8943
|
if (t === c) continue;
|
|
8038
8944
|
const key = c < t ? `${c}|${t}` : `${t}|${c}`;
|
|
@@ -8061,6 +8967,8 @@ var ContradictDetector = class {
|
|
|
8061
8967
|
//#endregion
|
|
8062
8968
|
//#region ../core/lib/lifecycle/trust.js
|
|
8063
8969
|
const DAY_MS = 864e5;
|
|
8970
|
+
/** The reinforcement quota window is a CALENDAR day ("每天" is a calendar notion, §2.3). */
|
|
8971
|
+
const QUOTA_WINDOW_MS = DAY_MS;
|
|
8064
8972
|
function clamp01(v) {
|
|
8065
8973
|
if (!Number.isFinite(v)) return 0;
|
|
8066
8974
|
return Math.min(1, Math.max(0, v));
|
|
@@ -8116,7 +9024,7 @@ function remainingDays(row, clock, cfg, enabled = cfg.enabled) {
|
|
|
8116
9024
|
function rollWindow(row, nowMs) {
|
|
8117
9025
|
const at = row.bonus_window_at;
|
|
8118
9026
|
const started = parseUtcTs(at);
|
|
8119
|
-
if (at === null || started === null || nowMs - started >=
|
|
9027
|
+
if (at === null || started === null || nowMs - started >= QUOTA_WINDOW_MS) return {
|
|
8120
9028
|
count: 0,
|
|
8121
9029
|
at: formatUtcTs(nowMs),
|
|
8122
9030
|
reset: true
|
|
@@ -8372,7 +9280,7 @@ const NO_CHECK = {
|
|
|
8372
9280
|
/**
|
|
8373
9281
|
* Facts one derived-state sweep pass may rebuild (`reindexEntities`'s default budget).
|
|
8374
9282
|
*
|
|
8375
|
-
* Bounds MEMORY, not only
|
|
9283
|
+
* Bounds MEMORY, not only mission: the pass selects `content` for every row it takes, so an
|
|
8376
9284
|
* "unbounded" pass (the CLI used to ask for `Number.MAX_SAFE_INTEGER`) builds an array of every
|
|
8377
9285
|
* stale fact's text before rebuilding the first one. Callers that want the corpus drained loop
|
|
8378
9286
|
* until `deferred === 0` instead.
|
|
@@ -8494,12 +9402,12 @@ var MemoryStore = class {
|
|
|
8494
9402
|
reportForeignVectors("memory", space, persisted.filter((r) => r.embedding_model !== space).length, "mem_admin vectors_fix (dry_run=true previews the count)");
|
|
8495
9403
|
}
|
|
8496
9404
|
async add(content, category, ttlDays) {
|
|
8497
|
-
const normalized = content.trim();
|
|
9405
|
+
const normalized = normalizeWrite(content).trim();
|
|
8498
9406
|
if (!normalized) throw new Error("内容不能为空");
|
|
8499
9407
|
const tokens = await tagText(normalized);
|
|
8500
|
-
const entities = entitiesFromTokens(tokens, normalized).map((e) => e.name);
|
|
8501
|
-
const triples = triplesFromTokens(tokens);
|
|
8502
|
-
const { fact_id, is_new, revived } = this.persistFact(normalized, category, ttlDays, entities, triples);
|
|
9408
|
+
const entities = normalizeWrites(entitiesFromTokens(tokens, normalized).map((e) => e.name));
|
|
9409
|
+
const triples = this.normalizeTriples(triplesFromTokens(tokens));
|
|
9410
|
+
const { fact_id, is_new, revived } = this.persistFact(normalized, category === void 0 ? void 0 : normalizeWrite(category), ttlDays, entities, triples);
|
|
8503
9411
|
if (is_new || revived) await this.maybeIndexSemantic(fact_id, normalized);
|
|
8504
9412
|
const check = is_new || revived ? this.detectContradictions(fact_id) : NO_CHECK;
|
|
8505
9413
|
if (check.complete) this.facts.markConflictChecked([fact_id]);
|
|
@@ -8563,12 +9471,12 @@ var MemoryStore = class {
|
|
|
8563
9471
|
}
|
|
8564
9472
|
async update(fact_id, content, category, ttlDays) {
|
|
8565
9473
|
if (!this.getRow(fact_id)) throw new Error(`找不到 fact_id=${fact_id}`);
|
|
8566
|
-
const normalized = content.trim();
|
|
9474
|
+
const normalized = normalizeWrite(content).trim();
|
|
8567
9475
|
if (!normalized) throw new Error("内容不能为空");
|
|
8568
9476
|
const tokens = await tagText(normalized);
|
|
8569
|
-
const entities = entitiesFromTokens(tokens, normalized).map((e) => e.name);
|
|
8570
|
-
const triples = triplesFromTokens(tokens);
|
|
8571
|
-
const { fact_id: newId, is_new, revived } = this.persistFact(normalized, category, ttlDays, entities, triples, {
|
|
9477
|
+
const entities = normalizeWrites(entitiesFromTokens(tokens, normalized).map((e) => e.name));
|
|
9478
|
+
const triples = this.normalizeTriples(triplesFromTokens(tokens));
|
|
9479
|
+
const { fact_id: newId, is_new, revived } = this.persistFact(normalized, category === void 0 ? void 0 : normalizeWrite(category), ttlDays, entities, triples, {
|
|
8572
9480
|
supersedesId: fact_id,
|
|
8573
9481
|
archiveOldIfActive: true
|
|
8574
9482
|
});
|
|
@@ -8590,7 +9498,7 @@ var MemoryStore = class {
|
|
|
8590
9498
|
const status = this.facts.statusOf(fact_id);
|
|
8591
9499
|
if (status === void 0) return false;
|
|
8592
9500
|
if (status === "archived") return true;
|
|
8593
|
-
const changed = this.facts.archive(fact_id, readClock(this.db), reason);
|
|
9501
|
+
const changed = this.facts.archive(fact_id, readClock(this.db), normalizeWrite(reason));
|
|
8594
9502
|
if (changed > 0) {
|
|
8595
9503
|
this.vstore.remove(fact_id);
|
|
8596
9504
|
this.contradictions.resolveForFact(fact_id);
|
|
@@ -9028,13 +9936,13 @@ var MemoryStore = class {
|
|
|
9028
9936
|
*
|
|
9029
9937
|
* Awaited by `maintenance` (one batch), so the caller knows the batch is done when it returns;
|
|
9030
9938
|
* `deferred` is what the next pass still has to do. Callers that want the whole corpus drained
|
|
9031
|
-
* loop until `deferred === 0` — `budget` bounds MEMORY as well as
|
|
9939
|
+
* loop until `deferred === 0` — `budget` bounds MEMORY as well as mission, since the pass reads
|
|
9032
9940
|
* every selected row's `content` before rebuilding the first one.
|
|
9033
9941
|
*
|
|
9034
9942
|
* The in-flight guard is here rather than at the callers because the callers CAN overlap: the
|
|
9035
9943
|
* plugin's heartbeat and `maintenance` are both periodic, and `staleEntityRows` is ORDERED, so
|
|
9036
9944
|
* two concurrent passes would select the same rows and re-tag them twice. Overlap is not
|
|
9037
|
-
* corruption (each fact is stamped in its own transaction) — it is duplicated
|
|
9945
|
+
* corruption (each fact is stamped in its own transaction) — it is duplicated mission in a
|
|
9038
9946
|
* single-threaded process. A pass that finds one running reports `skipped` and does nothing.
|
|
9039
9947
|
*
|
|
9040
9948
|
* `budget` is clamped at 0: SQLite reads a negative LIMIT as "no limit" (`LIMIT -1`), so a
|
|
@@ -9051,8 +9959,8 @@ var MemoryStore = class {
|
|
|
9051
9959
|
const rows = this.facts.staleEntityRows(1, Math.max(0, Math.floor(budget)));
|
|
9052
9960
|
for (const row of rows) {
|
|
9053
9961
|
const tokens = await tagText(row.content);
|
|
9054
|
-
const entities = entitiesFromTokens(tokens, row.content).map((e) => e.name);
|
|
9055
|
-
const triples = triplesFromTokens(tokens);
|
|
9962
|
+
const entities = normalizeWrites(entitiesFromTokens(tokens, row.content).map((e) => e.name));
|
|
9963
|
+
const triples = this.normalizeTriples(triplesFromTokens(tokens));
|
|
9056
9964
|
const hrr = hrrToBytes(encodeHrrEntityVector(entities));
|
|
9057
9965
|
this.db.transaction(() => {
|
|
9058
9966
|
this.entities.unlinkFact(row.fact_id);
|
|
@@ -9138,12 +10046,17 @@ var MemoryStore = class {
|
|
|
9138
10046
|
limit: input.limit,
|
|
9139
10047
|
maxTokens: input.maxTokens,
|
|
9140
10048
|
queryVector: input.queryVector,
|
|
9141
|
-
recordStats: input.recordStats
|
|
10049
|
+
recordStats: input.recordStats,
|
|
10050
|
+
floors: input.floors,
|
|
10051
|
+
...input.relaxLegs === void 0 ? {} : { relaxLegs: input.relaxLegs }
|
|
9142
10052
|
});
|
|
9143
10053
|
return {
|
|
9144
10054
|
hits: result.hits,
|
|
9145
10055
|
degraded: result.degraded,
|
|
9146
|
-
weights: result.weights
|
|
10056
|
+
weights: result.weights,
|
|
10057
|
+
floors: result.floors,
|
|
10058
|
+
dropped_by_floor: result.dropped_by_floor,
|
|
10059
|
+
...result.relaxed === true ? { relaxed: true } : {}
|
|
9147
10060
|
};
|
|
9148
10061
|
}
|
|
9149
10062
|
/**
|
|
@@ -9180,30 +10093,54 @@ var MemoryStore = class {
|
|
|
9180
10093
|
* rescale the survivors. That covers the Jaccard and FTS legs. It does NOT cover the HRR leg, whose
|
|
9181
10094
|
* candidates arrive in the JACCARD leg's order (or by recency in its fallback), so its cap can
|
|
9182
10095
|
* remove its own highest scorer and `v / max` really does move. Accepted — the cap bounds per-query
|
|
9183
|
-
*
|
|
10096
|
+
* mission and the pool is reranked afterwards — but do not read these scores as comparable across
|
|
9184
10097
|
* different cap/over-fetch settings; see `docs/PROVENANCE_REVIEW.md` N5 for the missing differential.
|
|
9185
10098
|
*/
|
|
9186
10099
|
async searchLegs(input, ctx) {
|
|
9187
10100
|
const qEntities = Array.from(new Set((await extractEntities(ctx.query)).map((e) => e.name)));
|
|
9188
10101
|
const candidates = qEntities.length > 0 ? this.entities.candidateFactsForAnyEntity(qEntities, input.category, ctx.legCap) : [];
|
|
9189
|
-
|
|
10102
|
+
/**
|
|
10103
|
+
* Wrap one leg's raw scores. `capped` must be measured on the RAW set (before the relevance
|
|
10104
|
+
* floor): the floor removes the tail anyway, and deriving the flag from the floored size would
|
|
10105
|
+
* erase the "this leg was cut at legCap" signal exactly when it bound.
|
|
10106
|
+
*/
|
|
10107
|
+
const leg = (scores, weight, raw) => ({
|
|
9190
10108
|
weight,
|
|
9191
10109
|
scores,
|
|
9192
|
-
capped: scores.size === ctx.legCap
|
|
10110
|
+
capped: (raw ?? scores).size === ctx.legCap
|
|
9193
10111
|
});
|
|
9194
|
-
const
|
|
10112
|
+
const ftsRaw = this.ftsPath(ctx.query, input.category, ctx.legCap);
|
|
10113
|
+
const ftsFloored = applyTermFloor(ftsRaw, this.loadTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
|
|
10114
|
+
const jaccardRaw = this.jaccardPath(qEntities, candidates);
|
|
10115
|
+
const jaccardFloored = applyScoreFloor(jaccardRaw, ctx.floors.jaccard);
|
|
10116
|
+
const jaccardLeg = {
|
|
10117
|
+
...leg(jaccardFloored.scores, ctx.weights.jaccard, jaccardRaw),
|
|
10118
|
+
leg: "jaccard",
|
|
10119
|
+
droppedByFloor: jaccardFloored.dropped
|
|
10120
|
+
};
|
|
9195
10121
|
const legs = [
|
|
9196
|
-
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector).then((
|
|
9197
|
-
|
|
9198
|
-
|
|
9199
|
-
|
|
10122
|
+
ctx.semAvail ? this.semanticPath(ctx.query, input.category, ctx.overFetch, ctx.queryVector, ctx.onQueryVector).then((raw) => {
|
|
10123
|
+
const floored = applyScoreFloor(raw, ctx.floors.semantic);
|
|
10124
|
+
return {
|
|
10125
|
+
weight: ctx.weights.semantic,
|
|
10126
|
+
scores: floored.scores,
|
|
10127
|
+
leg: "semantic",
|
|
10128
|
+
droppedByFloor: floored.dropped
|
|
10129
|
+
};
|
|
10130
|
+
}) : {
|
|
9200
10131
|
weight: ctx.weights.semantic,
|
|
9201
|
-
scores: /* @__PURE__ */ new Map()
|
|
10132
|
+
scores: /* @__PURE__ */ new Map(),
|
|
10133
|
+
leg: "semantic",
|
|
10134
|
+
droppedByFloor: 0
|
|
9202
10135
|
},
|
|
9203
|
-
Promise.resolve(
|
|
9204
|
-
|
|
10136
|
+
Promise.resolve(jaccardLeg),
|
|
10137
|
+
{
|
|
10138
|
+
...leg(ftsFloored.scores, ctx.weights.fts, ftsRaw),
|
|
10139
|
+
leg: "fts",
|
|
10140
|
+
droppedByFloor: ftsFloored.dropped
|
|
10141
|
+
}
|
|
9205
10142
|
];
|
|
9206
|
-
if (input.includeHrr) legs.push(Promise.resolve(leg(this.hrrPath(ctx.query, qEntities,
|
|
10143
|
+
if (input.includeHrr) legs.push(Promise.resolve(leg(this.hrrPath(ctx.query, qEntities, [...jaccardFloored.scores.keys()], input.category, ctx.legCap, candidates.length === 0), ctx.weights.jaccard)));
|
|
9207
10144
|
return legs;
|
|
9208
10145
|
}
|
|
9209
10146
|
/**
|
|
@@ -9300,11 +10237,14 @@ var MemoryStore = class {
|
|
|
9300
10237
|
}
|
|
9301
10238
|
/**
|
|
9302
10239
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
9303
|
-
* encoded this exact query (the cross-store router does, for both stores at once)
|
|
10240
|
+
* encoded this exact query (the cross-store router does, for both stores at once);
|
|
10241
|
+
* `onVector` publishes the vector this leg actually used, so the relaxed retry pass can reuse it
|
|
10242
|
+
* instead of re-encoding (performance review §7.7 / P8).
|
|
9304
10243
|
*/
|
|
9305
|
-
async semanticPath(query, category, k, queryVector) {
|
|
10244
|
+
async semanticPath(query, category, k, queryVector, onVector) {
|
|
9306
10245
|
const vec = queryVector ?? await this.semantic.encode(query);
|
|
9307
10246
|
if (vec.length !== this.vstore.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${this.vstore.dim}`);
|
|
10247
|
+
onVector?.(vec);
|
|
9308
10248
|
const topk = this.vstore.topk(vec, Math.max(50, k));
|
|
9309
10249
|
if (!topk.length) return /* @__PURE__ */ new Map();
|
|
9310
10250
|
const allowed = new Set(this.facts.activeIdsIn(topk.map((t) => t.id), category));
|
|
@@ -9317,10 +10257,11 @@ var MemoryStore = class {
|
|
|
9317
10257
|
* each active fact's persisted HRR vector. Recalls facts whose entities relate to
|
|
9318
10258
|
* the probe even when surface text shares no FTS trigrams.
|
|
9319
10259
|
*/
|
|
9320
|
-
hrrPath(query, entityNames, candidates, category, cap) {
|
|
10260
|
+
hrrPath(query, entityNames, candidates, category, cap, allowRecencyFallback = candidates.length === 0) {
|
|
9321
10261
|
if (entityNames.length === 0 && !query.trim()) return /* @__PURE__ */ new Map();
|
|
10262
|
+
if (candidates.length === 0 && !allowRecencyFallback) return /* @__PURE__ */ new Map();
|
|
9322
10263
|
const probe = encodeHrrEntityVector(entityNames.length ? entityNames : [query.trim()]);
|
|
9323
|
-
const truncatedFallback =
|
|
10264
|
+
const truncatedFallback = allowRecencyFallback;
|
|
9324
10265
|
const rows = truncatedFallback ? this.facts.activeHrrRows(category, cap) : this.facts.hrrRowsForFacts(candidates, category);
|
|
9325
10266
|
if (truncatedFallback && rows.length === cap && !this.hrrFallbackWarned) {
|
|
9326
10267
|
this.hrrFallbackWarned = true;
|
|
@@ -9502,6 +10443,21 @@ var MemoryStore = class {
|
|
|
9502
10443
|
linkEntities(fact_id, entities) {
|
|
9503
10444
|
this.entities.linkFact(fact_id, entities);
|
|
9504
10445
|
}
|
|
10446
|
+
/**
|
|
10447
|
+
* Normalize the SPO slots of extracted triples through the same write entry as fact content.
|
|
10448
|
+
*
|
|
10449
|
+
* The triples are DERIVED (from an already-normalized content string, in `add`/`update`), but a
|
|
10450
|
+
* tokenizer is free to hand back a half code unit, and a future truncation rule could split a
|
|
10451
|
+
* surrogate pair — so the derived rows cross the same entry instead of trusting the producer.
|
|
10452
|
+
*/
|
|
10453
|
+
normalizeTriples(triples) {
|
|
10454
|
+
return triples.map((t) => ({
|
|
10455
|
+
...t,
|
|
10456
|
+
subj: normalizeWrite(t.subj),
|
|
10457
|
+
pred: normalizeWrite(t.pred),
|
|
10458
|
+
obj: normalizeWrite(t.obj)
|
|
10459
|
+
}));
|
|
10460
|
+
}
|
|
9505
10461
|
insertTriples(fact_id, triples) {
|
|
9506
10462
|
this.triples.insertMany(fact_id, triples);
|
|
9507
10463
|
}
|
|
@@ -9725,8 +10681,8 @@ var ChunksDao = class {
|
|
|
9725
10681
|
texts(ids) {
|
|
9726
10682
|
const out = [];
|
|
9727
10683
|
for (const batch of batches(ids)) {
|
|
9728
|
-
const placeholders
|
|
9729
|
-
out.push(...this.db.prepare(`SELECT chunk_id, text FROM doc_chunks WHERE chunk_id IN (${placeholders})`).all(...
|
|
10684
|
+
const { placeholders, values } = inList(batch);
|
|
10685
|
+
out.push(...this.db.prepare(`SELECT chunk_id, text FROM doc_chunks WHERE chunk_id IN (${placeholders})`).all(...values));
|
|
9730
10686
|
}
|
|
9731
10687
|
return out;
|
|
9732
10688
|
}
|
|
@@ -9770,8 +10726,8 @@ var ChunksDao = class {
|
|
|
9770
10726
|
/** Record which extraction rules produced a chunk's entity rows (see `ENTITY_EXTRACTOR_VERSION`). */
|
|
9771
10727
|
setEntitiesVersion(ids, version) {
|
|
9772
10728
|
for (const batch of batches(ids)) {
|
|
9773
|
-
const placeholders
|
|
9774
|
-
this.db.prepare(`UPDATE doc_chunks SET entities_version = ? WHERE chunk_id IN (${placeholders})`).run(version, ...
|
|
10729
|
+
const { placeholders, values } = inList(batch);
|
|
10730
|
+
this.db.prepare(`UPDATE doc_chunks SET entities_version = ? WHERE chunk_id IN (${placeholders})`).run(version, ...values);
|
|
9775
10731
|
}
|
|
9776
10732
|
}
|
|
9777
10733
|
/** Every persisted chunk vector, for rebuilding the in-memory index at open. */
|
|
@@ -9806,8 +10762,8 @@ var ChunksDao = class {
|
|
|
9806
10762
|
entityBags(ids) {
|
|
9807
10763
|
const out = /* @__PURE__ */ new Map();
|
|
9808
10764
|
for (const batch of batches(ids)) {
|
|
9809
|
-
const placeholders
|
|
9810
|
-
const rows = this.db.prepare(`SELECT chunk_id, name FROM chunk_entities WHERE chunk_id IN (${placeholders})`).all(...
|
|
10765
|
+
const { placeholders, values } = inList(batch);
|
|
10766
|
+
const rows = this.db.prepare(`SELECT chunk_id, name FROM chunk_entities WHERE chunk_id IN (${placeholders})`).all(...values);
|
|
9811
10767
|
for (const r of rows) {
|
|
9812
10768
|
const list = out.get(r.chunk_id);
|
|
9813
10769
|
if (list) list.push(r.name);
|
|
@@ -9820,12 +10776,12 @@ var ChunksDao = class {
|
|
|
9820
10776
|
hits(ids) {
|
|
9821
10777
|
const out = [];
|
|
9822
10778
|
for (const batch of batches(ids)) {
|
|
9823
|
-
const placeholders
|
|
10779
|
+
const { placeholders, values } = inList(batch);
|
|
9824
10780
|
out.push(...this.db.prepare(`SELECT dc.chunk_id AS chunk_id, dc.text AS text, dc.source_ref AS source_ref,
|
|
9825
10781
|
d.domain AS domain, d.source AS source,
|
|
9826
10782
|
d.created_at AS created_at, d.updated_at AS updated_at
|
|
9827
10783
|
FROM doc_chunks dc JOIN documents d ON d.doc_id = dc.doc_id
|
|
9828
|
-
WHERE dc.chunk_id IN (${placeholders})`).all(...
|
|
10784
|
+
WHERE dc.chunk_id IN (${placeholders})`).all(...values));
|
|
9829
10785
|
}
|
|
9830
10786
|
return out;
|
|
9831
10787
|
}
|
|
@@ -9833,10 +10789,10 @@ var ChunksDao = class {
|
|
|
9833
10789
|
meta(ids) {
|
|
9834
10790
|
const out = [];
|
|
9835
10791
|
for (const batch of batches(ids)) {
|
|
9836
|
-
const placeholders
|
|
10792
|
+
const { placeholders, values } = inList(batch);
|
|
9837
10793
|
out.push(...this.db.prepare(`SELECT dc.chunk_id AS chunk_id, d.domain AS domain, d.source AS source
|
|
9838
10794
|
FROM doc_chunks dc JOIN documents d ON d.doc_id = dc.doc_id
|
|
9839
|
-
WHERE dc.chunk_id IN (${placeholders})`).all(...
|
|
10795
|
+
WHERE dc.chunk_id IN (${placeholders})`).all(...values));
|
|
9840
10796
|
}
|
|
9841
10797
|
return out;
|
|
9842
10798
|
}
|
|
@@ -9873,7 +10829,7 @@ var ChunksDao = class {
|
|
|
9873
10829
|
if (limit <= 0) return [];
|
|
9874
10830
|
const out = /* @__PURE__ */ new Set();
|
|
9875
10831
|
for (const batch of batches(names)) {
|
|
9876
|
-
const placeholders
|
|
10832
|
+
const { placeholders, values } = inList(batch);
|
|
9877
10833
|
const rows = this.db.prepare(`SELECT ce.chunk_id AS chunk_id, COUNT(*) AS shared,
|
|
9878
10834
|
(SELECT COUNT(*) FROM chunk_entities x WHERE x.chunk_id = ce.chunk_id) AS total
|
|
9879
10835
|
FROM chunk_entities ce
|
|
@@ -9887,7 +10843,7 @@ var ChunksDao = class {
|
|
|
9887
10843
|
AND (? IS NULL OR d.domain = ?) AND (? IS NULL OR d.source = ?)
|
|
9888
10844
|
GROUP BY ce.chunk_id
|
|
9889
10845
|
ORDER BY (CAST(shared AS REAL) / (total + ? - shared)) DESC, ce.chunk_id ASC
|
|
9890
|
-
LIMIT ?`).all(...
|
|
10846
|
+
LIMIT ?`).all(...values, domain ?? null, domain ?? null, source ?? null, source ?? null, batch.length, limit);
|
|
9891
10847
|
for (const r of rows) out.add(r.chunk_id);
|
|
9892
10848
|
}
|
|
9893
10849
|
return [...out];
|
|
@@ -10889,22 +11845,26 @@ var DocFiles = class {
|
|
|
10889
11845
|
/**
|
|
10890
11846
|
* Every `.md` under the root, with the `doc_id` its frontmatter names (when it names one).
|
|
10891
11847
|
* Used to report orphans — files left behind by a document that no longer exists.
|
|
11848
|
+
*
|
|
11849
|
+
* ASYNC and batched: the walk itself is an async recursive `readdir` (a `readdirSync` gathers
|
|
11850
|
+
* every entry in one ~41 ms call at 10k files), and the per-file parse is a `read` — at 10k
|
|
11851
|
+
* documents that is a second full read of the corpus on top of `sync`'s staleness pass, and it
|
|
11852
|
+
* used to be one uninterrupted synchronous block (performance review §7.5 / P6).
|
|
10892
11853
|
*/
|
|
10893
|
-
scan() {
|
|
11854
|
+
async scan() {
|
|
10894
11855
|
if (!existsSync(this.root)) return [];
|
|
10895
11856
|
const found = [];
|
|
10896
|
-
|
|
11857
|
+
await forEachYielding((await readdir(this.root, {
|
|
10897
11858
|
recursive: true,
|
|
10898
11859
|
withFileTypes: true
|
|
10899
|
-
})) {
|
|
10900
|
-
if (!entry.isFile() || !entry.name.endsWith(".md")) continue;
|
|
11860
|
+
})).filter((entry) => entry.isFile() && entry.name.endsWith(".md")), (entry) => {
|
|
10901
11861
|
const path = join(entry.parentPath ?? this.root, entry.name);
|
|
10902
11862
|
const parsed = this.read(path);
|
|
10903
11863
|
found.push({
|
|
10904
11864
|
path: relative(this.root, path).split(sep).join("/"),
|
|
10905
11865
|
...parsed?.meta.doc_id === void 0 ? {} : { doc_id: parsed.meta.doc_id }
|
|
10906
11866
|
});
|
|
10907
|
-
}
|
|
11867
|
+
});
|
|
10908
11868
|
return found;
|
|
10909
11869
|
}
|
|
10910
11870
|
};
|
|
@@ -11302,7 +12262,7 @@ var KnowledgeStore = class {
|
|
|
11302
12262
|
* list would silently turn it into a restricted one.
|
|
11303
12263
|
*/
|
|
11304
12264
|
addDomain(raw) {
|
|
11305
|
-
const domain = raw.trim();
|
|
12265
|
+
const domain = normalizeWrite(raw).trim();
|
|
11306
12266
|
if (domain === "") throw new Error("知识域不能为空");
|
|
11307
12267
|
if (domain.includes("/") || domain.includes("\\")) throw new Error(`知识域不能包含 “/” 或 “\\”:${domain}(它会成为受管目录的一级目录名)`);
|
|
11308
12268
|
if (this.allowedDomains.has(domain)) return this.domainCatalog();
|
|
@@ -11334,15 +12294,23 @@ var KnowledgeStore = class {
|
|
|
11334
12294
|
* request. All-or-nothing, so a partial write can never sit behind a confirmation dialog.
|
|
11335
12295
|
*/
|
|
11336
12296
|
async ingestRequest(req, mode) {
|
|
11337
|
-
const
|
|
11338
|
-
const plan = this.planIngest({
|
|
12297
|
+
const normalized = {
|
|
11339
12298
|
...req,
|
|
12299
|
+
domain: normalizeWrite(req.domain),
|
|
12300
|
+
...req.source === void 0 ? {} : { source: normalizeWrite(req.source) },
|
|
12301
|
+
...req.title === void 0 ? {} : { title: normalizeWrite(req.title) },
|
|
12302
|
+
...req.source_uri === void 0 ? {} : { source_uri: normalizeWrite(req.source_uri) },
|
|
12303
|
+
...req.paths === void 0 ? {} : { paths: normalizeWrites(req.paths) }
|
|
12304
|
+
};
|
|
12305
|
+
const source = normalized.source ?? "default";
|
|
12306
|
+
const plan = this.planIngest({
|
|
12307
|
+
...normalized,
|
|
11340
12308
|
source
|
|
11341
12309
|
});
|
|
11342
12310
|
const conflicts = plan.targets.flatMap((target) => target.existing === void 0 ? [] : [target.existing]);
|
|
11343
|
-
if (mode === "add") return this.runAdd(plan,
|
|
11344
|
-
if (conflicts.length > 0 &&
|
|
11345
|
-
return this.runReplace(plan,
|
|
12311
|
+
if (mode === "add") return this.runAdd(plan, normalized.domain, source);
|
|
12312
|
+
if (conflicts.length > 0 && normalized.overwrite !== true) return this.conflictReport(conflicts, plan.targets.length - conflicts.length);
|
|
12313
|
+
return this.runReplace(plan, normalized.domain, source);
|
|
11346
12314
|
}
|
|
11347
12315
|
/**
|
|
11348
12316
|
* Stage ①/②: resolve the request to its targets without reading anything.
|
|
@@ -11359,14 +12327,15 @@ var KnowledgeStore = class {
|
|
|
11359
12327
|
}
|
|
11360
12328
|
/** One target's identity plus whatever document already owns it. */
|
|
11361
12329
|
target(title, path, domain, source) {
|
|
11362
|
-
const
|
|
12330
|
+
const safeTitle = normalizeWrite(title);
|
|
12331
|
+
const row = this.docs.find(domain, source, safeTitle);
|
|
11363
12332
|
const existing = row === null ? void 0 : {
|
|
11364
12333
|
doc_id: row.doc_id,
|
|
11365
12334
|
title: row.title,
|
|
11366
12335
|
path: this.docFilePathOf(row)
|
|
11367
12336
|
};
|
|
11368
12337
|
return {
|
|
11369
|
-
title,
|
|
12338
|
+
title: safeTitle,
|
|
11370
12339
|
...path === void 0 ? {} : { path },
|
|
11371
12340
|
...existing === void 0 ? {} : { existing }
|
|
11372
12341
|
};
|
|
@@ -11507,16 +12476,19 @@ var KnowledgeStore = class {
|
|
|
11507
12476
|
* searchable through every path (FTS, entity, semantic).
|
|
11508
12477
|
*/
|
|
11509
12478
|
async ingest(text, domain, source = DEFAULT_KB_SOURCE, title, sourceUri, encoding, gitAction = "ingest", converter, warnings) {
|
|
11510
|
-
|
|
12479
|
+
const safeDomain = normalizeWrite(domain);
|
|
12480
|
+
const safeSource = normalizeWrite(source);
|
|
12481
|
+
const safeSourceUri = sourceUri === void 0 ? void 0 : normalizeWrite(sourceUri);
|
|
12482
|
+
this.assertDomainAllowed(safeDomain);
|
|
11511
12483
|
if (text.length > MAX_DOC_CHARS) throw new Error(`文档过大:${text.length} 字符(上限 ${MAX_DOC_CHARS})`);
|
|
11512
12484
|
const chunks = chunkText(text, this.kbConfig.chunk_size, this.kbConfig.chunk_overlap);
|
|
11513
|
-
const docTitle = title ?? deriveDocTitle(text);
|
|
12485
|
+
const docTitle = normalizeWrite(title ?? deriveDocTitle(text));
|
|
11514
12486
|
const result = this.db.transaction(() => {
|
|
11515
|
-
const docId = this.docs.upsert(
|
|
12487
|
+
const docId = this.docs.upsert(safeDomain, safeSource, docTitle, safeSourceUri);
|
|
11516
12488
|
const oldChunkIds = this.chunks.idsForDoc(docId);
|
|
11517
12489
|
const reusable = this.reusableByHash(docId, this.vectorSpace());
|
|
11518
12490
|
this.chunks.deleteForDoc(docId);
|
|
11519
|
-
this.docs.touch(docId,
|
|
12491
|
+
this.docs.touch(docId, safeSourceUri);
|
|
11520
12492
|
return {
|
|
11521
12493
|
doc_id: docId,
|
|
11522
12494
|
oldChunkIds,
|
|
@@ -11524,13 +12496,13 @@ var KnowledgeStore = class {
|
|
|
11524
12496
|
idx,
|
|
11525
12497
|
text: chunk.text,
|
|
11526
12498
|
headingsPath: chunk.headingsPath,
|
|
11527
|
-
sourceRef: `${
|
|
12499
|
+
sourceRef: `${safeDomain}:${safeSource}:${docId}:${idx}`,
|
|
11528
12500
|
charStart: chunk.start,
|
|
11529
12501
|
charEnd: chunk.end
|
|
11530
12502
|
}))),
|
|
11531
12503
|
reusable
|
|
11532
12504
|
};
|
|
11533
|
-
})();
|
|
12505
|
+
}).immediate();
|
|
11534
12506
|
this.evictVectors(result.oldChunkIds);
|
|
11535
12507
|
const indexed = await this.indexChunks(result.newChunkIds, result.reusable);
|
|
11536
12508
|
return {
|
|
@@ -11541,10 +12513,10 @@ var KnowledgeStore = class {
|
|
|
11541
12513
|
...converter === void 0 ? {} : { converter },
|
|
11542
12514
|
...warnings === void 0 || warnings.length === 0 ? {} : { warnings },
|
|
11543
12515
|
...this.writeManagedFile(result.doc_id, {
|
|
11544
|
-
domain,
|
|
11545
|
-
source,
|
|
12516
|
+
domain: safeDomain,
|
|
12517
|
+
source: safeSource,
|
|
11546
12518
|
title: docTitle,
|
|
11547
|
-
sourceUri,
|
|
12519
|
+
sourceUri: safeSourceUri,
|
|
11548
12520
|
converter
|
|
11549
12521
|
}, text, gitAction)
|
|
11550
12522
|
};
|
|
@@ -11891,24 +12863,30 @@ var KnowledgeStore = class {
|
|
|
11891
12863
|
* document. A stamp is a TRIGGER, not a judgement: the caller re-ingests through `sync`, which
|
|
11892
12864
|
* hashes the body and decides. The first call only seeds the baseline and reports nothing —
|
|
11893
12865
|
* use `sync` once at startup if a process may have been down while the corpus changed.
|
|
12866
|
+
*
|
|
12867
|
+
* ASYNC because one `stat` per document is exactly the cost that has to leave the event loop to
|
|
12868
|
+
* something else once in a while: at 10k documents the walk blocked the host for >0.1 s per trip
|
|
12869
|
+
* (measured — performance review §7.5 / P7), and this runs after every tool result behind a 2 s
|
|
12870
|
+
* throttle. The baseline is still taken ATOMICALLY at the end, so a caller sees the same
|
|
12871
|
+
* all-or-nothing update as before.
|
|
11894
12872
|
*/
|
|
11895
|
-
corpusDrift() {
|
|
12873
|
+
async corpusDrift() {
|
|
11896
12874
|
const fileSet = this.docFiles.listPaths().join("\n");
|
|
11897
12875
|
const previous = this.corpusStamps;
|
|
11898
12876
|
const previousSet = this.corpusFileSet;
|
|
11899
12877
|
const stamps = /* @__PURE__ */ new Map();
|
|
11900
12878
|
const changed = [];
|
|
11901
12879
|
const missing = [];
|
|
11902
|
-
|
|
12880
|
+
await forEachYielding(this.docs.list(), (doc) => {
|
|
11903
12881
|
const path = this.docFiles.basePath(doc.domain, doc.source, doc.title);
|
|
11904
12882
|
const stamp = this.docFiles.stamp(path);
|
|
11905
12883
|
if (stamp === null) {
|
|
11906
12884
|
if (previous?.has(path) === true) missing.push(doc.doc_id);
|
|
11907
|
-
|
|
12885
|
+
return;
|
|
11908
12886
|
}
|
|
11909
12887
|
stamps.set(path, stamp);
|
|
11910
12888
|
if (previous !== null && previous.get(path) !== stamp) changed.push(doc.doc_id);
|
|
11911
|
-
}
|
|
12889
|
+
});
|
|
11912
12890
|
this.corpusStamps = stamps;
|
|
11913
12891
|
this.corpusFileSet = fileSet;
|
|
11914
12892
|
return {
|
|
@@ -11961,7 +12939,7 @@ var KnowledgeStore = class {
|
|
|
11961
12939
|
const docs = opts.docId === void 0 ? all : all.filter((doc) => doc.doc_id === opts.docId);
|
|
11962
12940
|
const stale = [];
|
|
11963
12941
|
const missing = [];
|
|
11964
|
-
|
|
12942
|
+
await forEachYielding(docs, (doc) => {
|
|
11965
12943
|
const state = this.docFiles.state(doc.doc_id, doc.domain, doc.source, doc.title);
|
|
11966
12944
|
const entry = {
|
|
11967
12945
|
doc_id: doc.doc_id,
|
|
@@ -11972,26 +12950,27 @@ var KnowledgeStore = class {
|
|
|
11972
12950
|
};
|
|
11973
12951
|
if (state.missing) missing.push(entry);
|
|
11974
12952
|
else if (state.stale) stale.push(entry);
|
|
11975
|
-
}
|
|
12953
|
+
});
|
|
11976
12954
|
const known = new Set(all.map((doc) => doc.doc_id));
|
|
11977
|
-
const orphans = opts.docId !== void 0 ? [] : this.docFiles.scan().filter((file) => file.doc_id === void 0 || !known.has(file.doc_id)).map((file) => file.path).sort();
|
|
12955
|
+
const orphans = opts.docId !== void 0 ? [] : (await this.docFiles.scan()).filter((file) => file.doc_id === void 0 || !known.has(file.doc_id)).map((file) => file.path).sort();
|
|
11978
12956
|
const baseCounts = /* @__PURE__ */ new Map();
|
|
11979
12957
|
for (const doc of all) {
|
|
11980
12958
|
const base = this.docFiles.basePath(doc.domain, doc.source, doc.title);
|
|
11981
12959
|
baseCounts.set(base, (baseCounts.get(base) ?? 0) + 1);
|
|
11982
12960
|
}
|
|
11983
|
-
const unclaimed =
|
|
12961
|
+
const unclaimed = [];
|
|
12962
|
+
await forEachYielding(missing, (entry) => {
|
|
11984
12963
|
const doc = docs.find((candidate) => candidate.doc_id === entry.doc_id);
|
|
11985
|
-
if (doc === void 0) return
|
|
12964
|
+
if (doc === void 0) return;
|
|
11986
12965
|
const base = this.docFiles.basePath(doc.domain, doc.source, doc.title);
|
|
11987
|
-
if ((baseCounts.get(base) ?? 0) !== 1) return
|
|
12966
|
+
if ((baseCounts.get(base) ?? 0) !== 1) return;
|
|
11988
12967
|
const parsed = this.docFiles.read(base);
|
|
11989
|
-
if (parsed === null || parsed.meta.doc_id !== void 0 || parsed.body.trim() === "") return
|
|
11990
|
-
|
|
12968
|
+
if (parsed === null || parsed.meta.doc_id !== void 0 || parsed.body.trim() === "") return;
|
|
12969
|
+
unclaimed.push({
|
|
11991
12970
|
doc_id: doc.doc_id,
|
|
11992
12971
|
title: doc.title,
|
|
11993
12972
|
path: base
|
|
11994
|
-
}
|
|
12973
|
+
});
|
|
11995
12974
|
});
|
|
11996
12975
|
if (opts.dryRun === true) return {
|
|
11997
12976
|
checked: docs.length,
|
|
@@ -12031,7 +13010,7 @@ var KnowledgeStore = class {
|
|
|
12031
13010
|
* `domain` scopes the rebuild to one knowledge domain. One corpus SELECT feeds both legs, and
|
|
12032
13011
|
* each row is skipped when its derivation is provably still valid (same text digest, same
|
|
12033
13012
|
* vector space, same extractor version). Without that, the only way to know whether a chunk
|
|
12034
|
-
* needed
|
|
13013
|
+
* needed mission was to redo the mission: an ONNX forward pass plus a jieba pass per chunk over the
|
|
12035
13014
|
* whole corpus, every time.
|
|
12036
13015
|
*
|
|
12037
13016
|
* `dryRun` answers "how much would this cost?" without writing anything — including without
|
|
@@ -12074,14 +13053,18 @@ var KnowledgeStore = class {
|
|
|
12074
13053
|
}
|
|
12075
13054
|
/** Chunk search over the knowledge corpus (semantic + FTS + entity, joint fusion). */
|
|
12076
13055
|
async search(query, opts) {
|
|
12077
|
-
|
|
13056
|
+
const result = await hybridSearch(this.hybridDeps(opts), {
|
|
12078
13057
|
query,
|
|
12079
13058
|
limit: opts?.limit,
|
|
12080
13059
|
overFetch: opts?.overFetch,
|
|
12081
13060
|
maxTokens: opts?.maxTokens,
|
|
12082
13061
|
queryVector: opts?.queryVector,
|
|
12083
|
-
recordStats: opts?.recordStats
|
|
12084
|
-
|
|
13062
|
+
recordStats: opts?.recordStats,
|
|
13063
|
+
floors: opts?.floors,
|
|
13064
|
+
...opts?.relaxLegs === void 0 ? {} : { relaxLegs: opts.relaxLegs }
|
|
13065
|
+
});
|
|
13066
|
+
opts?.onResult?.(result);
|
|
13067
|
+
return result.hits;
|
|
12085
13068
|
}
|
|
12086
13069
|
/**
|
|
12087
13070
|
* This store's half of the shared orchestration (`store/hybrid.ts`): the legs, the chunk-row load
|
|
@@ -12135,30 +13118,57 @@ var KnowledgeStore = class {
|
|
|
12135
13118
|
* matching corpus only makes `fuse` normalize and sort it (see `MemoryStore.searchLegs`).
|
|
12136
13119
|
*/
|
|
12137
13120
|
async searchLegs(opts, ctx) {
|
|
12138
|
-
|
|
13121
|
+
/**
|
|
13122
|
+
* Wrap one leg's floored scores. `raw` is the PRE-floor set: `capped` must be measured there,
|
|
13123
|
+
* because the relevance floor removes the tail anyway and re-deriving the flag afterwards would
|
|
13124
|
+
* erase the "cut at legCap" signal exactly when it bound.
|
|
13125
|
+
*/
|
|
13126
|
+
const leg = (scores, weight, name, dropped, raw) => ({
|
|
12139
13127
|
weight,
|
|
12140
13128
|
scores,
|
|
12141
|
-
capped: scores.size === ctx.legCap
|
|
13129
|
+
capped: (raw ?? scores).size === ctx.legCap,
|
|
13130
|
+
leg: name,
|
|
13131
|
+
droppedByFloor: dropped
|
|
12142
13132
|
});
|
|
13133
|
+
const ftsRaw = this.ftsPath(ctx.query, opts?.domain, opts?.source, ctx.legCap);
|
|
13134
|
+
const ftsFloored = applyTermFloor(ftsRaw, this.chunkTexts([...ftsRaw.keys()]), ctx.query, ctx.floors.fts);
|
|
12143
13135
|
return [
|
|
12144
|
-
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector).then((
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
13136
|
+
ctx.semAvail ? this.semanticPath(ctx.query, ctx.overFetch, opts, ctx.queryVector, ctx.onQueryVector).then((raw) => {
|
|
13137
|
+
const floored = applyScoreFloor(raw, ctx.floors.semantic);
|
|
13138
|
+
return {
|
|
13139
|
+
weight: ctx.weights.semantic,
|
|
13140
|
+
scores: floored.scores,
|
|
13141
|
+
leg: "semantic",
|
|
13142
|
+
droppedByFloor: floored.dropped
|
|
13143
|
+
};
|
|
13144
|
+
}) : {
|
|
12148
13145
|
weight: ctx.weights.semantic,
|
|
12149
|
-
scores: /* @__PURE__ */ new Map()
|
|
13146
|
+
scores: /* @__PURE__ */ new Map(),
|
|
13147
|
+
leg: "semantic",
|
|
13148
|
+
droppedByFloor: 0
|
|
12150
13149
|
},
|
|
12151
|
-
leg(
|
|
12152
|
-
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((
|
|
13150
|
+
leg(ftsFloored.scores, ctx.weights.fts, "fts", ftsFloored.dropped, ftsRaw),
|
|
13151
|
+
this.jaccardPath(ctx.query, opts?.domain, opts?.source, ctx.legCap).then((raw) => {
|
|
13152
|
+
const floored = applyScoreFloor(raw, ctx.floors.jaccard);
|
|
13153
|
+
return leg(floored.scores, ctx.weights.jaccard, "jaccard", floored.dropped, raw);
|
|
13154
|
+
})
|
|
12153
13155
|
];
|
|
12154
13156
|
}
|
|
13157
|
+
/** The texts of a candidate set, for the per-row FTS floor (one batched query). */
|
|
13158
|
+
chunkTexts(ids) {
|
|
13159
|
+
if (ids.length === 0) return /* @__PURE__ */ new Map();
|
|
13160
|
+
return new Map(this.chunks.hits(ids).map((row) => [row.chunk_id, row.text]));
|
|
13161
|
+
}
|
|
12155
13162
|
/**
|
|
12156
13163
|
* Semantic leg. `queryVector` short-circuits the encode when the caller already
|
|
12157
|
-
* encoded this exact query (the cross-store router does, for both stores at once)
|
|
13164
|
+
* encoded this exact query (the cross-store router does, for both stores at once); `onVector`
|
|
13165
|
+
* publishes the vector this leg actually used, so the relaxed retry pass can reuse it instead of
|
|
13166
|
+
* re-encoding (performance review §7.7 / P8).
|
|
12158
13167
|
*/
|
|
12159
|
-
async semanticPath(query, k, opts, queryVector) {
|
|
13168
|
+
async semanticPath(query, k, opts, queryVector, onVector) {
|
|
12160
13169
|
const vec = queryVector ?? await this.semantic.encode(query);
|
|
12161
13170
|
if (vec.length !== this.vstore.dim) throw new RetrievalInputError(`queryVector 维度不符:${vec.length} != ${this.vstore.dim}`);
|
|
13171
|
+
onVector?.(vec);
|
|
12162
13172
|
const topk = this.vstore.topk(vec, Math.max(50, k));
|
|
12163
13173
|
if (!topk.length) return /* @__PURE__ */ new Map();
|
|
12164
13174
|
if (!opts?.domain && !opts?.source) return new Map(topk.map((t) => [t.id, t.score]));
|
|
@@ -12355,6 +13365,8 @@ function normalizeMergedScores(pool) {
|
|
|
12355
13365
|
function crossQuery(memory, kb, opts) {
|
|
12356
13366
|
const limit = opts?.limit ?? 10;
|
|
12357
13367
|
const weights = memory.weights ?? DEGRADED_WEIGHTS;
|
|
13368
|
+
const floors = memory.floors;
|
|
13369
|
+
const droppedByFloor = mergeFloorDrops(memory.dropped_by_floor, opts?.kbDroppedByFloor);
|
|
12358
13370
|
let pool = [...memory.hits, ...kb];
|
|
12359
13371
|
if (opts?.kind === "fact") pool = pool.filter((h) => h.kind === "fact");
|
|
12360
13372
|
if (opts?.kind === "doc_chunk") pool = pool.filter((h) => h.kind === "doc_chunk");
|
|
@@ -12363,7 +13375,9 @@ function crossQuery(memory, kb, opts) {
|
|
|
12363
13375
|
if (pool.length === 0) return {
|
|
12364
13376
|
hits: [],
|
|
12365
13377
|
degraded: memory.degraded,
|
|
12366
|
-
weights
|
|
13378
|
+
weights,
|
|
13379
|
+
floors,
|
|
13380
|
+
dropped_by_floor: droppedByFloor
|
|
12367
13381
|
};
|
|
12368
13382
|
const scaled = normalizeMergedScores(pool);
|
|
12369
13383
|
scaled.sort((a, b) => b.score - a.score);
|
|
@@ -12371,19 +13385,25 @@ function crossQuery(memory, kb, opts) {
|
|
|
12371
13385
|
if (opts?.maxTokens === void 0) return {
|
|
12372
13386
|
hits,
|
|
12373
13387
|
degraded: memory.degraded,
|
|
12374
|
-
weights
|
|
13388
|
+
weights,
|
|
13389
|
+
floors,
|
|
13390
|
+
dropped_by_floor: droppedByFloor
|
|
12375
13391
|
};
|
|
12376
13392
|
if (!Number.isFinite(opts.maxTokens) || opts.maxTokens <= 0) return {
|
|
12377
13393
|
hits,
|
|
12378
13394
|
degraded: memory.degraded,
|
|
12379
|
-
weights
|
|
13395
|
+
weights,
|
|
13396
|
+
floors,
|
|
13397
|
+
dropped_by_floor: droppedByFloor
|
|
12380
13398
|
};
|
|
12381
13399
|
const budgeted = fitToTokenBudget(hits, { maxTokens: opts.maxTokens });
|
|
12382
13400
|
if (budgeted.truncated > 0) recordTruncation("output");
|
|
12383
13401
|
return {
|
|
12384
13402
|
hits: budgeted.kept,
|
|
12385
13403
|
degraded: memory.degraded,
|
|
12386
|
-
weights
|
|
13404
|
+
weights,
|
|
13405
|
+
floors,
|
|
13406
|
+
dropped_by_floor: droppedByFloor
|
|
12387
13407
|
};
|
|
12388
13408
|
}
|
|
12389
13409
|
//#endregion
|
|
@@ -12406,7 +13426,7 @@ function buildRuntime(opts) {
|
|
|
12406
13426
|
autoInstall: config.common.tools.auto_install,
|
|
12407
13427
|
logger
|
|
12408
13428
|
});
|
|
12409
|
-
logger.info(`runtime init: dataHome=${config.home} memory.db=${memDb} knowledge.db=${config.knowledge.db.path} semantic=${config.common.semantic.backend}/${config.common.semantic.local_model} rerank=${config.common.rerank.backend} vectorStore=${config.common.vectorStore.backend}`);
|
|
13429
|
+
logger.info(`runtime init: dataHome=${config.home} memory.db=${memDb} knowledge.db=${config.knowledge.db.path} semantic=${config.common.semantic.backend}/${config.common.semantic.local_model} rerank=${config.common.rerank.backend} vectorStore=${config.common.vectorStore.backend} sqlite=${describeSqlite()}`);
|
|
12410
13430
|
const memoryStore = openMemoryStore(memDb);
|
|
12411
13431
|
if (wasUpgraded(memoryStore.migration)) logger.info(`memory: ${describeMigrationOutcome(MEMORY_SCHEMA, memoryStore.migration)}`);
|
|
12412
13432
|
const db = memoryStore.db;
|
|
@@ -12418,6 +13438,12 @@ function buildRuntime(opts) {
|
|
|
12418
13438
|
const memory = new MemoryStore(db, config.common, semantic, vstore, reranker, memDb);
|
|
12419
13439
|
const knowledge = new KnowledgeStore(config.knowledge.db.path, config.common, config.knowledge, semantic, kbVstore, reranker, knowledgeConfigPath(config.home), logger);
|
|
12420
13440
|
logger.info(`runtime ready in ${Date.now() - started}ms (embeddings warm asynchronously)`);
|
|
13441
|
+
/**
|
|
13442
|
+
* Set by {@link AvantfRuntime.shutdown}. Exposed because a background job that the plugin starts
|
|
13443
|
+
* behind a startup gate (the entity sweep) can otherwise fire after the databases are closed and
|
|
13444
|
+
* log a spurious failure.
|
|
13445
|
+
*/
|
|
13446
|
+
let closed = false;
|
|
12421
13447
|
const api = {
|
|
12422
13448
|
config,
|
|
12423
13449
|
db,
|
|
@@ -12442,7 +13468,8 @@ function buildRuntime(opts) {
|
|
|
12442
13468
|
query: req.query,
|
|
12443
13469
|
category: req.category,
|
|
12444
13470
|
limit: req.limit,
|
|
12445
|
-
maxTokens: req.max_tokens
|
|
13471
|
+
maxTokens: req.max_tokens,
|
|
13472
|
+
floors: req.floors
|
|
12446
13473
|
});
|
|
12447
13474
|
case "ask": {
|
|
12448
13475
|
if (req.query) return this.memory.ask(req.query, req.limit ?? 10);
|
|
@@ -12534,28 +13561,62 @@ function buildRuntime(opts) {
|
|
|
12534
13561
|
const startedAt = Date.now();
|
|
12535
13562
|
const limit = req.limit ?? 10;
|
|
12536
13563
|
const queryVector = semantic.isAvailable() ? await semantic.encode(req.query) : void 0;
|
|
12537
|
-
|
|
12538
|
-
|
|
12539
|
-
|
|
12540
|
-
|
|
12541
|
-
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
12547
|
-
|
|
12548
|
-
|
|
12549
|
-
|
|
12550
|
-
|
|
12551
|
-
|
|
12552
|
-
|
|
12553
|
-
|
|
12554
|
-
|
|
12555
|
-
|
|
12556
|
-
|
|
12557
|
-
|
|
12558
|
-
|
|
13564
|
+
/**
|
|
13565
|
+
* ONE cross-store pass under ONE floor policy.
|
|
13566
|
+
*
|
|
13567
|
+
* The profile is pinned on both store calls (`'strict'` / `'loose'`) so neither store runs its
|
|
13568
|
+
* own relaxed fallback: only the MERGED result may decide whether relaxing is warranted,
|
|
13569
|
+
* otherwise a store that happens to be empty would inject its relaxed tail into an answer the
|
|
13570
|
+
* other store filled strictly.
|
|
13571
|
+
*
|
|
13572
|
+
* `relaxLegs` is the same "only the legs that actually dropped" rule the single-store retry
|
|
13573
|
+
* applies (`store/floors.ts`'s `droppedLegs`), read off the MERGED strict pass — relaxing a leg
|
|
13574
|
+
* that dropped nothing admits no candidate, so this only keeps the reported `floors` honest.
|
|
13575
|
+
*/
|
|
13576
|
+
const crossPass = async (profile, relaxLegs) => {
|
|
13577
|
+
let kbDroppedByFloor;
|
|
13578
|
+
const [memory, kb] = await Promise.all([this.memory.search({
|
|
13579
|
+
query: req.query,
|
|
13580
|
+
limit: limit * 3,
|
|
13581
|
+
track: false,
|
|
13582
|
+
recordStats: false,
|
|
13583
|
+
queryVector,
|
|
13584
|
+
maxTokens: 0,
|
|
13585
|
+
floors: profile,
|
|
13586
|
+
...relaxLegs === void 0 ? {} : { relaxLegs }
|
|
13587
|
+
}), this.knowledge.search(req.query, {
|
|
13588
|
+
domain: req.domain,
|
|
13589
|
+
source: req.source,
|
|
13590
|
+
limit: limit * 3,
|
|
13591
|
+
recordStats: false,
|
|
13592
|
+
queryVector,
|
|
13593
|
+
maxTokens: 0,
|
|
13594
|
+
floors: profile,
|
|
13595
|
+
...relaxLegs === void 0 ? {} : { relaxLegs },
|
|
13596
|
+
onResult: (r) => {
|
|
13597
|
+
kbDroppedByFloor = r.dropped_by_floor;
|
|
13598
|
+
}
|
|
13599
|
+
})]);
|
|
13600
|
+
return crossQuery(memory, kb, {
|
|
13601
|
+
limit,
|
|
13602
|
+
kind: req.kind ?? "all",
|
|
13603
|
+
domain: req.domain,
|
|
13604
|
+
source: req.source,
|
|
13605
|
+
maxTokens: req.max_tokens ?? config.common.retriever.max_output_tokens,
|
|
13606
|
+
...kbDroppedByFloor === void 0 ? {} : { kbDroppedByFloor }
|
|
13607
|
+
});
|
|
13608
|
+
};
|
|
13609
|
+
let result = await crossPass(req.floors === "loose" ? "loose" : "strict");
|
|
13610
|
+
if (req.floors === void 0 && result.hits.length === 0) {
|
|
13611
|
+
const dropped = result.dropped_by_floor ?? emptyFloorDrops();
|
|
13612
|
+
if (totalFloorDrops(dropped) > 0) {
|
|
13613
|
+
const loosened = await crossPass("loose", droppedLegs(dropped));
|
|
13614
|
+
if (loosened.hits.length > 0) result = {
|
|
13615
|
+
...loosened,
|
|
13616
|
+
relaxed: true
|
|
13617
|
+
};
|
|
13618
|
+
}
|
|
13619
|
+
}
|
|
12559
13620
|
const rerank = this.memory.rerankState();
|
|
12560
13621
|
recordRetrieval({
|
|
12561
13622
|
kind: "cross",
|
|
@@ -12563,12 +13624,15 @@ function buildRuntime(opts) {
|
|
|
12563
13624
|
latencyMs: Date.now() - startedAt,
|
|
12564
13625
|
semanticLive: semantic.isAvailable(),
|
|
12565
13626
|
rerankUsed: rerank.used,
|
|
12566
|
-
rerankFallback: rerank.fallback
|
|
13627
|
+
rerankFallback: rerank.fallback,
|
|
13628
|
+
...result.dropped_by_floor === void 0 ? {} : { droppedByFloor: result.dropped_by_floor }
|
|
12567
13629
|
});
|
|
12568
13630
|
this.memory.reinforce(result.hits.filter((h) => h.kind === "fact").map((h) => h.ref_id));
|
|
12569
13631
|
return result;
|
|
12570
13632
|
},
|
|
12571
13633
|
shutdown() {
|
|
13634
|
+
if (closed) return;
|
|
13635
|
+
closed = true;
|
|
12572
13636
|
memory.flushHealth();
|
|
12573
13637
|
vstore.flush?.();
|
|
12574
13638
|
kbVstore.flush?.();
|
|
@@ -12578,6 +13642,9 @@ function buildRuntime(opts) {
|
|
|
12578
13642
|
};
|
|
12579
13643
|
return {
|
|
12580
13644
|
...api,
|
|
13645
|
+
get closed() {
|
|
13646
|
+
return closed;
|
|
13647
|
+
},
|
|
12581
13648
|
remember: api.remember,
|
|
12582
13649
|
recall: api.recall,
|
|
12583
13650
|
admin: api.admin,
|
|
@@ -12682,7 +13749,8 @@ const descriptors = [
|
|
|
12682
13749
|
second_pred: optionalText.optional(),
|
|
12683
13750
|
category: optionalText.optional(),
|
|
12684
13751
|
limit: count.optional(),
|
|
12685
|
-
max_tokens: count.optional()
|
|
13752
|
+
max_tokens: count.optional(),
|
|
13753
|
+
floors: optionalText.optional()
|
|
12686
13754
|
})),
|
|
12687
13755
|
direct("admin", z.object({
|
|
12688
13756
|
action: optionalText.optional(),
|
|
@@ -12718,7 +13786,8 @@ const descriptors = [
|
|
|
12718
13786
|
domain: optionalText.optional(),
|
|
12719
13787
|
source: optionalText.optional(),
|
|
12720
13788
|
limit: count.optional(),
|
|
12721
|
-
max_tokens: count.optional()
|
|
13789
|
+
max_tokens: count.optional(),
|
|
13790
|
+
floors: optionalText.optional()
|
|
12722
13791
|
})),
|
|
12723
13792
|
direct("classifySource", z.object({ text: optionalText.optional() })),
|
|
12724
13793
|
direct("browseDir", z.object({ path: optionalText.optional() })),
|
|
@@ -12729,6 +13798,33 @@ const descriptors = [
|
|
|
12729
13798
|
target: optionalText.optional()
|
|
12730
13799
|
}))
|
|
12731
13800
|
];
|
|
13801
|
+
/**
|
|
13802
|
+
* The successful application envelope, with the wire revision stamped on.
|
|
13803
|
+
*
|
|
13804
|
+
* A helper rather than a literal at each return site (the shared `call` covers the five tool-backed
|
|
13805
|
+
* methods; the five UI-only methods return directly): "every answer carries the marker" is then a
|
|
13806
|
+
* property of the code, not of six copies staying in step. The result is structurally the contract's
|
|
13807
|
+
* `RemoteOk`.
|
|
13808
|
+
*/
|
|
13809
|
+
function wireOk(value) {
|
|
13810
|
+
return {
|
|
13811
|
+
ok: true,
|
|
13812
|
+
value,
|
|
13813
|
+
wire: 1
|
|
13814
|
+
};
|
|
13815
|
+
}
|
|
13816
|
+
/**
|
|
13817
|
+
* The failed application envelope, with the wire revision stamped on.
|
|
13818
|
+
*
|
|
13819
|
+
* Takes the failure body already built (`{ok:false, error}` / validation's `{ok:false,error,violations}`)
|
|
13820
|
+
* so the message and violations keep coming from their existing single sources.
|
|
13821
|
+
*/
|
|
13822
|
+
function wireErr(failure) {
|
|
13823
|
+
return {
|
|
13824
|
+
...failure,
|
|
13825
|
+
wire: 1
|
|
13826
|
+
};
|
|
13827
|
+
}
|
|
12732
13828
|
/** The zod schema a strict codec carries, whichever generation's codec type is in play. */
|
|
12733
13829
|
function codecSchema(codec) {
|
|
12734
13830
|
return codec.schema;
|
|
@@ -12792,13 +13888,13 @@ const hostContribution = {
|
|
|
12792
13888
|
* caller value → ④ `$AVANTF_HOME` → ② this configured value → `~/.avantf`. Handing it straight to the
|
|
12793
13889
|
* engine's explicit slot instead — which is what this plugin used to do — promotes ② above ④, and the
|
|
12794
13890
|
* same profile then resolves to two different directories depending on which half of the family reads
|
|
12795
|
-
* it: the
|
|
12796
|
-
*
|
|
13891
|
+
* it: the mission plugin resolves the same knob through its configured layer, so with `$AVANTF_HOME` set
|
|
13892
|
+
* mission read `<env>/prompts` while this plugin read `<config>/prompts`. That directory is SHARED (both
|
|
12797
13893
|
* plugins' editable prompt files live in it), so "the file you edited is read by one plugin and not
|
|
12798
13894
|
* the other" is a real failure, not a nuance. `AGENTS.md` 「边界与路径」 states the rule this
|
|
12799
13895
|
* implements: the configured value travels in the config layer, never in the explicit one.
|
|
12800
13896
|
*
|
|
12801
|
-
* The engine's own copy of the rule does the
|
|
13897
|
+
* The engine's own copy of the rule does the mission (it is the one the CLI and MCP resolve with, and
|
|
12802
13898
|
* `test/family_pin.spec.ts` pins it against the base kit's). This module exists so the plugin's USE of
|
|
12803
13899
|
* it is testable without the DSH harness: the entry module imports `@deepseek-ai/*` at load, which
|
|
12804
13900
|
* resolves only inside a harness workspace.
|
|
@@ -12973,7 +14069,7 @@ function registerCompatMegaphone(ctx, verdict, log, compat) {
|
|
|
12973
14069
|
heading: "记忆插件未加载:与当前 dsh 的兼容性检查未通过,插件已拒绝加载(不注册服务、工具、prompt 段与真实 typert face)。",
|
|
12974
14070
|
warningsLabel: "风险提示:",
|
|
12975
14071
|
warnings: verdict.warnings,
|
|
12976
|
-
fix: "
|
|
14072
|
+
fix: "修复:换到本插件声明兼容的 dsh 版本,或升级本插件到与当前 dsh 匹配的版本。",
|
|
12977
14073
|
logPointer: (prefix) => `完整诊断见宿主日志里 ${prefix} 开头的行。`
|
|
12978
14074
|
})
|
|
12979
14075
|
});
|
|
@@ -12995,6 +14091,94 @@ function verifyRegisteredFaces(input) {
|
|
|
12995
14091
|
toolNames: input.toolNames
|
|
12996
14092
|
});
|
|
12997
14093
|
}
|
|
14094
|
+
//#endregion
|
|
14095
|
+
//#region lib/types/reconcile.js
|
|
14096
|
+
/**
|
|
14097
|
+
* Corpus reconciliation: the throttled "did anything under `knowledge.docs.dir` change?" sweep behind
|
|
14098
|
+
* `tools/result`.
|
|
14099
|
+
*
|
|
14100
|
+
* WHY IT IS ITS OWN MODULE. The plugin reaches the corpus through tool calls, so a finished tool call
|
|
14101
|
+
* is the moment to ask the question — and the answer is cheap (`corpusDrift()` is one `stat` per
|
|
14102
|
+
* document, reading nothing), which is what makes it affordable after EVERY tool result. Two things
|
|
14103
|
+
* about it are easy to get wrong and are pinned by `test/reconcile.spec.ts` instead of by a mounted
|
|
14104
|
+
* plugin: the THROTTLE (with a trailing run — skipping alone would drop the LAST edit, the one that
|
|
14105
|
+
* matters) and the STOP rule (an in-flight pass must not keep writing after the plugin unmounted and
|
|
14106
|
+
* its databases closed). `index.ts` is the assembly hub; this is the rule.
|
|
14107
|
+
*
|
|
14108
|
+
* @module @avantf/dsh-mem/reconcile
|
|
14109
|
+
*/
|
|
14110
|
+
/**
|
|
14111
|
+
* Build the reconciler. All state is closed over here, so a fresh mount gets a fresh one — the
|
|
14112
|
+
* plugin can be unmounted and mounted again in one process.
|
|
14113
|
+
*/
|
|
14114
|
+
function createCorpusReconciler(deps) {
|
|
14115
|
+
const now = deps.now ?? Date.now;
|
|
14116
|
+
const schedule = deps.schedule ?? ((fn, ms) => setTimeout(fn, ms));
|
|
14117
|
+
const cancel = deps.cancel ?? ((handle) => {
|
|
14118
|
+
clearTimeout(handle);
|
|
14119
|
+
});
|
|
14120
|
+
let stopped = false;
|
|
14121
|
+
let running = false;
|
|
14122
|
+
let lastDriftAt = 0;
|
|
14123
|
+
let trailing = null;
|
|
14124
|
+
const request = async (full) => {
|
|
14125
|
+
if (stopped) return;
|
|
14126
|
+
if (running) return;
|
|
14127
|
+
if (!full) {
|
|
14128
|
+
const wait = deps.minIntervalMs - (now() - lastDriftAt);
|
|
14129
|
+
if (wait > 0) {
|
|
14130
|
+
if (trailing === null) {
|
|
14131
|
+
trailing = schedule(() => {
|
|
14132
|
+
trailing = null;
|
|
14133
|
+
request(false);
|
|
14134
|
+
}, wait);
|
|
14135
|
+
const unref = trailing.unref;
|
|
14136
|
+
if (typeof unref === "function") unref.call(trailing);
|
|
14137
|
+
}
|
|
14138
|
+
return;
|
|
14139
|
+
}
|
|
14140
|
+
lastDriftAt = now();
|
|
14141
|
+
}
|
|
14142
|
+
running = true;
|
|
14143
|
+
try {
|
|
14144
|
+
if (full) {
|
|
14145
|
+
await deps.corpus.sync({});
|
|
14146
|
+
if (stopped) return;
|
|
14147
|
+
await deps.corpus.corpusDrift();
|
|
14148
|
+
return;
|
|
14149
|
+
}
|
|
14150
|
+
const drift = await deps.corpus.corpusDrift();
|
|
14151
|
+
if (stopped) return;
|
|
14152
|
+
if (drift.changed.length === 0 && drift.missing.length === 0 && !drift.fileSetChanged) return;
|
|
14153
|
+
if (drift.fileSetChanged || drift.missing.length > 0) {
|
|
14154
|
+
await deps.corpus.sync({});
|
|
14155
|
+
if (stopped) return;
|
|
14156
|
+
return;
|
|
14157
|
+
}
|
|
14158
|
+
for (const docId of drift.changed) {
|
|
14159
|
+
await deps.corpus.sync({ docId });
|
|
14160
|
+
if (stopped) return;
|
|
14161
|
+
}
|
|
14162
|
+
} catch (error) {
|
|
14163
|
+
deps.onError(error);
|
|
14164
|
+
} finally {
|
|
14165
|
+
running = false;
|
|
14166
|
+
}
|
|
14167
|
+
};
|
|
14168
|
+
return {
|
|
14169
|
+
request,
|
|
14170
|
+
stop() {
|
|
14171
|
+
stopped = true;
|
|
14172
|
+
if (trailing !== null) {
|
|
14173
|
+
cancel(trailing);
|
|
14174
|
+
trailing = null;
|
|
14175
|
+
}
|
|
14176
|
+
},
|
|
14177
|
+
get running() {
|
|
14178
|
+
return running;
|
|
14179
|
+
}
|
|
14180
|
+
};
|
|
14181
|
+
}
|
|
12998
14182
|
const PACKAGE = "@avantf/dsh-plugin-base";
|
|
12999
14183
|
function parseVersion(input) {
|
|
13000
14184
|
const match = /^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/.exec(input.trim());
|
|
@@ -13443,6 +14627,12 @@ const PLUGIN = "mem";
|
|
|
13443
14627
|
const PANDOC_ITEM = "mem:pandoc";
|
|
13444
14628
|
const MODEL_ITEM = "mem:model";
|
|
13445
14629
|
/**
|
|
14630
|
+
* The default ONNX embedding repository — the one `semantic.local_model` defaults to and the one
|
|
14631
|
+
* {@link MODEL_FILES} describes. A DIFFERENT repo is a different file set, which is why the item
|
|
14632
|
+
* omits `spec.files` for it (see {@link managedModelSpec}).
|
|
14633
|
+
*/
|
|
14634
|
+
const DEFAULT_MODEL_REPO = "Xenova/bge-small-zh-v1.5";
|
|
14635
|
+
/**
|
|
13446
14636
|
* The files the local embedding runtime actually requests for the DEFAULT ONNX model.
|
|
13447
14637
|
*
|
|
13448
14638
|
* Measured, not guessed: with an empty cache, `@huggingface/transformers@4.x` resolving
|
|
@@ -13460,6 +14650,22 @@ const MODEL_FILES = [
|
|
|
13460
14650
|
];
|
|
13461
14651
|
/** Startup budget for the blocking set; the framework's default is the same. */
|
|
13462
14652
|
const DEADLINE_MS = 15e3;
|
|
14653
|
+
/**
|
|
14654
|
+
* The `mem:model` item's per-request download budget, declared explicitly on the item.
|
|
14655
|
+
*
|
|
14656
|
+
* It is the base's own default (`net.ts`'s `DEFAULT_REQUEST_TIMEOUT_MS`) made VISIBLE: an item that
|
|
14657
|
+
* declares no `timeoutMs` silently inherits it, which is exactly the state the review flagged (the
|
|
14658
|
+
* mechanism existed, no item used it, and a multi-gigabyte model on a slow link had no documented
|
|
14659
|
+
* budget). `0` would mean "no timeout"; that is not this plugin's call to make by default.
|
|
14660
|
+
*
|
|
14661
|
+
* WHY THE DEADLINE BELOW IS RAISED TO MATCH IT. The base checks every declaration for an INVERSION
|
|
14662
|
+
* (`timeoutMs > deadlineMs`): a per-request budget larger than the ensure deadline means a request
|
|
14663
|
+
* could outlive the budget that is supposed to bound it. Both mem items are `startup: 'background'`,
|
|
14664
|
+
* so they spend no part of the startup budget and the larger deadline costs no wall clock — but the
|
|
14665
|
+
* declaration has to be consistent, and a caller that sets its own (smaller) deadline must enlarge it
|
|
14666
|
+
* rather than ship the inversion.
|
|
14667
|
+
*/
|
|
14668
|
+
const MODEL_REQUEST_TIMEOUT_MS = 3e5;
|
|
13463
14669
|
/** This module's warning token, so one grep finds every environment-initialisation line. */
|
|
13464
14670
|
const OWN_PREFIX = "envinit:";
|
|
13465
14671
|
/**
|
|
@@ -13484,7 +14690,7 @@ function managedModelSpec(semantic, modelsRoot) {
|
|
|
13484
14690
|
if (!sameDir(semantic.cache_dir, modelsRoot)) return void 0;
|
|
13485
14691
|
return {
|
|
13486
14692
|
repo: semantic.local_model,
|
|
13487
|
-
files: semantic.local_model ===
|
|
14693
|
+
files: semantic.local_model === DEFAULT_MODEL_REPO ? MODEL_FILES : [],
|
|
13488
14694
|
endpoint: semantic.mirror
|
|
13489
14695
|
};
|
|
13490
14696
|
}
|
|
@@ -13499,10 +14705,18 @@ function sameDir(a, b) {
|
|
|
13499
14705
|
* and that is deliberate: setting it redirects a whole tree (data AND managed resources), which is
|
|
13500
14706
|
* exactly what a test or a sandboxed host needs. Unset — the normal case — the family root is the
|
|
13501
14707
|
* framework's documented default and the data home stays `~/.avantf`.
|
|
14708
|
+
*
|
|
14709
|
+
* The value is TRIMMED and `~`-EXPANDED, exactly like the family kit's `familyHome()` and the
|
|
14710
|
+
* engine's own `expandHome`. Returning the raw string was a silent divergence from the other three
|
|
14711
|
+
* copies of this rule: `AVANTF_HOME=~/x` reached the base's layout verbatim, so the provisioner
|
|
14712
|
+
* wrote pandoc/models into the RELATIVE path `<cwd>/~/x/…` (a directory literally named `~`) while
|
|
14713
|
+
* the engine expanded the same string and read `/home/<user>/x/…` — model re-downloaded, semantic
|
|
14714
|
+
* leg degraded, pandoc not found. The kit is loaded dynamically (never a static import, see the
|
|
14715
|
+
* module header), so the local `expandHome` from the contract is the one used here.
|
|
13502
14716
|
*/
|
|
13503
14717
|
function resolveHome() {
|
|
13504
|
-
const configured = process.env["AVANTF_HOME"];
|
|
13505
|
-
return configured !==
|
|
14718
|
+
const configured = process.env["AVANTF_HOME"]?.trim() ?? "";
|
|
14719
|
+
return configured !== "" ? expandHome$1(configured) : join(homedir(), ".avantf", "env");
|
|
13506
14720
|
}
|
|
13507
14721
|
/**
|
|
13508
14722
|
* The download gate. Off when EITHER switch is off:
|
|
@@ -13585,6 +14799,9 @@ function pandocItem(framework) {
|
|
|
13585
14799
|
*
|
|
13586
14800
|
* A missing model is a degraded semantic path, NOT a refused mount: both `onMissing` axes are
|
|
13587
14801
|
* `degrade`, and the engine falls back to FTS+entity.
|
|
14802
|
+
*
|
|
14803
|
+
* `policy.timeoutMs` is the per-request budget (see {@link MODEL_REQUEST_TIMEOUT_MS}); it rides into
|
|
14804
|
+
* the provider through the base's `policyFor` and is what `signalFor` reads.
|
|
13588
14805
|
*/
|
|
13589
14806
|
function modelItem(framework, model) {
|
|
13590
14807
|
return {
|
|
@@ -13603,7 +14820,8 @@ function modelItem(framework, model) {
|
|
|
13603
14820
|
atUse: "degrade"
|
|
13604
14821
|
},
|
|
13605
14822
|
startup: "background",
|
|
13606
|
-
schemaVersion: framework.ITEM_SCHEMA_VERSION
|
|
14823
|
+
schemaVersion: framework.ITEM_SCHEMA_VERSION,
|
|
14824
|
+
policy: { timeoutMs: MODEL_REQUEST_TIMEOUT_MS }
|
|
13607
14825
|
};
|
|
13608
14826
|
}
|
|
13609
14827
|
let loaded;
|
|
@@ -13692,13 +14910,14 @@ async function loadOnce(options) {
|
|
|
13692
14910
|
if (plan.pandoc) items.push(pandocItem(framework));
|
|
13693
14911
|
if (plan.model !== void 0) items.push(modelItem(framework, plan.model));
|
|
13694
14912
|
if (items.length === 0) return;
|
|
14913
|
+
const maxItemTimeoutMs = items.reduce((max, item) => Math.max(max, item.policy?.timeoutMs ?? 0), 0);
|
|
13695
14914
|
const resources = framework.createProvisioner({
|
|
13696
14915
|
home,
|
|
13697
14916
|
envinitRange: range,
|
|
13698
14917
|
logger: provisionLogger(options.log),
|
|
13699
14918
|
policy: {
|
|
13700
14919
|
autoDownload: autoDownload ? { [framework.MODEL_CACHE_KIND]: plan.modelAutoDownload } : false,
|
|
13701
|
-
deadlineMs,
|
|
14920
|
+
deadlineMs: Math.max(deadlineMs, maxItemTimeoutMs),
|
|
13702
14921
|
mirrors: { archive: [...plan.archiveMirrors] }
|
|
13703
14922
|
}
|
|
13704
14923
|
});
|
|
@@ -13752,6 +14971,15 @@ async function loadOnce(options) {
|
|
|
13752
14971
|
* that outlives the host (or a browser window that closes immediately) is normal. The returned
|
|
13753
14972
|
* `opener` is what the UI shows, so a wrong editor is diagnosable without reading the host log.
|
|
13754
14973
|
*
|
|
14974
|
+
* WINDOWS. `code` is `code.cmd` there, and Node refuses to spawn a `.cmd`/`.bat` without a shell
|
|
14975
|
+
* (CVE-2024-27980). That refusal is a synchronous `EINVAL`; the old `catch {}` swallowed it, so the
|
|
14976
|
+
* documented "prefer VS Code" step silently demoted every launch to the `cmd /c start` fallback and
|
|
14977
|
+
* the user was never told. Two things changed: the command is resolved to the file that actually
|
|
14978
|
+
* exists (`code` → `code.cmd`) and a batch shim goes through `shell: true`; and the catch now
|
|
14979
|
+
* SEPARATES "the editor is not installed here" (normal fallback — silent) from "the launch was
|
|
14980
|
+
* refused" (a defect — warned through the injectable `warn`, default stderr). A refused launch is
|
|
14981
|
+
* also carried in the thrown message, which the `openDoc` Remote returns to the UI.
|
|
14982
|
+
*
|
|
13755
14983
|
* @module open
|
|
13756
14984
|
*/
|
|
13757
14985
|
/**
|
|
@@ -13761,31 +14989,103 @@ async function loadOnce(options) {
|
|
|
13761
14989
|
* bare name silently loses the editor and falls back to the OS default application. `PATHEXT` is
|
|
13762
14990
|
* what the shell itself uses; the default list is the fallback when it is not exported.
|
|
13763
14991
|
*/
|
|
13764
|
-
function commandFileNames(command, platform = process.platform) {
|
|
14992
|
+
function commandFileNames(command, platform = process.platform, pathext = process.env["PATHEXT"]) {
|
|
13765
14993
|
if (platform !== "win32") return [command];
|
|
13766
|
-
const exts = (
|
|
14994
|
+
const exts = (pathext ?? ".COM;.EXE;.BAT;.CMD").split(";").filter((ext) => ext !== "");
|
|
13767
14995
|
return [
|
|
13768
14996
|
command,
|
|
13769
14997
|
...exts.map((ext) => `${command}${ext.toLowerCase()}`),
|
|
13770
14998
|
...exts.map((ext) => `${command}${ext}`)
|
|
13771
14999
|
];
|
|
13772
15000
|
}
|
|
13773
|
-
/**
|
|
13774
|
-
|
|
13775
|
-
|
|
13776
|
-
|
|
15001
|
+
/**
|
|
15002
|
+
* The file name of `command` that actually exists on `env.PATH`, or `undefined`.
|
|
15003
|
+
*
|
|
15004
|
+
* Returning the NAME (not a boolean) is what lets the launcher see `code.cmd` and know it holds a
|
|
15005
|
+
* batch shim. `[win]` — the win32 form is asserted from Linux with an injected platform/env.
|
|
15006
|
+
*/
|
|
15007
|
+
function resolveOnPath(command, platform = process.platform, env = process.env) {
|
|
15008
|
+
const path = env["PATH"];
|
|
15009
|
+
if (path === void 0 || path === "") return void 0;
|
|
13777
15010
|
for (const dir of path.split(delimiter)) {
|
|
13778
15011
|
if (dir === "") continue;
|
|
13779
|
-
for (const name of commandFileNames(command)) try {
|
|
15012
|
+
for (const name of commandFileNames(command, platform, env["PATHEXT"])) try {
|
|
13780
15013
|
accessSync(join(dir, name), constants.X_OK);
|
|
13781
|
-
return
|
|
15014
|
+
return name;
|
|
13782
15015
|
} catch {}
|
|
13783
15016
|
}
|
|
13784
|
-
|
|
15017
|
+
}
|
|
15018
|
+
/** True for a Windows batch shim — the one shape Node will not spawn without a shell. */
|
|
15019
|
+
function isWindowsBatchShim(command, platform = process.platform) {
|
|
15020
|
+
return platform === "win32" && /\.(cmd|bat)$/iu.test(command);
|
|
15021
|
+
}
|
|
15022
|
+
/**
|
|
15023
|
+
* Classify a synchronous spawn failure.
|
|
15024
|
+
*
|
|
15025
|
+
* `ENOENT` means the command is not there after all — the documented priority order just moves on,
|
|
15026
|
+
* with no warning. Everything else (the `EINVAL` a `.cmd` gets without a shell, `EACCES`, a bad
|
|
15027
|
+
* argument) is a wiring defect, and swallowing it is exactly how "open in VS Code" disappeared.
|
|
15028
|
+
*/
|
|
15029
|
+
function classifyLaunchFailure(error) {
|
|
15030
|
+
return error?.code === "ENOENT" ? "missing" : "refused";
|
|
15031
|
+
}
|
|
15032
|
+
/**
|
|
15033
|
+
* `cmd.exe` builds the line for `shell: true`, so anything that would split it has to be quoted.
|
|
15034
|
+
*
|
|
15035
|
+
* Wrapping alone is not enough: an embedded `"` would END the quoted run early and hand the rest of
|
|
15036
|
+
* the value to cmd as syntax (a `$EDITOR` like `C:\Program "Files"\ed.exe`, or a document under a
|
|
15037
|
+
* folder whose name holds a quote, breaks the launch). cmd's convention for a literal quote inside a
|
|
15038
|
+
* quoted argument is to DOUBLE it, so `a"b` becomes `"a""b"` — the same shape
|
|
15039
|
+
* `mission/scripts/build.mjs` `quoteForCmd` produces for its own `shell: true` line.
|
|
15040
|
+
* A value with nothing to quote is returned verbatim.
|
|
15041
|
+
*/
|
|
15042
|
+
function cmdQuote(value) {
|
|
15043
|
+
return /[\s"&|<>^()]/u.test(value) ? `"${value.replace(/"/gu, "\"\"")}"` : value;
|
|
15044
|
+
}
|
|
15045
|
+
/**
|
|
15046
|
+
* The concrete spawn shape for one candidate on `platform`.
|
|
15047
|
+
*
|
|
15048
|
+
* TWO Windows-only rules, both from the same root cause (CVE-2024-27980): a `.cmd`/`.bat` shim is
|
|
15049
|
+
* spawned through `shell: true`; and the last-resort fallback is `cmd /c start`, whose quoted path
|
|
15050
|
+
* must follow its own title argument or cmd treats the path AS the title. The plan is pure and
|
|
15051
|
+
* takes the platform, so the win32 shapes are asserted on Linux (`[win]`, real-machine re-check
|
|
15052
|
+
* still owed).
|
|
15053
|
+
*/
|
|
15054
|
+
function launchPlan(command, arg, platform = process.platform) {
|
|
15055
|
+
if (isWindowsBatchShim(command, platform)) return {
|
|
15056
|
+
command: cmdQuote(command),
|
|
15057
|
+
args: [cmdQuote(arg)],
|
|
15058
|
+
options: {
|
|
15059
|
+
detached: true,
|
|
15060
|
+
stdio: "ignore",
|
|
15061
|
+
shell: true
|
|
15062
|
+
}
|
|
15063
|
+
};
|
|
15064
|
+
if (command === "cmd") return {
|
|
15065
|
+
command: "cmd",
|
|
15066
|
+
args: [
|
|
15067
|
+
"/c",
|
|
15068
|
+
"start",
|
|
15069
|
+
"",
|
|
15070
|
+
`"${arg}"`
|
|
15071
|
+
],
|
|
15072
|
+
options: {
|
|
15073
|
+
detached: true,
|
|
15074
|
+
stdio: "ignore"
|
|
15075
|
+
}
|
|
15076
|
+
};
|
|
15077
|
+
return {
|
|
15078
|
+
command,
|
|
15079
|
+
args: [arg],
|
|
15080
|
+
options: {
|
|
15081
|
+
detached: true,
|
|
15082
|
+
stdio: "ignore"
|
|
15083
|
+
}
|
|
15084
|
+
};
|
|
13785
15085
|
}
|
|
13786
15086
|
/** Is this a WSL process (where a Windows-side opener is often the only one that works)? */
|
|
13787
|
-
function isWsl() {
|
|
13788
|
-
return
|
|
15087
|
+
function isWsl(platform, env) {
|
|
15088
|
+
return platform === "linux" && (env["WSL_DISTRO_NAME"] !== void 0 || env["WSL_INTEROP"] !== void 0);
|
|
13789
15089
|
}
|
|
13790
15090
|
/** The Windows form of a WSL path, or `undefined` when `wslpath` cannot answer. */
|
|
13791
15091
|
function windowsPath(path) {
|
|
@@ -13797,79 +15097,104 @@ function windowsPath(path) {
|
|
|
13797
15097
|
return;
|
|
13798
15098
|
}
|
|
13799
15099
|
}
|
|
13800
|
-
/** Candidates in the order above, resolved against
|
|
13801
|
-
function candidates(configured) {
|
|
15100
|
+
/** Candidates in the order above, resolved against the given machine. */
|
|
15101
|
+
function candidates(configured, platform, env) {
|
|
13802
15102
|
const list = [];
|
|
13803
|
-
const explicit = configured !== "" ? configured :
|
|
15103
|
+
const explicit = configured !== "" ? configured : env["AVANTF_EDITOR"] ?? "";
|
|
13804
15104
|
if (explicit !== "") list.push({
|
|
13805
15105
|
command: explicit,
|
|
13806
15106
|
arg: (path) => path
|
|
13807
15107
|
});
|
|
13808
|
-
const shellEditor =
|
|
15108
|
+
const shellEditor = env["VISUAL"] ?? env["EDITOR"] ?? "";
|
|
13809
15109
|
if (shellEditor !== "") list.push({
|
|
13810
15110
|
command: shellEditor,
|
|
13811
15111
|
arg: (path) => path
|
|
13812
15112
|
});
|
|
13813
|
-
for (const editor of ["code", "cursor"]) if (
|
|
15113
|
+
for (const editor of ["code", "cursor"]) if (resolveOnPath(editor, platform, env) !== void 0) list.push({
|
|
13814
15114
|
command: editor,
|
|
13815
15115
|
arg: (path) => path
|
|
13816
15116
|
});
|
|
13817
|
-
if (
|
|
15117
|
+
if (platform === "darwin") list.push({
|
|
13818
15118
|
command: "open",
|
|
13819
15119
|
arg: (path) => path
|
|
13820
15120
|
});
|
|
13821
|
-
if (
|
|
13822
|
-
if (
|
|
15121
|
+
if (platform === "linux") {
|
|
15122
|
+
if (resolveOnPath("xdg-open", platform, env) !== void 0) list.push({
|
|
13823
15123
|
command: "xdg-open",
|
|
13824
15124
|
arg: (path) => path
|
|
13825
15125
|
});
|
|
13826
|
-
if (isWsl()) list.push({
|
|
15126
|
+
if (isWsl(platform, env)) list.push({
|
|
13827
15127
|
command: "explorer.exe",
|
|
13828
15128
|
arg: (path) => windowsPath(path)
|
|
13829
15129
|
});
|
|
13830
15130
|
}
|
|
13831
|
-
if (
|
|
15131
|
+
if (platform === "win32") list.push({
|
|
13832
15132
|
command: "cmd",
|
|
15133
|
+
raw: true,
|
|
13833
15134
|
arg: (path) => path
|
|
13834
15135
|
});
|
|
13835
15136
|
return list;
|
|
13836
15137
|
}
|
|
13837
15138
|
/**
|
|
15139
|
+
* The file that will actually be spawned for `command`: a bare name is resolved on PATH (so
|
|
15140
|
+
* `code` becomes `code.cmd` on Windows), an explicit path is kept when it exists — and `undefined`
|
|
15141
|
+
* means the editor is simply not installed here.
|
|
15142
|
+
*/
|
|
15143
|
+
function resolveCommand(command, platform, env) {
|
|
15144
|
+
if (command.includes("/") || command.includes("\\")) return existsSync(command) ? command : void 0;
|
|
15145
|
+
return resolveOnPath(command, platform, env);
|
|
15146
|
+
}
|
|
15147
|
+
/**
|
|
13838
15148
|
* Launch the configured/detected tool on one document path.
|
|
13839
15149
|
*
|
|
13840
15150
|
* @param documentPath - the managed `.md` path.
|
|
13841
15151
|
* @param target - `file` opens the document, `dir` opens the directory that holds it.
|
|
13842
15152
|
* @param configured - `knowledge.open.editor` from config (empty ⇒ auto-detect).
|
|
15153
|
+
* @param deps - platform / env / spawn / warning sink, for tests.
|
|
13843
15154
|
* @throws when no candidate is available or every one of them fails to start.
|
|
13844
15155
|
*/
|
|
13845
|
-
function openDocumentPath(documentPath, target, configured) {
|
|
15156
|
+
function openDocumentPath(documentPath, target, configured, deps = {}) {
|
|
15157
|
+
const platform = deps.platform ?? process.platform;
|
|
15158
|
+
const env = deps.env ?? process.env;
|
|
15159
|
+
const launch = deps.spawn ?? ((command, args, options) => spawn(command, args, options));
|
|
15160
|
+
const warn = deps.warn ?? ((message) => {
|
|
15161
|
+
console.warn(message);
|
|
15162
|
+
});
|
|
13846
15163
|
const wanted = target === "dir" ? dirname(documentPath) : documentPath;
|
|
13847
15164
|
const tried = [];
|
|
13848
|
-
|
|
15165
|
+
const missing = [];
|
|
15166
|
+
const refused = [];
|
|
15167
|
+
for (const candidate of candidates(configured, platform, env)) {
|
|
13849
15168
|
const arg = candidate.arg(wanted);
|
|
13850
15169
|
if (arg === void 0) continue;
|
|
13851
|
-
|
|
15170
|
+
const command = candidate.raw === true ? candidate.command : resolveCommand(candidate.command, platform, env);
|
|
15171
|
+
if (command === void 0) {
|
|
15172
|
+
missing.push(candidate.command);
|
|
15173
|
+
continue;
|
|
15174
|
+
}
|
|
15175
|
+
tried.push(command);
|
|
15176
|
+
const plan = launchPlan(command, arg, platform);
|
|
13852
15177
|
try {
|
|
13853
|
-
(
|
|
13854
|
-
"/c",
|
|
13855
|
-
"start",
|
|
13856
|
-
"",
|
|
13857
|
-
`"${arg}"`
|
|
13858
|
-
], {
|
|
13859
|
-
detached: true,
|
|
13860
|
-
stdio: "ignore"
|
|
13861
|
-
}) : spawn(candidate.command, [arg], {
|
|
13862
|
-
detached: true,
|
|
13863
|
-
stdio: "ignore"
|
|
13864
|
-
})).unref();
|
|
15178
|
+
launch(plan.command, plan.args, plan.options).unref();
|
|
13865
15179
|
return {
|
|
13866
15180
|
path: wanted,
|
|
13867
15181
|
target,
|
|
13868
|
-
opener:
|
|
15182
|
+
opener: command
|
|
13869
15183
|
};
|
|
13870
|
-
} catch {
|
|
15184
|
+
} catch (error) {
|
|
15185
|
+
if (classifyLaunchFailure(error) === "missing") {
|
|
15186
|
+
tried.pop();
|
|
15187
|
+
missing.push(command);
|
|
15188
|
+
continue;
|
|
15189
|
+
}
|
|
15190
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
15191
|
+
refused.push(`${command}(${reason})`);
|
|
15192
|
+
warn(`[avantf-mem] 无法用 ${command} 打开「${wanted}」:${reason};改用下一个候选`);
|
|
15193
|
+
}
|
|
13871
15194
|
}
|
|
13872
|
-
throw new Error(
|
|
15195
|
+
if (tried.length === 0) throw new Error(missing.length === 0 ? "没有可用的编辑器:设置 knowledge.open.editor(或 $EDITOR / $AVANTF_EDITOR),或安装 code / xdg-open" : `没有可用的编辑器:${missing.join("、")} 不存在(PATH 上没有该命令,或该路径不可用)`);
|
|
15196
|
+
const detail = refused.length === 0 ? "" : `;启动被拒绝:${refused.join("、")}`;
|
|
15197
|
+
throw new Error(`无法启动编辑器(试过:${tried.join("、")}${detail})`);
|
|
13873
15198
|
}
|
|
13874
15199
|
//#endregion
|
|
13875
15200
|
//#region lib/types/tool_schema.js
|
|
@@ -14008,6 +15333,19 @@ function flattenToolSpec(spec) {
|
|
|
14008
15333
|
return properties;
|
|
14009
15334
|
}
|
|
14010
15335
|
//#endregion
|
|
15336
|
+
//#region lib/types/render.js
|
|
15337
|
+
/** The tool output spec registered on every mem tool. */
|
|
15338
|
+
const OUTPUT = {
|
|
15339
|
+
schema: {
|
|
15340
|
+
type: "object",
|
|
15341
|
+
additionalProperties: true
|
|
15342
|
+
},
|
|
15343
|
+
render: (_args, value) => [{
|
|
15344
|
+
type: "text",
|
|
15345
|
+
text: JSON.stringify(toWellFormedDeep(value), null, 2)
|
|
15346
|
+
}]
|
|
15347
|
+
};
|
|
15348
|
+
//#endregion
|
|
14011
15349
|
//#region lib/types/inject.js
|
|
14012
15350
|
/**
|
|
14013
15351
|
* The services this plugin injects.
|
|
@@ -14092,16 +15430,6 @@ async function runToolSpec(spec, rt, args) {
|
|
|
14092
15430
|
return toolErr(error);
|
|
14093
15431
|
}
|
|
14094
15432
|
}
|
|
14095
|
-
const OUTPUT = {
|
|
14096
|
-
schema: {
|
|
14097
|
-
type: "object",
|
|
14098
|
-
additionalProperties: true
|
|
14099
|
-
},
|
|
14100
|
-
render: (_args, value) => [{
|
|
14101
|
-
type: "text",
|
|
14102
|
-
text: JSON.stringify(value, null, 2)
|
|
14103
|
-
}]
|
|
14104
|
-
};
|
|
14105
15433
|
function present(spec, args) {
|
|
14106
15434
|
return {
|
|
14107
15435
|
card: "generic",
|
|
@@ -14119,6 +15447,11 @@ function defined(args) {
|
|
|
14119
15447
|
* re-validated with the contract zod union before dispatch (same guarantee the
|
|
14120
15448
|
* tools give the model), and results are wrapped in the contract's
|
|
14121
15449
|
* `{ok:true,value}` / `{ok:false,error,violations}` RemoteEnvelope the client decodes.
|
|
15450
|
+
*
|
|
15451
|
+
* Every envelope — success and failure, tool-backed and UI-only — is built through `wireOk`/`wireErr`
|
|
15452
|
+
* so it carries this build's `WIRE_VERSION`. That is what lets a freshly reloaded browser half tell
|
|
15453
|
+
* whether the host process predates a method it is about to call, instead of reading the gateway's
|
|
15454
|
+
* 404 as "the record is gone" (see `remote.ts`).
|
|
14122
15455
|
*/
|
|
14123
15456
|
let AvantfMemGateway = (() => {
|
|
14124
15457
|
let _classSuper = TypertRemoteService;
|
|
@@ -14280,17 +15613,14 @@ let AvantfMemGateway = (() => {
|
|
|
14280
15613
|
*/
|
|
14281
15614
|
async call(key, spec, payload) {
|
|
14282
15615
|
const parsed = spec.input.safeParse(payload);
|
|
14283
|
-
if (!parsed.success) return validationError(spec.name, parsed.error.issues);
|
|
15616
|
+
if (!parsed.success) return wireErr(validationError(spec.name, parsed.error.issues));
|
|
14284
15617
|
try {
|
|
14285
|
-
return
|
|
14286
|
-
ok: true,
|
|
14287
|
-
value: await dispatchToolKey(this.rt, key, parsed.data) ?? null
|
|
14288
|
-
};
|
|
15618
|
+
return wireOk(await dispatchToolKey(this.rt, key, parsed.data) ?? null);
|
|
14289
15619
|
} catch (error) {
|
|
14290
|
-
return {
|
|
15620
|
+
return wireErr({
|
|
14291
15621
|
ok: false,
|
|
14292
15622
|
error: remoteErrorText(error)
|
|
14293
|
-
};
|
|
15623
|
+
});
|
|
14294
15624
|
}
|
|
14295
15625
|
}
|
|
14296
15626
|
/**
|
|
@@ -14339,30 +15669,24 @@ let AvantfMemGateway = (() => {
|
|
|
14339
15669
|
async classifySource(args) {
|
|
14340
15670
|
try {
|
|
14341
15671
|
const payload = defined(args);
|
|
14342
|
-
return
|
|
14343
|
-
ok: true,
|
|
14344
|
-
value: classifySource(typeof payload.text === "string" ? payload.text : "", this.rt.config.knowledge.ingest)
|
|
14345
|
-
};
|
|
15672
|
+
return wireOk(classifySource(typeof payload.text === "string" ? payload.text : "", this.rt.config.knowledge.ingest));
|
|
14346
15673
|
} catch (error) {
|
|
14347
|
-
return {
|
|
15674
|
+
return wireErr({
|
|
14348
15675
|
ok: false,
|
|
14349
15676
|
error: remoteErrorText(error)
|
|
14350
|
-
};
|
|
15677
|
+
});
|
|
14351
15678
|
}
|
|
14352
15679
|
}
|
|
14353
15680
|
/** UI-only: one directory listing for the 选择 picker (same boundary as ingestion). */
|
|
14354
15681
|
async browseDir(args) {
|
|
14355
15682
|
try {
|
|
14356
15683
|
const payload = defined(args);
|
|
14357
|
-
return
|
|
14358
|
-
ok: true,
|
|
14359
|
-
value: browseDirectory(typeof payload.path === "string" && payload.path !== "" ? payload.path : void 0, this.rt.config.knowledge.ingest)
|
|
14360
|
-
};
|
|
15684
|
+
return wireOk(browseDirectory(typeof payload.path === "string" && payload.path !== "" ? payload.path : void 0, this.rt.config.knowledge.ingest));
|
|
14361
15685
|
} catch (error) {
|
|
14362
|
-
return {
|
|
15686
|
+
return wireErr({
|
|
14363
15687
|
ok: false,
|
|
14364
15688
|
error: remoteErrorText(error)
|
|
14365
|
-
};
|
|
15689
|
+
});
|
|
14366
15690
|
}
|
|
14367
15691
|
}
|
|
14368
15692
|
/**
|
|
@@ -14372,15 +15696,12 @@ let AvantfMemGateway = (() => {
|
|
|
14372
15696
|
*/
|
|
14373
15697
|
async kbDomains(_args) {
|
|
14374
15698
|
try {
|
|
14375
|
-
return
|
|
14376
|
-
ok: true,
|
|
14377
|
-
value: this.rt.knowledge.domainCatalog()
|
|
14378
|
-
};
|
|
15699
|
+
return wireOk(this.rt.knowledge.domainCatalog());
|
|
14379
15700
|
} catch (error) {
|
|
14380
|
-
return {
|
|
15701
|
+
return wireErr({
|
|
14381
15702
|
ok: false,
|
|
14382
15703
|
error: remoteErrorText(error)
|
|
14383
|
-
};
|
|
15704
|
+
});
|
|
14384
15705
|
}
|
|
14385
15706
|
}
|
|
14386
15707
|
/**
|
|
@@ -14392,15 +15713,12 @@ let AvantfMemGateway = (() => {
|
|
|
14392
15713
|
try {
|
|
14393
15714
|
const payload = defined(args);
|
|
14394
15715
|
const domain = typeof payload.domain === "string" ? payload.domain : "";
|
|
14395
|
-
return
|
|
14396
|
-
ok: true,
|
|
14397
|
-
value: this.rt.knowledge.addDomain(domain)
|
|
14398
|
-
};
|
|
15716
|
+
return wireOk(this.rt.knowledge.addDomain(domain));
|
|
14399
15717
|
} catch (error) {
|
|
14400
|
-
return {
|
|
15718
|
+
return wireErr({
|
|
14401
15719
|
ok: false,
|
|
14402
15720
|
error: remoteErrorText(error)
|
|
14403
|
-
};
|
|
15721
|
+
});
|
|
14404
15722
|
}
|
|
14405
15723
|
}
|
|
14406
15724
|
/**
|
|
@@ -14413,29 +15731,26 @@ let AvantfMemGateway = (() => {
|
|
|
14413
15731
|
try {
|
|
14414
15732
|
const payload = defined(args);
|
|
14415
15733
|
const docId = typeof payload.doc_id === "number" ? payload.doc_id : void 0;
|
|
14416
|
-
if (docId === void 0) return {
|
|
15734
|
+
if (docId === void 0) return wireErr({
|
|
14417
15735
|
ok: false,
|
|
14418
15736
|
error: "openDoc 需要 doc_id"
|
|
14419
|
-
};
|
|
15737
|
+
});
|
|
14420
15738
|
const path = this.rt.knowledge.docFilePath(docId);
|
|
14421
|
-
if (path === null) return {
|
|
15739
|
+
if (path === null) return wireErr({
|
|
14422
15740
|
ok: false,
|
|
14423
15741
|
error: `文档 #${String(docId)} 不存在`
|
|
14424
|
-
};
|
|
15742
|
+
});
|
|
14425
15743
|
const target = payload.target === "dir" ? "dir" : "file";
|
|
14426
|
-
if (target === "file" && !existsSync(path)) return {
|
|
15744
|
+
if (target === "file" && !existsSync(path)) return wireErr({
|
|
14427
15745
|
ok: false,
|
|
14428
15746
|
error: `受管文件不存在:${path}(重新摄入会重建它)`
|
|
14429
|
-
};
|
|
14430
|
-
return
|
|
14431
|
-
ok: true,
|
|
14432
|
-
value: openDocumentPath(path, target, this.rt.config.knowledge.open.editor)
|
|
14433
|
-
};
|
|
15747
|
+
});
|
|
15748
|
+
return wireOk(openDocumentPath(path, target, this.rt.config.knowledge.open.editor));
|
|
14434
15749
|
} catch (error) {
|
|
14435
|
-
return {
|
|
15750
|
+
return wireErr({
|
|
14436
15751
|
ok: false,
|
|
14437
15752
|
error: remoteErrorText(error)
|
|
14438
|
-
};
|
|
15753
|
+
});
|
|
14439
15754
|
}
|
|
14440
15755
|
}
|
|
14441
15756
|
};
|
|
@@ -14552,6 +15867,8 @@ async function apply(ctx, config) {
|
|
|
14552
15867
|
logger.warn("unmounted while preparing the environment; stopping before registering anything");
|
|
14553
15868
|
return;
|
|
14554
15869
|
}
|
|
15870
|
+
const wellFormed = adoptWellFormed(env?.kit);
|
|
15871
|
+
logger.info(wellFormed.text && wellFormed.deep ? "well-formed repair: the base kit's wellFormedText/wellFormedDeep (interface v2)" : "well-formed repair: the mem mirror (the base is absent, or predates the v2 helpers)");
|
|
14555
15872
|
const compat = env?.compat;
|
|
14556
15873
|
let verdict;
|
|
14557
15874
|
if (compat === void 0) logger.warn("plugin mount: the dsh compatibility gate is ABSENT (@avantf/dsh-plugin-base unavailable — see the WARNING above); mounting the full plugin anyway");
|
|
@@ -14559,7 +15876,7 @@ async function apply(ctx, config) {
|
|
|
14559
15876
|
const run = provision(ctx, logger, compat);
|
|
14560
15877
|
if (!run.verdict.load) {
|
|
14561
15878
|
registerCompatMegaphone(ctx, run.verdict, logger, compat);
|
|
14562
|
-
logger.warn("plugin not loaded: the host dsh API is incompatible (compat check above) — nothing was registered;
|
|
15879
|
+
logger.warn("plugin not loaded: the host dsh API is incompatible (compat check above) — nothing was registered; switch to a dsh version this package declares compatible, or upgrade this plugin");
|
|
14563
15880
|
return;
|
|
14564
15881
|
}
|
|
14565
15882
|
verdict = run.verdict;
|
|
@@ -14612,7 +15929,14 @@ async function apply(ctx, config) {
|
|
|
14612
15929
|
logger.warn(`entity sweep failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
14613
15930
|
});
|
|
14614
15931
|
};
|
|
14615
|
-
|
|
15932
|
+
awaitStartupGate({
|
|
15933
|
+
...waitFor === void 0 ? {} : { waitFor },
|
|
15934
|
+
deferTokenizerUntilIdle: true
|
|
15935
|
+
}).then(() => {
|
|
15936
|
+
if (!rt.closed && stillActive(ctx)) sweepEntities();
|
|
15937
|
+
}).catch((error) => {
|
|
15938
|
+
logger.warn(`entity sweep gate failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
15939
|
+
});
|
|
14616
15940
|
const heartbeatMinutes = rt.config.common.trust.presence.heartbeat_minutes;
|
|
14617
15941
|
if (heartbeatMinutes > 0) ctx.effect(() => {
|
|
14618
15942
|
const timer = setInterval(() => {
|
|
@@ -14638,7 +15962,8 @@ async function apply(ctx, config) {
|
|
|
14638
15962
|
wrote: false
|
|
14639
15963
|
})) : new kit.PromptFiles({
|
|
14640
15964
|
dir: promptDir,
|
|
14641
|
-
logger
|
|
15965
|
+
logger,
|
|
15966
|
+
namespace: "mem"
|
|
14642
15967
|
}).load(promptFileSpecs());
|
|
14643
15968
|
const promptSections = buildPromptSections(loadedPrompts);
|
|
14644
15969
|
for (const section of promptSections) {
|
|
@@ -14668,45 +15993,20 @@ async function apply(ctx, config) {
|
|
|
14668
15993
|
}
|
|
14669
15994
|
});
|
|
14670
15995
|
logger.info("prompt context registered: avantf:mem-hint (order 130)");
|
|
14671
|
-
|
|
14672
|
-
|
|
14673
|
-
|
|
14674
|
-
|
|
14675
|
-
|
|
14676
|
-
|
|
14677
|
-
|
|
14678
|
-
if (wait > 0) {
|
|
14679
|
-
if (trailingDrift === null) trailingDrift = setTimeout(() => {
|
|
14680
|
-
trailingDrift = null;
|
|
14681
|
-
reconcile(false);
|
|
14682
|
-
}, wait);
|
|
14683
|
-
return;
|
|
14684
|
-
}
|
|
14685
|
-
lastDriftAt = Date.now();
|
|
14686
|
-
}
|
|
14687
|
-
reconciling = true;
|
|
14688
|
-
try {
|
|
14689
|
-
if (full) {
|
|
14690
|
-
await rt.knowledge.sync({});
|
|
14691
|
-
rt.knowledge.corpusDrift();
|
|
14692
|
-
return;
|
|
14693
|
-
}
|
|
14694
|
-
const drift = rt.knowledge.corpusDrift();
|
|
14695
|
-
if (drift.changed.length === 0 && drift.missing.length === 0 && !drift.fileSetChanged) return;
|
|
14696
|
-
if (drift.fileSetChanged || drift.missing.length > 0) {
|
|
14697
|
-
await rt.knowledge.sync({});
|
|
14698
|
-
return;
|
|
14699
|
-
}
|
|
14700
|
-
for (const docId of drift.changed) await rt.knowledge.sync({ docId });
|
|
14701
|
-
} catch (error) {
|
|
15996
|
+
const reconciler = createCorpusReconciler({
|
|
15997
|
+
corpus: {
|
|
15998
|
+
sync: (options) => rt.knowledge.sync(options),
|
|
15999
|
+
corpusDrift: () => rt.knowledge.corpusDrift()
|
|
16000
|
+
},
|
|
16001
|
+
minIntervalMs: RECONCILE_MIN_INTERVAL_MS,
|
|
16002
|
+
onError: (error) => {
|
|
14702
16003
|
logger.warn(`知识库自动同步失败(不影响已有内容):${error instanceof Error ? error.message : String(error)}`);
|
|
14703
|
-
} finally {
|
|
14704
|
-
reconciling = false;
|
|
14705
16004
|
}
|
|
14706
|
-
};
|
|
14707
|
-
|
|
16005
|
+
});
|
|
16006
|
+
ctx.effect(() => () => reconciler.stop());
|
|
16007
|
+
reconciler.request(true);
|
|
14708
16008
|
ctx.on("tools/result", () => {
|
|
14709
|
-
|
|
16009
|
+
reconciler.request(false);
|
|
14710
16010
|
});
|
|
14711
16011
|
for (const spec of toolSpecs) {
|
|
14712
16012
|
ctx.tools.register(defineTool({
|
|
@@ -14739,4 +16039,4 @@ async function apply(ctx, config) {
|
|
|
14739
16039
|
logger.info(`plugin ready: ${toolSpecs.length} tools + avantfMemory service + avantfMem remote`);
|
|
14740
16040
|
}
|
|
14741
16041
|
//#endregion
|
|
14742
|
-
export { AvantfMemGateway, Config, KNOWLEDGE_PROMPT_SECTION, MEMORY_PROMPT_SECTION, apply, inject, name };
|
|
16042
|
+
export { AvantfMemGateway, Config, KNOWLEDGE_PROMPT_SECTION, MEMORY_PROMPT_SECTION, apply, inject, name, registerReranker, registerSemanticBackend, registerVectorStore };
|