open-memex 0.11.0-alpha.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +13 -3
- package/CHANGELOG.md +3 -1
- package/README.md +22 -4
- package/README.zh-CN.md +7 -2
- package/dist/.build-stamp +1 -1
- package/dist/cli.js +31 -1
- package/dist/config.js +17 -0
- package/dist/doctor.js +49 -1
- package/dist/retrieve/embedding.js +191 -0
- package/dist/retrieve/rerank.js +74 -0
- package/dist/retrieve/search.js +10 -4
- package/dist/store/curation-feedback.js +10 -10
- package/dist/store/db.js +15 -0
- package/dist/store/embeddings.js +75 -0
- package/dist/tools/ops.js +6 -1
- package/docs/V2-DESIGN.md +7 -4
- package/docs/design/decisions.md +3 -0
- package/docs/retrieval-ablation-study.md +11 -0
- package/package.json +66 -66
- package/scripts/bench-semantic.ts +376 -111
- package/scripts/smoke-pure.ts +48 -1
- package/server.json +2 -2
- package/src/cli.ts +31 -1
- package/src/config.ts +28 -0
- package/src/doctor.ts +48 -1
- package/src/retrieve/embedding.ts +214 -0
- package/src/retrieve/rerank.ts +96 -0
- package/src/retrieve/search.ts +11 -4
- package/src/store/curate.ts +675 -675
- package/src/store/curation-feedback.ts +81 -81
- package/src/store/db.ts +16 -0
- package/src/store/embeddings.ts +82 -0
- package/src/tools/ops.ts +6 -1
package/AGENTS.md
CHANGED
|
@@ -54,6 +54,8 @@ npm run cli -- resolve [id-or-path] # list / 3-way-merg
|
|
|
54
54
|
npm run mcp # start the stdio MCP server
|
|
55
55
|
node --experimental-strip-types scripts\smoke-pure.ts # runs pure-logic checks (no sqlite)
|
|
56
56
|
node --experimental-strip-types scripts\smoke-mcp.ts # MCP handshake + tool round-trip (temp dirs, no real data)
|
|
57
|
+
node --experimental-strip-types scripts\bench-semantic.ts --arms bm25 # D96 gate bench, lexical arm only
|
|
58
|
+
node --experimental-strip-types scripts\bench-semantic.ts --cache-dir <dir> # adds the model arms (needs `npm i --no-save @huggingface/transformers`)
|
|
57
59
|
```
|
|
58
60
|
|
|
59
61
|
After `npm i -g open-memex` (or `npm link` from source), the `open-memex` bin is on
|
|
@@ -228,9 +230,9 @@ through the normal flow, not write them.
|
|
|
228
230
|
|
|
229
231
|
## Design constraints — read the frozen design first
|
|
230
232
|
|
|
231
|
-
`docs/V2-DESIGN.md` is the frozen protocol v0.2 (decisions settled through
|
|
233
|
+
`docs/V2-DESIGN.md` is the frozen protocol v0.2 (decisions settled through D96 — log in `docs/design/decisions.md`; open questions tracked at the end of the doc). Per its §12:
|
|
232
234
|
AGENTS.md answers "how should AI work here"; the design doc answers "why is it
|
|
233
|
-
built this way" (principles, iron rules, the append-only decision log in `docs/design/decisions.md`, currently D1–
|
|
235
|
+
built this way" (principles, iron rules, the append-only decision log in `docs/design/decisions.md`, currently D1–D96). Before changing
|
|
234
236
|
architecture, scope semantics, lifecycle, or the protocol surface (frontmatter
|
|
235
237
|
schema, MCP tools, CLI contract), read the relevant design section — the decision
|
|
236
238
|
log records what was already considered and rejected.
|
|
@@ -238,7 +240,15 @@ log records what was already considered and rejected.
|
|
|
238
240
|
Still hard: no cloud, no silent sync (explicit pull only), Markdown is the source
|
|
239
241
|
of truth, `personal` scope never leaves the machine. Embeddings are an *optional
|
|
240
242
|
capability* per the design — do not add them (or LLM-driven extraction, or a
|
|
241
|
-
knowledge graph) without updating the design doc first.
|
|
243
|
+
knowledge graph) without updating the design doc first. **D96 is the shape that shipped, and it is
|
|
244
|
+
narrow on purpose:** BM25 retrieves, embeddings may only *rerank* EN<->ZH (cross-script) hits, inside
|
|
245
|
+
one review tier, inside a contiguous run, above `DEFAULT_RERANK_MIN_COSINE`; `crossLingualRerank` is
|
|
246
|
+
default off; the runtime (`@huggingface/transformers`) is deliberately not a dependency, so a missing
|
|
247
|
+
package must degrade to BM25, never fail a search; vectors live in `embedding_vectors` (keyed by
|
|
248
|
+
`(model, sha256(text))`, never in frontmatter, never dropped by a rebuild). Do not turn this into a
|
|
249
|
+
retriever, a default, or a general RRF/hybrid layer without a new gate - RRF scores better on the
|
|
250
|
+
fixture and is still out of scope, because reordering same-language hits is a different decision (see
|
|
251
|
+
D96 and `docs/retrieval-ablation-study.md` §4.7). `docs/V2-DESIGN.md` §18 tracks the
|
|
242
252
|
build roadmap; the design doc tracks the *why*.
|
|
243
253
|
|
|
244
254
|
## Branch workflow
|
package/CHANGELOG.md
CHANGED
|
@@ -7,7 +7,9 @@ rationale behind each decision lives in the decision log in
|
|
|
7
7
|
[docs/V2-DESIGN.md](docs/V2-DESIGN.md). Pre-release builds published under
|
|
8
8
|
the `@alpha` tag are development snapshots and are not listed individually.
|
|
9
9
|
|
|
10
|
-
## [0.11.0
|
|
10
|
+
## [0.11.0] - 2026-11-20
|
|
11
|
+
|
|
12
|
+
- D96 embedding gate + opt-in cross-lingual rerank (EN↔ZH only, §18 item 4). **The gate first:** `scripts/bench-semantic.ts` was re-run on current code (now with `--dtype`, `--arms`, `--cache-dir`, a tax block and an explicit cross-script slice) with no retrieval change; its BM25 arm reproduces the published baseline exactly (60 memories / 47 queries, feedback off: R@1 0.660 / R@5 0.979 / MRR 0.804). Adding a local multilingual model does **not** beat BM25 as a retriever (vec-only 76.6 / 91.5 / 0.832; naive RRF 78.7 / 97.9 / 0.864), and it does not move the pooled 22,419-session ceiling — but on the six EN↔ZH queries a **BM25-first rerank** lifts R@1 from 0% to 33.3% and MRR from 0.431 to 0.597 while leaving the 41 same-language queries byte-identical. Verdict: **PASS as an opt-in rerank, FAIL as a default or a retriever**, so it ships off by default as `crossLingualRerank` (flat key, `config set`) with `embeddingModel` for the model id. When on, BM25 still chooses the hits; the local model may only reorder cross-script hits inside a contiguous run, inside one review tier, above a cosine floor — the BM25 score is never modified and a hit with too little signal keeps its exact slot. The model (q8 MiniLM, **112.8 MB**, 0.76 s warm load, 497 MB RSS, **3.1 ms per query**) downloads into the data root on first opt-in use; `@huggingface/transformers` is deliberately **not** a dependency — `npm install -g open-memex @huggingface/transformers` enables it, and without it search degrades to BM25 with a warning instead of failing. Text vectors cache in SQLite (`embedding_vectors`, 1.5 KB/vector, never dropped by a rebuild), and a same-language search never loads a model at all. `search --explain` shows whether the rerank ran, the eligible hits and every promotion with its cosine; `open-memex doctor` reports the flag, the package resolution and the cached vector count. No new MCP tool (stays 13), no new CLI verb, no general fusion and no write-path embeddings — an RRF/general-hybrid decision needs its own gate.
|
|
11
13
|
|
|
12
14
|
- D78 pre-submit curation, second slice (D95): submit now has a real **batch-confirm** tier next to the deterministic one. Semantic merges (near-duplicates whose similarity is >= 0.9 and whose lengths are within 60% - 0.8-0.9 stays report-only, because one flipped word lands there), **splits** of an overlong memory whose paragraphs share < 25% vocabulary (accepting keeps BOTH halves: the original keeps the first, a new memory with `derived_from` takes the second and rides the same submit), and uncertain **type/tag inferences** all appear as ONE summary screen. With proposals and no decision flag, submit stops **before writing anything** and prints the summary; `--accept-batch` applies the whole batch, `--decline-batch` changes none of it (never per-item approval). The auto tier gained high-confidence type/tag inference (e.g. an explicit `TODO:`, `We decided`, `#hashtag`) that applies **only** to keyword captures, where the hook never chose a type or tag - a tool/CLI save may have chosen it deliberately, so it goes to the batch screen instead, and an explicit non-default type is never touched. Tags are additive only, capped at 3 per memory. Every action stays reversible: `open-memex curate undo <id>` reverses the latest one, and the new `open-memex curate undo --batch <id>` reverses a whole accepted batch by replaying its exact receipts (a split undo rejoins the halves and retracts the derived part). Accepted/declined/undone counts are recorded per signal in a local `curation_feedback` table (created with `CREATE TABLE IF NOT EXISTS` next to the D77 counters, never dropped by a rebuild) and printed by `open-memex curate stats` - data for future threshold learning only, no threshold changes yet. The generated `gh pr create` body now carries the batch decision too. Fixture (`scripts/curate-eval.ts`) pins the new cases before and after: no-flag run leaves the outbox byte-identical, accepted batch = 5 auto actions + 4 batch items + 1 split part, declined batch changes nothing, and one command reverses the batch. MCP `memory_submit` gains `curateBatch="accept"|"decline"` and returns the summary as a normal result, so an agent asks once for the whole batch.
|
|
13
15
|
|
package/README.md
CHANGED
|
@@ -658,9 +658,21 @@ e.g. Banana Plan → payment system refactor project — and a query mentioning
|
|
|
658
658
|
triggers a second round on the other side, both directions, merged evenly
|
|
659
659
|
with round one so alias hits are never down-weighted; no reindex when the
|
|
660
660
|
alias changes); and **semantic vectors**
|
|
661
|
-
as an opt-in
|
|
662
|
-
|
|
663
|
-
|
|
661
|
+
as an opt-in rerank (D96): with `crossLingualRerank` on, a search whose
|
|
662
|
+
results include a memory written in the other script (English query → Chinese
|
|
663
|
+
memory, or the reverse) may reorder those hits by local embedding similarity.
|
|
664
|
+
BM25 still retrieves — the model only reorders, only inside one review tier,
|
|
665
|
+
only inside a contiguous run, and only above a similarity floor; the BM25
|
|
666
|
+
score is never edited and a hit with too little signal keeps its exact slot.
|
|
667
|
+
Measured on the checked-in fixture (feedback weighting off): **MRR 0.804 →
|
|
668
|
+
0.825** overall, and on the six cross-lingual queries **R@1 0% → 33.3%, MRR
|
|
669
|
+
0.431 → 0.597**, with the 41 same-language queries unchanged. The tax is
|
|
670
|
+
published, not hidden: a quantized multilingual MiniLM is **112.8 MB**, loads
|
|
671
|
+
in ~0.76 s, uses ~497 MB RSS and adds ~3.1 ms per query. That is why it is off
|
|
672
|
+
by default, why the model downloads only after you opt in (into the data root,
|
|
673
|
+
never the repo), and why open-memex itself carries no embedding dependency.
|
|
674
|
+
A same-language search never loads a model. Full numbers and the gate verdict:
|
|
675
|
+
`docs/design/decisions.md` D96 and `docs/retrieval-ablation-study.md` §4.7.
|
|
664
676
|
|
|
665
677
|
External check on public data: the same pipeline scores **recall@5 97.0% /
|
|
666
678
|
MRR 0.909** on LongMemEval-S (470 questions, retrieval stage only — no answer
|
|
@@ -773,6 +785,8 @@ Defaults:
|
|
|
773
785
|
"graphOutline": true, // inject the compiled graph outline at session start
|
|
774
786
|
"graphOutlineLines": 10, // outline length cap (max 30)
|
|
775
787
|
"feedbackWeighting": false, // D79: reorder near-tied hits by reuse_rate (opt-in)
|
|
788
|
+
"crossLingualRerank": false, // D96: EN<->ZH embedding rerank (opt-in; downloads a model)
|
|
789
|
+
"embeddingModel": "Xenova/paraphrase-multilingual-MiniLM-L12-v2", // D96
|
|
776
790
|
"memoryDir": ".ai/open-memex" // in-repo project-memory dir, relative to repo root
|
|
777
791
|
}
|
|
778
792
|
```
|
|
@@ -786,11 +800,15 @@ open-memex config set maxProjectMemories 12
|
|
|
786
800
|
open-memex config set sync.autoPull true # best-effort pull at MCP session start
|
|
787
801
|
open-memex config set graphOutline false # stop injecting the graph outline
|
|
788
802
|
open-memex config set feedbackWeighting true # D79: let reuse signal break near-ties
|
|
803
|
+
# D96: opt-in EN<->ZH rerank. Install the runtime once, then flip the flag:
|
|
804
|
+
npm install -g open-memex @huggingface/transformers
|
|
805
|
+
open-memex config set crossLingualRerank true
|
|
789
806
|
```
|
|
790
807
|
|
|
791
808
|
Settable keys: `maxProjectMemories`, `maxProfileItems`, `injectOnFirstTurn`,
|
|
792
809
|
`keywordCaptureEnabled`, `captureAliases`, `logLevel`, `memoryDir`,
|
|
793
|
-
`graphOutline`, `graphOutlineLines`, `feedbackWeighting`,
|
|
810
|
+
`graphOutline`, `graphOutlineLines`, `feedbackWeighting`, `crossLingualRerank`,
|
|
811
|
+
`embeddingModel`, and `sync.autoPull`
|
|
794
812
|
(a dotted key that writes into the nested `sync` object). Full design:
|
|
795
813
|
[docs/V2-DESIGN.md](./docs/V2-DESIGN.md).
|
|
796
814
|
|
package/README.zh-CN.md
CHANGED
|
@@ -397,7 +397,7 @@ open-memex 的会话从已有上下文开始,而不是从提问开始:
|
|
|
397
397
|
|
|
398
398
|
可选的反馈加权(D79 第一刀,**默认关**)用 D77 复用计数器打破 BM25 的近平分:一条至少有 3 次检索记录、且 `reuse_rate` 更高的记忆,会在同一个 25% 分数段内排到前面。它不改 BM25 分数、不跨评审层级、不动样本不足的命中(新记忆保持原 BM25 位置);有用性仍只来自显式信号,被动曝光永远不算。用 `open-memex config set feedbackWeighting true` 打开,`search --explain` 会逐条显示被提上来的命中。
|
|
399
399
|
|
|
400
|
-
三层检索,各补各的短板:**BM25 关键词检索**(SQLite FTS5——精确词匹配,快速,可用 `search --explain` 解释);**同义词展开分两半**:(a)index-time(D80——一份随仓库提交的精选映射表:开发领域同义词 + EN↔ZH 翻译对,在写入时展开进索引,`ship` 能找到 `deploy`、`单点登录` 能找到 `SSO`,查询侧只需要普通单轮;确定性,不下载模型);(b)query-time 用户别名(D81——带 `alias:`/`target:` frontmatter 的别名记忆,每个 scope 定义一次自己的词汇,比如香蕉计划→支付系统重构项目;查询提到任一侧就触发对另一侧的第二轮搜索,双向,与第一轮等权重合并,别名命中不降权;别名变更不用 reindex);**语义向量**作为 opt-in
|
|
400
|
+
三层检索,各补各的短板:**BM25 关键词检索**(SQLite FTS5——精确词匹配,快速,可用 `search --explain` 解释);**同义词展开分两半**:(a)index-time(D80——一份随仓库提交的精选映射表:开发领域同义词 + EN↔ZH 翻译对,在写入时展开进索引,`ship` 能找到 `deploy`、`单点登录` 能找到 `SSO`,查询侧只需要普通单轮;确定性,不下载模型);(b)query-time 用户别名(D81——带 `alias:`/`target:` frontmatter 的别名记忆,每个 scope 定义一次自己的词汇,比如香蕉计划→支付系统重构项目;查询提到任一侧就触发对另一侧的第二轮搜索,双向,与第一轮等权重合并,别名命中不降权;别名变更不用 reindex);**语义向量**作为 opt-in 重排(D96):打开 `crossLingualRerank` 后,如果搜索结果里出现用另一种书写系统写的记忆(英文查询 → 中文记忆,或反过来),这些命中可以按本地 embedding 相似度重排。BM25 仍然是召回方——模型只重排,只在同一个评审层级内、只在连续的一段里、只在相似度过线时;BM25 分数永不修改,信号不足的命中保持原位置。在仓库内的 fixture 上实测(反馈加权关闭):整体 **MRR 0.804 → 0.825**;六条真正跨语言的查询 **R@1 0% → 33.3%、MRR 0.431 → 0.597**;41 条同语言查询完全不变。代价是公开的:量化多语言 MiniLM **112.8 MB**、加载约 0.76 秒、占用约 497 MB RSS、每条查询多约 3.1 毫秒。所以它默认关、模型只在你 opt-in 之后才下载(下载到数据根目录,绝不进仓库),open-memex 本身也不带任何 embedding 依赖。同语言查询根本不会加载模型。完整数字与 gate 结论见 `docs/design/decisions.md` D96 与 `docs/retrieval-ablation-study.md` §4.7。
|
|
401
401
|
|
|
402
402
|
公开数据上的外部验证:同一管线在 LongMemEval-S 上拿到 **recall@5 97.0% / MRR 0.909**(470 个问题,只测检索阶段——不生成答案、不用 judge 模型;这不是 LongMemEval 官方分数)。同协议下 agentmemory 公布的数字是 86.2% / 0.715(纯 BM25)和 95.2% / 0.882(BM25+向量)。22k 条记忆的 pooled 压力测试把 recall@5 拉到 40.6%——词法检索的天花板,这次是测出来的。用 `node --experimental-strip-types scripts/bench-longmemeval.ts --data <路径>` 复现(数据集 `xiaowu0162/longmemeval-cleaned`,MIT 协议)。
|
|
403
403
|
|
|
@@ -462,6 +462,8 @@ npm install -g open-memex@latest # 或 @alpha
|
|
|
462
462
|
"graphOutline": true, // 会话开始时注入图谱编译大纲
|
|
463
463
|
"graphOutlineLines": 10, // 大纲长度上限(最大 30)
|
|
464
464
|
"feedbackWeighting": false, // D79:按 reuse_rate 调整近平分命中顺序(默认关)
|
|
465
|
+
"crossLingualRerank": false, // D96:EN↔ZH embedding 重排(opt-in;会下载模型)
|
|
466
|
+
"embeddingModel": "Xenova/paraphrase-multilingual-MiniLM-L12-v2", // D96
|
|
465
467
|
"memoryDir": ".ai/open-memex" // 仓库内项目记忆目录,相对于仓库根
|
|
466
468
|
}
|
|
467
469
|
```
|
|
@@ -474,9 +476,12 @@ open-memex config set maxProjectMemories 12
|
|
|
474
476
|
open-memex config set sync.autoPull true # MCP 会话开始时尽力而为地 pull
|
|
475
477
|
open-memex config set graphOutline false # 不再注入图谱大纲
|
|
476
478
|
open-memex config set feedbackWeighting true # D79:让复用信号打破近平分
|
|
479
|
+
# D96:opt-in 的 EN↔ZH 重排。先装运行时,再打开开关:
|
|
480
|
+
npm install -g open-memex @huggingface/transformers
|
|
481
|
+
open-memex config set crossLingualRerank true
|
|
477
482
|
```
|
|
478
483
|
|
|
479
|
-
可设置的键:`maxProjectMemories`、`maxProfileItems`、`injectOnFirstTurn`、`keywordCaptureEnabled`、`captureAliases`、`logLevel`、`memoryDir`、`graphOutline`、`graphOutlineLines`、`feedbackWeighting`,以及 `sync.autoPull`(带点的键,写入嵌套的 `sync` 对象)。完整设计:[docs/V2-DESIGN.md](./docs/V2-DESIGN.md)(英文)。
|
|
484
|
+
可设置的键:`maxProjectMemories`、`maxProfileItems`、`injectOnFirstTurn`、`keywordCaptureEnabled`、`captureAliases`、`logLevel`、`memoryDir`、`graphOutline`、`graphOutlineLines`、`feedbackWeighting`、`crossLingualRerank`、`embeddingModel`,以及 `sync.autoPull`(带点的键,写入嵌套的 `sync` 对象)。完整设计:[docs/V2-DESIGN.md](./docs/V2-DESIGN.md)(英文)。
|
|
480
485
|
|
|
481
486
|
## CLI 参考
|
|
482
487
|
|
package/dist/.build-stamp
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
6c7376515aaea200659929294bca99bfcb645a1591f2af662661a6a4383d8cb2
|
package/dist/cli.js
CHANGED
|
@@ -10,6 +10,8 @@ import { proposeMemories, promoteMemory, listConflicts, resolveConflict, formatR
|
|
|
10
10
|
import { getSyncStatus, formatSyncStatus, submitMemories, BatchDecisionRequired } from "./submit.js";
|
|
11
11
|
import { getPrStatus, formatPrStatus, applyPrStatus } from "./github.js";
|
|
12
12
|
import { search, list, countList, hitStateLabel } from "./retrieve/search.js";
|
|
13
|
+
import { applyCrossLingualRerank, embeddingModelDir } from "./retrieve/embedding.js";
|
|
14
|
+
import { DEFAULT_RERANK_MIN_COSINE } from "./retrieve/rerank.js";
|
|
13
15
|
import { recordUseful, getUsage, reuseRate, usefulCount, usefulKindFor, deleteUsage, loadReuseReport } from "./store/usage.js";
|
|
14
16
|
import { formatReuseReport } from "./reuse-report.js";
|
|
15
17
|
import { DEFAULT_EXPANSIONS_WEIGHT } from "./retrieve/weights.js";
|
|
@@ -1178,7 +1180,7 @@ async function main() {
|
|
|
1178
1180
|
const stats = explain
|
|
1179
1181
|
? { ftsQuery: "", candidates: 0, hiddenSuperseded: 0, hiddenExcluded: 0, aliasExpansions: [] }
|
|
1180
1182
|
: undefined;
|
|
1181
|
-
const
|
|
1183
|
+
const found = search(query, {
|
|
1182
1184
|
scopeKeys: keys,
|
|
1183
1185
|
limit: flags.limit ? Number(flags.limit) : undefined,
|
|
1184
1186
|
type: flags.type,
|
|
@@ -1186,6 +1188,10 @@ async function main() {
|
|
|
1186
1188
|
record: true,
|
|
1187
1189
|
feedback: cfg.feedbackWeighting,
|
|
1188
1190
|
});
|
|
1191
|
+
// D96: opt-in cross-lingual rerank — reorders only what BM25 returned, and
|
|
1192
|
+
// only hits written in the other script (see src/retrieve/rerank.ts).
|
|
1193
|
+
const rerank = await applyCrossLingualRerank(query, found, cfg);
|
|
1194
|
+
const hits = rerank.hits;
|
|
1189
1195
|
if (explain && stats) {
|
|
1190
1196
|
console.log(`fts: ${stats.ftsQuery || "(empty query)"}`);
|
|
1191
1197
|
console.log(`candidates: ${stats.candidates}, hidden by lifecycle: ${stats.hiddenSuperseded} superseded, ${stats.hiddenExcluded} retracted/archived`);
|
|
@@ -1212,6 +1218,30 @@ async function main() {
|
|
|
1212
1218
|
}
|
|
1213
1219
|
}
|
|
1214
1220
|
}
|
|
1221
|
+
// D96: report whether the cross-lingual rerank ran, which hits were
|
|
1222
|
+
// eligible, and every hit it lifted. The BM25 score is never changed,
|
|
1223
|
+
// so the score column stays the raw lexical value.
|
|
1224
|
+
if (!cfg.crossLingualRerank) {
|
|
1225
|
+
console.log("cross-lingual rerank: off (default; enable with: open-memex config set crossLingualRerank true)");
|
|
1226
|
+
}
|
|
1227
|
+
else if (rerank.skipped === "no-cross-lingual-hits") {
|
|
1228
|
+
console.log("cross-lingual rerank: on, no cross-script hits in this result (nothing to rerank)");
|
|
1229
|
+
}
|
|
1230
|
+
else if (rerank.skipped === "model-unavailable") {
|
|
1231
|
+
console.log("cross-lingual rerank: on, but the local embedding runtime is unavailable — BM25 order kept");
|
|
1232
|
+
}
|
|
1233
|
+
else {
|
|
1234
|
+
console.log(`cross-lingual rerank: on (model ${cfg.embeddingModel}, min cosine ${DEFAULT_RERANK_MIN_COSINE}, models in ${embeddingModelDir()})`);
|
|
1235
|
+
console.log(` candidates: ${rerank.candidates} cross-script hit(s) in ${hits.length} result(s)`);
|
|
1236
|
+
if (rerank.promotions.length === 0) {
|
|
1237
|
+
console.log(" no hits moved by embeddings");
|
|
1238
|
+
}
|
|
1239
|
+
else {
|
|
1240
|
+
for (const p of rerank.promotions) {
|
|
1241
|
+
console.log(` ↑ ${p.id} +${p.promoted} (cosine ${p.cosine.toFixed(3)})`);
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1215
1245
|
// D83: which indexed column each hit matched on. `expansion` means the
|
|
1216
1246
|
// hit only surfaced through a D80 derived variant, so it ranks below an
|
|
1217
1247
|
// equal body match by construction (BM25_WEIGHTS) — this is how a
|
package/dist/config.js
CHANGED
|
@@ -12,6 +12,8 @@ export const DEFAULT_CONFIG = {
|
|
|
12
12
|
graphOutline: true,
|
|
13
13
|
graphOutlineLines: 10,
|
|
14
14
|
feedbackWeighting: false,
|
|
15
|
+
crossLingualRerank: false,
|
|
16
|
+
embeddingModel: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
|
|
15
17
|
keywordPatterns: [
|
|
16
18
|
"^\\s*remember(?!\\s+for\\s+me)(?:\\s+that|\\s+to)?[:,]?\\s+(.+)$",
|
|
17
19
|
"^\\s*(?:please\\s+)?note(?:\\s+that)?[:,]?\\s+(.+)$",
|
|
@@ -148,6 +150,19 @@ function toRelativeDir(v) {
|
|
|
148
150
|
throw new Error('must be a relative path without ".." (e.g. ".ai/open-memex")');
|
|
149
151
|
return t;
|
|
150
152
|
}
|
|
153
|
+
function toModelId(v) {
|
|
154
|
+
if (typeof v !== "string" || !v.trim())
|
|
155
|
+
throw new Error("must be a model id (e.g. Xenova/all-MiniLM-L6-v2)");
|
|
156
|
+
const t = v.trim();
|
|
157
|
+
// The id becomes a directory name under the model cache, so it must not be
|
|
158
|
+
// able to escape it: at most one "/", every segment starts alphanumeric
|
|
159
|
+
// (which rules out ".", "..", and absolute paths).
|
|
160
|
+
const segment = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
161
|
+
const parts = t.split("/");
|
|
162
|
+
if (parts.length > 2 || !parts.every((p) => segment.test(p)))
|
|
163
|
+
throw new Error('must be a Hugging Face model id like "Xenova/all-MiniLM-L6-v2"');
|
|
164
|
+
return t;
|
|
165
|
+
}
|
|
151
166
|
/** Keys users may change via `open-memex config set <key> <value>`, with validators. */
|
|
152
167
|
export const SETTABLE_KEYS = {
|
|
153
168
|
maxProjectMemories: toNonNegInt,
|
|
@@ -164,6 +179,8 @@ export const SETTABLE_KEYS = {
|
|
|
164
179
|
},
|
|
165
180
|
graphOutline: toBool,
|
|
166
181
|
feedbackWeighting: toBool,
|
|
182
|
+
crossLingualRerank: toBool,
|
|
183
|
+
embeddingModel: toModelId,
|
|
167
184
|
graphOutlineLines: (v) => {
|
|
168
185
|
const n = toNonNegInt(v);
|
|
169
186
|
if (n < 1)
|
package/dist/doctor.js
CHANGED
|
@@ -12,6 +12,9 @@ import { paths, homeOverride } from "./paths.js";
|
|
|
12
12
|
import { resolveCwdScope } from "./scope.js";
|
|
13
13
|
import { MIN_NODE_VERSION, nativeDriverFloorMessage, nodeSupportsNativeDriver, } from "./runtime-floor.js";
|
|
14
14
|
import { db, backendName } from "./store/db.js";
|
|
15
|
+
import { countVectors } from "./store/embeddings.js";
|
|
16
|
+
import { embeddingModelDir } from "./retrieve/embedding.js";
|
|
17
|
+
import { createRequire } from "node:module";
|
|
15
18
|
import { LEGACY_PACKAGE_NAME, opencodeGlobalConfigPath, userMcpConfigPath } from "./init.js";
|
|
16
19
|
// D64: the MCP server exposes thirteen tools; checking only the seven the
|
|
17
20
|
// opencode plugin carries let a half-registered server pass as healthy.
|
|
@@ -572,9 +575,54 @@ function vscodeMcpCheck() {
|
|
|
572
575
|
: "VS Code settings not found on this machine — nothing to check",
|
|
573
576
|
};
|
|
574
577
|
}
|
|
578
|
+
/**
|
|
579
|
+
* D96: the cross-lingual rerank is opt-in and depends on an optional package
|
|
580
|
+
* the default install deliberately does not pull. When the flag is on, say
|
|
581
|
+
* plainly whether that package resolves and how much is cached — a silent
|
|
582
|
+
* downgrade to BM25 is the failure mode this check exists to catch.
|
|
583
|
+
*/
|
|
584
|
+
function rerankCheck() {
|
|
585
|
+
const name = "cross-lingual rerank (D96)";
|
|
586
|
+
const cfg = loadConfig();
|
|
587
|
+
if (!cfg.crossLingualRerank) {
|
|
588
|
+
return {
|
|
589
|
+
name,
|
|
590
|
+
ok: true,
|
|
591
|
+
detail: "off (default; enable with: open-memex config set crossLingualRerank true)",
|
|
592
|
+
};
|
|
593
|
+
}
|
|
594
|
+
let installed = false;
|
|
595
|
+
try {
|
|
596
|
+
const req = createRequire(import.meta.url);
|
|
597
|
+
req.resolve("@huggingface/transformers");
|
|
598
|
+
installed = true;
|
|
599
|
+
}
|
|
600
|
+
catch {
|
|
601
|
+
installed = false;
|
|
602
|
+
}
|
|
603
|
+
if (!installed) {
|
|
604
|
+
return {
|
|
605
|
+
name,
|
|
606
|
+
ok: false,
|
|
607
|
+
detail: `on, but @huggingface/transformers is not installed — run: npm install -g @huggingface/transformers (search stays BM25 until then)`,
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
let vectors = 0;
|
|
611
|
+
try {
|
|
612
|
+
vectors = countVectors(cfg.embeddingModel);
|
|
613
|
+
}
|
|
614
|
+
catch {
|
|
615
|
+
vectors = 0;
|
|
616
|
+
}
|
|
617
|
+
return {
|
|
618
|
+
name,
|
|
619
|
+
ok: true,
|
|
620
|
+
detail: `on (model ${cfg.embeddingModel}, ${vectors} cached vector(s), models in ${embeddingModelDir()})`,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
575
623
|
export async function runDoctor() {
|
|
576
624
|
console.log("open-memex doctor");
|
|
577
|
-
const checks = [nodeCheck(), configCheck(), driverCheck(), scopeCheck(), storageCheck(), lockingCheck(), opencodePluginCheck(), vscodeMcpCheck()];
|
|
625
|
+
const checks = [nodeCheck(), configCheck(), driverCheck(), scopeCheck(), storageCheck(), lockingCheck(), rerankCheck(), opencodePluginCheck(), vscodeMcpCheck()];
|
|
578
626
|
checks.push(await mcpCheck());
|
|
579
627
|
checks.push(legacyCheck());
|
|
580
628
|
let allOk = true;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
2
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
3
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
4
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
5
|
+
});
|
|
6
|
+
}
|
|
7
|
+
return path;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* D96: local embedding runtime for the opt-in cross-lingual rerank.
|
|
11
|
+
*
|
|
12
|
+
* Local-first, no API: the model runs through `@huggingface/transformers`
|
|
13
|
+
* (ONNX, CPU) and is downloaded into `<data root>/models/` on first use.
|
|
14
|
+
* Vectors are cached in SQLite (`embedding_vectors`, keyed by a hash of the
|
|
15
|
+
* embedded text), so a warm store never re-embeds a memory.
|
|
16
|
+
*
|
|
17
|
+
* The package is deliberately NOT a dependency of open-memex: it pulls
|
|
18
|
+
* ~100-500 MB of model + onnxruntime, and the default install must stay
|
|
19
|
+
* zero-download. When the flag is on and the package is missing, the rerank
|
|
20
|
+
* degrades to plain BM25 with one stderr warning — never an error.
|
|
21
|
+
*
|
|
22
|
+
* Nothing here loads a model unless the caller already decided that a
|
|
23
|
+
* cross-lingual candidate exists; a same-language search pays zero cost.
|
|
24
|
+
*/
|
|
25
|
+
import fs from "node:fs";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import crypto from "node:crypto";
|
|
28
|
+
import { dataRootPath } from "../paths.js";
|
|
29
|
+
import { db } from "../store/db.js";
|
|
30
|
+
import { loadVectors, saveVector } from "../store/embeddings.js";
|
|
31
|
+
import { hasCjk } from "./cjk.js";
|
|
32
|
+
import { rerankCrossLingual } from "./rerank.js";
|
|
33
|
+
import { hitReviewTier } from "./search.js";
|
|
34
|
+
/** The model the D96 gate measured. User-configurable via `embeddingModel`. */
|
|
35
|
+
export const DEFAULT_EMBEDDING_MODEL = "Xenova/paraphrase-multilingual-MiniLM-L12-v2";
|
|
36
|
+
/** Package name, spelled once so the dynamic import stays non-literal (it is optional). */
|
|
37
|
+
const PACKAGE = "@huggingface/transformers";
|
|
38
|
+
let runtimeCache = null;
|
|
39
|
+
const warned = new Set();
|
|
40
|
+
function warnOnce(key, message) {
|
|
41
|
+
if (warned.has(key))
|
|
42
|
+
return;
|
|
43
|
+
warned.add(key);
|
|
44
|
+
// stdout is the MCP protocol channel — diagnostics go to stderr.
|
|
45
|
+
process.stderr.write(`[open-memex] ${message}\n`);
|
|
46
|
+
}
|
|
47
|
+
/** Cache root the ONNX model files are downloaded into (never the repo). */
|
|
48
|
+
export function embeddingModelDir() {
|
|
49
|
+
return path.join(dataRootPath(), "models");
|
|
50
|
+
}
|
|
51
|
+
async function loadRuntime(model) {
|
|
52
|
+
try {
|
|
53
|
+
const spec = PACKAGE;
|
|
54
|
+
const mod = await import(__rewriteRelativeImportExtension(spec));
|
|
55
|
+
const cacheDir = embeddingModelDir();
|
|
56
|
+
fs.mkdirSync(cacheDir, { recursive: true });
|
|
57
|
+
mod.env.cacheDir = cacheDir;
|
|
58
|
+
const pipeline = mod.pipeline;
|
|
59
|
+
// D96: the gate measured q8 identical to fp32 on the fixture at ~1/4 the
|
|
60
|
+
// download (113 MB vs 449 MB). If a user override lacks a quantized file,
|
|
61
|
+
// the load fails and we degrade to BM25 with a warning.
|
|
62
|
+
const extractor = await pipeline("feature-extraction", model, { dtype: "q8" });
|
|
63
|
+
return { embedder: extractor };
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
warnOnce(`load:${model}`, `cross-lingual rerank is on but the local embedding runtime is unavailable (${err.message}). ` +
|
|
67
|
+
`Install it with: npm install -g ${PACKAGE}. Search falls back to BM25.`);
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function runtime(model) {
|
|
72
|
+
if (runtimeCache?.model !== model)
|
|
73
|
+
runtimeCache = { model, promise: loadRuntime(model) };
|
|
74
|
+
return runtimeCache.promise;
|
|
75
|
+
}
|
|
76
|
+
function hashText(text) {
|
|
77
|
+
return crypto.createHash("sha256").update(text, "utf8").digest("hex").slice(0, 32);
|
|
78
|
+
}
|
|
79
|
+
/** The text a memory contributes to the embedding — the same fields the bench embedded. */
|
|
80
|
+
function memoryText(content, aliases, tags) {
|
|
81
|
+
const split = (s) => s
|
|
82
|
+
.split(",")
|
|
83
|
+
.map((x) => x.trim())
|
|
84
|
+
.filter(Boolean);
|
|
85
|
+
return [content, ...split(aliases), ...split(tags)]
|
|
86
|
+
.filter((s) => s.trim().length > 0)
|
|
87
|
+
.join("\n")
|
|
88
|
+
.slice(0, 2000);
|
|
89
|
+
}
|
|
90
|
+
function cosine(a, b) {
|
|
91
|
+
const n = Math.min(a.length, b.length);
|
|
92
|
+
let s = 0;
|
|
93
|
+
for (let i = 0; i < n; i++)
|
|
94
|
+
s += a[i] * b[i];
|
|
95
|
+
return s;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Rerank an already-retrieved list for EN<->ZH queries only.
|
|
99
|
+
*
|
|
100
|
+
* Called after `search()`, so BM25 has already chosen the set and the review
|
|
101
|
+
* tiers are already ordered; this only reorders within each contiguous tier
|
|
102
|
+
* block. A query with no cross-lingual hit returns immediately and never
|
|
103
|
+
* loads a model.
|
|
104
|
+
*/
|
|
105
|
+
export async function applyCrossLingualRerank(query, hits, cfg) {
|
|
106
|
+
const untouched = { hits, candidates: 0, info: new Map(), promotions: [] };
|
|
107
|
+
if (!cfg.crossLingualRerank)
|
|
108
|
+
return { ...untouched, skipped: "off" };
|
|
109
|
+
if (hits.length === 0)
|
|
110
|
+
return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
111
|
+
const queryCjk = hasCjk(query);
|
|
112
|
+
const ids = hits.map((h) => h.id);
|
|
113
|
+
const rows = db()
|
|
114
|
+
.prepare(`SELECT id, content, COALESCE(aliases, '') AS aliases, COALESCE(tags, '') AS tags
|
|
115
|
+
FROM memories WHERE id IN (${ids.map(() => "?").join(",")})`)
|
|
116
|
+
.all(...ids);
|
|
117
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
118
|
+
// Cross-lingual = the memory's script differs from the query's. Everything
|
|
119
|
+
// else is a barrier that keeps its exact BM25 slot.
|
|
120
|
+
const texts = new Map();
|
|
121
|
+
for (const h of hits) {
|
|
122
|
+
const r = byId.get(h.id);
|
|
123
|
+
if (!r)
|
|
124
|
+
continue;
|
|
125
|
+
if (hasCjk(r.content) === queryCjk)
|
|
126
|
+
continue;
|
|
127
|
+
texts.set(h.id, memoryText(r.content, r.aliases, r.tags));
|
|
128
|
+
}
|
|
129
|
+
if (texts.size === 0)
|
|
130
|
+
return { ...untouched, skipped: "no-cross-lingual-hits" };
|
|
131
|
+
const rt = await runtime(cfg.embeddingModel);
|
|
132
|
+
if (!rt)
|
|
133
|
+
return { ...untouched, skipped: "model-unavailable" };
|
|
134
|
+
const hashes = new Map();
|
|
135
|
+
for (const [id, text] of texts)
|
|
136
|
+
hashes.set(id, hashText(text));
|
|
137
|
+
let cached;
|
|
138
|
+
try {
|
|
139
|
+
cached = loadVectors(cfg.embeddingModel, [...new Set(hashes.values())]);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
cached = new Map();
|
|
143
|
+
}
|
|
144
|
+
const pending = [...texts].filter(([id]) => !cached.has(hashes.get(id)));
|
|
145
|
+
if (pending.length > 0) {
|
|
146
|
+
const vectors = await rt.embedder(pending.map(([, text]) => text), {
|
|
147
|
+
pooling: "mean",
|
|
148
|
+
normalize: true,
|
|
149
|
+
});
|
|
150
|
+
const list = vectors.tolist();
|
|
151
|
+
pending.forEach(([id], i) => {
|
|
152
|
+
const vec = list[i];
|
|
153
|
+
cached.set(hashes.get(id), vec);
|
|
154
|
+
try {
|
|
155
|
+
saveVector(cfg.embeddingModel, hashes.get(id), vec);
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
// A cache that cannot be written is a slowdown, not a failure.
|
|
159
|
+
warnOnce("vector-cache-write", `could not cache the embedding (${err.message})`);
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
const queryVector = (await rt.embedder([query], { pooling: "mean", normalize: true })).tolist()[0];
|
|
164
|
+
const cosines = new Map();
|
|
165
|
+
for (const id of texts.keys()) {
|
|
166
|
+
const v = cached.get(hashes.get(id));
|
|
167
|
+
if (v)
|
|
168
|
+
cosines.set(id, cosine(queryVector, v));
|
|
169
|
+
}
|
|
170
|
+
// Rerank inside each contiguous review-tier block — a draft can never be
|
|
171
|
+
// lifted over reviewed knowledge (D30).
|
|
172
|
+
const ordered = [];
|
|
173
|
+
const info = new Map();
|
|
174
|
+
const promotions = [];
|
|
175
|
+
let i = 0;
|
|
176
|
+
while (i < hits.length) {
|
|
177
|
+
const tier = hitReviewTier(hits[i]);
|
|
178
|
+
let j = i + 1;
|
|
179
|
+
while (j < hits.length && hitReviewTier(hits[j]) === tier)
|
|
180
|
+
j++;
|
|
181
|
+
const block = rerankCrossLingual(hits.slice(i, j), cosines);
|
|
182
|
+
for (const [id, x] of block.info) {
|
|
183
|
+
info.set(id, { ...x, from: x.from + i, to: x.to + i });
|
|
184
|
+
}
|
|
185
|
+
for (const p of block.promotions)
|
|
186
|
+
promotions.push(p);
|
|
187
|
+
ordered.push(...block.ordered);
|
|
188
|
+
i = j;
|
|
189
|
+
}
|
|
190
|
+
return { hits: ordered, candidates: texts.size, info, promotions };
|
|
191
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* D96: opt-in cross-lingual rerank (EN<->ZH only).
|
|
3
|
+
*
|
|
4
|
+
* Cold-start position: BM25 is the retriever, embeddings only ever *reorder*
|
|
5
|
+
* what BM25 already returned. The layout is identical to D79's reuse band —
|
|
6
|
+
* a hit is eligible only when the query and the memory are written in
|
|
7
|
+
* different scripts (one side has CJK, the other does not) *and* its cosine
|
|
8
|
+
* clears `DEFAULT_RERANK_MIN_COSINE`. Eligible hits are re-sorted by cosine
|
|
9
|
+
* inside their contiguous run; everything else — same-language hits and
|
|
10
|
+
* cross-lingual hits with too little signal — keeps its exact BM25 slot and
|
|
11
|
+
* acts as a barrier, so a run can never leapfrog them.
|
|
12
|
+
*
|
|
13
|
+
* Guardrails (D79, reused):
|
|
14
|
+
* - the BM25 score is never modified, only the order inside a run;
|
|
15
|
+
* - the caller has already split review tiers, so a rerank can never lift a
|
|
16
|
+
* draft over reviewed knowledge (search.ts resolves tiers before any
|
|
17
|
+
* caller-side reordering);
|
|
18
|
+
* - a low-similarity cross-lingual hit is a "no signal" hit, not a loser:
|
|
19
|
+
* it stays exactly where BM25 put it.
|
|
20
|
+
*
|
|
21
|
+
* Pure by construction: smoke-pure.ts tests it without SQLite or a model.
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* Minimum cosine for a cross-lingual pair to participate. Set by the D96
|
|
25
|
+
* fixture sweep (0.2 = the highest floor that still captures the measured
|
|
26
|
+
* cross-lingual gains; 0.3 keeps one of six, 0.4+ is inert). Lower it only
|
|
27
|
+
* with a new measurement.
|
|
28
|
+
*/
|
|
29
|
+
export const DEFAULT_RERANK_MIN_COSINE = 0.2;
|
|
30
|
+
/**
|
|
31
|
+
* Reorder one already-retrieved list. `cosines` carries only the hits the
|
|
32
|
+
* caller considers cross-lingual (same-language hits must be absent, not
|
|
33
|
+
* present with a low value — absence, not score, is what marks a barrier).
|
|
34
|
+
*/
|
|
35
|
+
export function rerankCrossLingual(hits, cosines, opts = {}) {
|
|
36
|
+
const minCosine = opts.minCosine ?? DEFAULT_RERANK_MIN_COSINE;
|
|
37
|
+
const info = new Map();
|
|
38
|
+
const ordered = [];
|
|
39
|
+
const eligible = (h) => {
|
|
40
|
+
const c = cosines.get(h.id);
|
|
41
|
+
return c !== undefined && Number.isFinite(c) && c >= minCosine;
|
|
42
|
+
};
|
|
43
|
+
let i = 0;
|
|
44
|
+
while (i < hits.length) {
|
|
45
|
+
if (!eligible(hits[i])) {
|
|
46
|
+
info.set(hits[i].id, { cosine: cosines.get(hits[i].id) ?? NaN, from: i, to: i });
|
|
47
|
+
ordered.push(hits[i]);
|
|
48
|
+
i++;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
let j = i + 1;
|
|
52
|
+
while (j < hits.length && eligible(hits[j]))
|
|
53
|
+
j++;
|
|
54
|
+
const run = hits.slice(i, j).map((h, k) => ({ h, k }));
|
|
55
|
+
run.sort((a, b) => {
|
|
56
|
+
const ca = cosines.get(a.h.id);
|
|
57
|
+
const cb = cosines.get(b.h.id);
|
|
58
|
+
return ca !== cb ? cb - ca : a.k - b.k;
|
|
59
|
+
});
|
|
60
|
+
run.forEach((x, k) => {
|
|
61
|
+
info.set(x.h.id, { cosine: cosines.get(x.h.id), from: i + x.k, to: i + k });
|
|
62
|
+
ordered.push(x.h);
|
|
63
|
+
});
|
|
64
|
+
i = j;
|
|
65
|
+
}
|
|
66
|
+
const promotions = [];
|
|
67
|
+
for (const h of ordered) {
|
|
68
|
+
const i2 = info.get(h.id);
|
|
69
|
+
if (i2.to < i2.from) {
|
|
70
|
+
promotions.push({ id: h.id, promoted: i2.from - i2.to, cosine: i2.cosine });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return { ordered, info, promotions };
|
|
74
|
+
}
|
package/dist/retrieve/search.js
CHANGED
|
@@ -62,16 +62,22 @@ function toHit(r) {
|
|
|
62
62
|
* Reviewed knowledge (approved/published) outranks unreviewed outbox drafts;
|
|
63
63
|
* personal memories are always `draft` by design and are NOT demoted.
|
|
64
64
|
* Stable: order within a tier keeps the incoming score/recency order.
|
|
65
|
+
*
|
|
66
|
+
* Exported for the D96 cross-lingual rerank, which must reorder inside a
|
|
67
|
+
* tier block only.
|
|
65
68
|
*/
|
|
66
|
-
function
|
|
67
|
-
if (
|
|
69
|
+
export function hitReviewTier(h) {
|
|
70
|
+
if (h.scope_key === "personal")
|
|
68
71
|
return 1;
|
|
69
|
-
if (
|
|
72
|
+
if (h.review_state === "approved" || h.review_state === "published")
|
|
70
73
|
return 0;
|
|
71
|
-
if (
|
|
74
|
+
if (h.review_state === "draft" || h.review_state === "rejected")
|
|
72
75
|
return 2;
|
|
73
76
|
return 1; // proposed and anything unexpected
|
|
74
77
|
}
|
|
78
|
+
function reviewTier(r) {
|
|
79
|
+
return hitReviewTier(r);
|
|
80
|
+
}
|
|
75
81
|
/** D30: `[draft]` / `[proposed]` / … marker for project memories.
|
|
76
82
|
* Personal memories stay unmarked (always draft — the marker would be noise). */
|
|
77
83
|
export function hitStateLabel(h) {
|
|
@@ -13,28 +13,28 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { db } from "./db.js";
|
|
15
15
|
function ensureTable() {
|
|
16
|
-
db().exec(`CREATE TABLE IF NOT EXISTS curation_feedback (
|
|
17
|
-
kind TEXT NOT NULL,
|
|
18
|
-
signal TEXT NOT NULL,
|
|
19
|
-
accepted INTEGER NOT NULL DEFAULT 0,
|
|
20
|
-
declined INTEGER NOT NULL DEFAULT 0,
|
|
21
|
-
undone INTEGER NOT NULL DEFAULT 0,
|
|
22
|
-
PRIMARY KEY (kind, signal)
|
|
16
|
+
db().exec(`CREATE TABLE IF NOT EXISTS curation_feedback (
|
|
17
|
+
kind TEXT NOT NULL,
|
|
18
|
+
signal TEXT NOT NULL,
|
|
19
|
+
accepted INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
declined INTEGER NOT NULL DEFAULT 0,
|
|
21
|
+
undone INTEGER NOT NULL DEFAULT 0,
|
|
22
|
+
PRIMARY KEY (kind, signal)
|
|
23
23
|
)`);
|
|
24
24
|
}
|
|
25
25
|
export function recordCurationOutcome(kind, signal, outcome) {
|
|
26
26
|
ensureTable();
|
|
27
27
|
const col = outcome === "accepted" ? "accepted" : outcome === "declined" ? "declined" : "undone";
|
|
28
28
|
db()
|
|
29
|
-
.prepare(`INSERT INTO curation_feedback (kind, signal, accepted, declined, undone)
|
|
30
|
-
VALUES (?, ?, ${col === "accepted" ? 1 : 0}, ${col === "declined" ? 1 : 0}, ${col === "undone" ? 1 : 0})
|
|
29
|
+
.prepare(`INSERT INTO curation_feedback (kind, signal, accepted, declined, undone)
|
|
30
|
+
VALUES (?, ?, ${col === "accepted" ? 1 : 0}, ${col === "declined" ? 1 : 0}, ${col === "undone" ? 1 : 0})
|
|
31
31
|
ON CONFLICT(kind, signal) DO UPDATE SET ${col} = curation_feedback.${col} + 1`)
|
|
32
32
|
.run(kind, signal);
|
|
33
33
|
}
|
|
34
34
|
export function loadCurationFeedback() {
|
|
35
35
|
ensureTable();
|
|
36
36
|
return db()
|
|
37
|
-
.prepare(`SELECT kind, signal, accepted, declined, undone FROM curation_feedback
|
|
37
|
+
.prepare(`SELECT kind, signal, accepted, declined, undone FROM curation_feedback
|
|
38
38
|
ORDER BY kind, signal`)
|
|
39
39
|
.all();
|
|
40
40
|
}
|