@coralai/sps-cli 0.58.16 → 0.58.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/memory2/config.d.ts +1 -1
- package/dist/core/memory2/config.js +1 -1
- package/dist/core/memory2/ftsIndex.d.ts +8 -0
- package/dist/core/memory2/ftsIndex.d.ts.map +1 -0
- package/dist/core/memory2/ftsIndex.js +172 -0
- package/dist/core/memory2/ftsIndex.js.map +1 -0
- package/dist/core/memory2/retrieve.d.ts +5 -1
- package/dist/core/memory2/retrieve.d.ts.map +1 -1
- package/dist/core/memory2/retrieve.js +11 -2
- package/dist/core/memory2/retrieve.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
* external | none。此处只做「本地读写是否启用」这个**当前真需要**的判定:mode=none
|
|
5
5
|
* → 本地注入/捕获全关。
|
|
6
6
|
*
|
|
7
|
+
* ✅ FTS5 加速已实现(见 ftsIndex.ts):store ≥ 阈值且 node:sqlite 可用时自动切,否则回退内存。
|
|
7
8
|
* ⚠️ 刻意未实现(YAGNI,设计 §4/§5 明列「按需、默认关、需要才建」):
|
|
8
9
|
* - external / local+external 的外部后端适配器与合并(RRF)——无已接外部后端、无 scale 压力;
|
|
9
|
-
* - FTS5 加速(仅 store 很大且 node:sqlite 可用时才值得);
|
|
10
10
|
* - embedding 向量混检、LLM 蒸馏。
|
|
11
11
|
* 需求出现时,在此扩 `memoryMode()` 的分支 + 加一层可选适配器即可,基线纯本地永远能跑。
|
|
12
12
|
* @layer core
|
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
* external | none。此处只做「本地读写是否启用」这个**当前真需要**的判定:mode=none
|
|
5
5
|
* → 本地注入/捕获全关。
|
|
6
6
|
*
|
|
7
|
+
* ✅ FTS5 加速已实现(见 ftsIndex.ts):store ≥ 阈值且 node:sqlite 可用时自动切,否则回退内存。
|
|
7
8
|
* ⚠️ 刻意未实现(YAGNI,设计 §4/§5 明列「按需、默认关、需要才建」):
|
|
8
9
|
* - external / local+external 的外部后端适配器与合并(RRF)——无已接外部后端、无 scale 压力;
|
|
9
|
-
* - FTS5 加速(仅 store 很大且 node:sqlite 可用时才值得);
|
|
10
10
|
* - embedding 向量混检、LLM 蒸馏。
|
|
11
11
|
* 需求出现时,在此扩 `memoryMode()` 的分支 + 加一层可选适配器即可,基线纯本地永远能跑。
|
|
12
12
|
* @layer core
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type SearchHit, type SearchOpts } from './retrieve.js';
|
|
2
|
+
import type { ScopeRef } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* FTS5 检索。返回 SearchHit[] 表示"走了 DB 路";返回 null 表示"未启用/不可用"→ 调用方回退内存。
|
|
5
|
+
* 排序仍用内存 searchEntries(FTS 仅预筛候选),故结果与纯内存一致。
|
|
6
|
+
*/
|
|
7
|
+
export declare function ftsRecall(refs: ScopeRef[], query: string, opts?: SearchOpts): SearchHit[] | null;
|
|
8
|
+
//# sourceMappingURL=ftsIndex.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ftsIndex.d.ts","sourceRoot":"","sources":["../../../src/core/memory2/ftsIndex.ts"],"names":[],"mappings":"AAcA,OAAO,EAA2B,KAAK,SAAS,EAAE,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AACzF,OAAO,KAAK,EAAe,QAAQ,EAAE,MAAM,YAAY,CAAC;AAoHxD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAe,GAAG,SAAS,EAAE,GAAG,IAAI,CAgCpG"}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module core/memory2/ftsIndex
|
|
3
|
+
* @description 可选 FTS5 加速层(设计 §4 第二级)。**markdown 仍是唯一真源**,本索引是**派生、可重建**的
|
|
4
|
+
* 只读加速器:大 store 时用 FTS5 把候选从几千缩到几十,排序仍交给内存 BM25(searchEntries)
|
|
5
|
+
* → 结果与纯内存一致、只更快。门控:node:sqlite 可用(Node 22.5+)+ 条目数 ≥ 阈值 +
|
|
6
|
+
* 未被 SPS_MEMORY_INDEX=off 关;任一不满足或任何异常 → 返回 null,调用方静默回退内存。
|
|
7
|
+
* 与 codegraph 共用 `node:sqlite`;索引落 `~/.coral/memory/index.db`。
|
|
8
|
+
* @layer core
|
|
9
|
+
* @boundedContext memory
|
|
10
|
+
*/
|
|
11
|
+
import { existsSync, readdirSync, statSync } from 'node:fs';
|
|
12
|
+
import { createRequire } from 'node:module';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { listAllScopes, listEntries, memoryRoot, scopeDir } from './store.js';
|
|
15
|
+
import { searchEntries, tokenize } from './retrieve.js';
|
|
16
|
+
/** 条目数达到此阈值才值得用 FTS5('auto' 模式下);'on' 忽略阈值。 */
|
|
17
|
+
const INDEX_MIN_ENTRIES = 500;
|
|
18
|
+
function indexMode() {
|
|
19
|
+
const v = (process.env.SPS_MEMORY_INDEX ?? 'auto').toLowerCase();
|
|
20
|
+
return v === 'off' ? 'off' : v === 'on' ? 'on' : 'auto';
|
|
21
|
+
}
|
|
22
|
+
// node:sqlite 惰性探测(旧 Node < 22.5 不存在)。undefined=没试过,null=不可用。
|
|
23
|
+
const nodeRequire = createRequire(import.meta.url);
|
|
24
|
+
let DatabaseSyncCtor;
|
|
25
|
+
function loadSqlite() {
|
|
26
|
+
if (DatabaseSyncCtor === undefined) {
|
|
27
|
+
try {
|
|
28
|
+
DatabaseSyncCtor = nodeRequire('node:sqlite').DatabaseSync ?? null;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
DatabaseSyncCtor = null; // 旧 Node / 未编译 sqlite → 回退内存
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return DatabaseSyncCtor;
|
|
35
|
+
}
|
|
36
|
+
function indexPath() { return join(memoryRoot(), 'index.db'); }
|
|
37
|
+
/** 全库指纹(不解析正文,只 readdir+stat):文件总数 + 最新 mtime。变了就重建。 */
|
|
38
|
+
function fingerprint() {
|
|
39
|
+
let count = 0;
|
|
40
|
+
let maxMtime = 0;
|
|
41
|
+
for (const ref of listAllScopes()) {
|
|
42
|
+
const dir = join(scopeDir(ref), 'entries');
|
|
43
|
+
if (!existsSync(dir))
|
|
44
|
+
continue;
|
|
45
|
+
for (const name of readdirSync(dir)) {
|
|
46
|
+
if (!name.endsWith('.md'))
|
|
47
|
+
continue;
|
|
48
|
+
count++;
|
|
49
|
+
try {
|
|
50
|
+
const m = statSync(join(dir, name)).mtimeMs;
|
|
51
|
+
if (m > maxMtime)
|
|
52
|
+
maxMtime = m;
|
|
53
|
+
}
|
|
54
|
+
catch { /* 文件刚被删 */ }
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { count, maxMtime };
|
|
58
|
+
}
|
|
59
|
+
function openDb() {
|
|
60
|
+
const Ctor = loadSqlite();
|
|
61
|
+
if (!Ctor)
|
|
62
|
+
return null;
|
|
63
|
+
try {
|
|
64
|
+
const db = new Ctor(indexPath());
|
|
65
|
+
db.exec(`
|
|
66
|
+
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
|
|
67
|
+
CREATE TABLE IF NOT EXISTS entries (
|
|
68
|
+
id TEXT PRIMARY KEY, scope TEXT, project TEXT, agentId TEXT,
|
|
69
|
+
title TEXT, tags TEXT, category TEXT, source TEXT, salience INTEGER,
|
|
70
|
+
created TEXT, updated TEXT, body TEXT
|
|
71
|
+
);
|
|
72
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(id UNINDEXED, tokens);
|
|
73
|
+
`);
|
|
74
|
+
return db;
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return null; // fts5 不可用 / 打不开 → 回退内存
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** 若指纹变化则全量重建索引(markdown → DB)。 */
|
|
81
|
+
function ensureFresh(db) {
|
|
82
|
+
const fp = fingerprint();
|
|
83
|
+
const stored = db.prepare('SELECT v FROM meta WHERE k=?').get('fingerprint')?.v;
|
|
84
|
+
const cur = `${fp.count}:${Math.round(fp.maxMtime)}`;
|
|
85
|
+
if (stored === cur)
|
|
86
|
+
return;
|
|
87
|
+
db.exec('DELETE FROM entries; DELETE FROM fts;');
|
|
88
|
+
const insE = db.prepare(`INSERT INTO entries (id,scope,project,agentId,title,tags,category,source,salience,created,updated,body)
|
|
89
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`);
|
|
90
|
+
const insF = db.prepare('INSERT INTO fts (id, tokens) VALUES (?, ?)');
|
|
91
|
+
for (const ref of listAllScopes()) {
|
|
92
|
+
for (const e of listEntries(ref)) {
|
|
93
|
+
insE.run(e.id, e.scope, e.project ?? null, ref.scope === 'agent' ? ref.agentId : null, e.title, JSON.stringify(e.tags), e.category, e.source, e.salience, e.created, e.updated, e.body);
|
|
94
|
+
insF.run(e.id, tokenize(`${e.title} ${e.tags.join(' ')} ${e.body}`).join(' '));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
db.prepare('INSERT OR REPLACE INTO meta (k,v) VALUES (?,?)').run('fingerprint', cur);
|
|
98
|
+
}
|
|
99
|
+
function rowToEntry(r) {
|
|
100
|
+
return {
|
|
101
|
+
id: String(r.id), title: String(r.title), scope: r.scope,
|
|
102
|
+
category: r.category, tags: JSON.parse(String(r.tags || '[]')),
|
|
103
|
+
salience: Number(r.salience), source: r.source,
|
|
104
|
+
project: r.project ? String(r.project) : undefined,
|
|
105
|
+
created: String(r.created), updated: String(r.updated), body: String(r.body),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/** 作用域集 → SQL WHERE 片段 + 参数(匹配任一 ref)。 */
|
|
109
|
+
function scopeWhere(refs) {
|
|
110
|
+
const clauses = [];
|
|
111
|
+
const params = [];
|
|
112
|
+
for (const r of refs) {
|
|
113
|
+
if (r.scope === 'global')
|
|
114
|
+
clauses.push("scope='global'");
|
|
115
|
+
else if (r.scope === 'project') {
|
|
116
|
+
clauses.push('(scope=\'project\' AND project=?)');
|
|
117
|
+
params.push(r.project);
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
clauses.push('(scope=\'agent\' AND agentId=?)');
|
|
121
|
+
params.push(r.agentId);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return { sql: clauses.length ? `(${clauses.join(' OR ')})` : '0', params };
|
|
125
|
+
}
|
|
126
|
+
/** 总条目数(门控用;复用指纹的 count,不解析正文)。 */
|
|
127
|
+
function totalCount() { return fingerprint().count; }
|
|
128
|
+
/**
|
|
129
|
+
* FTS5 检索。返回 SearchHit[] 表示"走了 DB 路";返回 null 表示"未启用/不可用"→ 调用方回退内存。
|
|
130
|
+
* 排序仍用内存 searchEntries(FTS 仅预筛候选),故结果与纯内存一致。
|
|
131
|
+
*/
|
|
132
|
+
export function ftsRecall(refs, query, opts = {}) {
|
|
133
|
+
const mode = indexMode();
|
|
134
|
+
if (mode === 'off')
|
|
135
|
+
return null;
|
|
136
|
+
if (!loadSqlite())
|
|
137
|
+
return null;
|
|
138
|
+
if (mode === 'auto' && totalCount() < INDEX_MIN_ENTRIES)
|
|
139
|
+
return null;
|
|
140
|
+
const db = openDb();
|
|
141
|
+
if (!db)
|
|
142
|
+
return null;
|
|
143
|
+
try {
|
|
144
|
+
ensureFresh(db);
|
|
145
|
+
const where = scopeWhere(refs);
|
|
146
|
+
const qTokens = tokenize(query);
|
|
147
|
+
let rows;
|
|
148
|
+
if (qTokens.length === 0) {
|
|
149
|
+
// 空查询:该作用域全集交给 searchEntries 按 salience/新鲜度排(镜像内存路)。
|
|
150
|
+
rows = db.prepare(`SELECT * FROM entries WHERE ${where.sql}`).all(...where.params);
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
// FTS MATCH 预筛:任一 token 命中即候选;取较宽候选(limit*40,上限 400)再交内存精排。
|
|
154
|
+
const match = qTokens.map((t) => `"${t.replace(/"/g, '')}"`).join(' OR ');
|
|
155
|
+
const cap = Math.min(400, (opts.limit ?? 6) * 40);
|
|
156
|
+
rows = db.prepare(`SELECT e.* FROM fts JOIN entries e ON e.id = fts.id
|
|
157
|
+
WHERE fts MATCH ? AND ${where.sql} ORDER BY bm25(fts) LIMIT ?`).all(match, ...where.params, cap);
|
|
158
|
+
}
|
|
159
|
+
const candidates = rows.map(rowToEntry);
|
|
160
|
+
return searchEntries(candidates, query, opts);
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
return null; // 任何 DB 异常 → 回退内存
|
|
164
|
+
}
|
|
165
|
+
finally {
|
|
166
|
+
try {
|
|
167
|
+
db.close();
|
|
168
|
+
}
|
|
169
|
+
catch { /* ignore */ }
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=ftsIndex.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ftsIndex.js","sourceRoot":"","sources":["../../../src/core/memory2/ftsIndex.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAmC,MAAM,eAAe,CAAC;AAGzF,+CAA+C;AAC/C,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAG9B,SAAS,SAAS;IAChB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IACjE,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC1D,CAAC;AAED,8DAA8D;AAC9D,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACnD,IAAI,gBAAyB,CAAC;AAC9B,SAAS,UAAU;IACjB,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,gBAAgB,GAAI,WAAW,CAAC,aAAa,CAA+B,CAAC,YAAY,IAAI,IAAI,CAAC;QACpG,CAAC;QAAC,MAAM,CAAC;YACP,gBAAgB,GAAG,IAAI,CAAC,CAAC,6BAA6B;QACxD,CAAC;IACH,CAAC;IACD,OAAO,gBAA2D,CAAC;AACrE,CAAC;AAMD,SAAS,SAAS,KAAa,OAAO,IAAI,CAAC,UAAU,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;AAEvE,wDAAwD;AACxD,SAAS,WAAW;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAAE,SAAS;YACpC,KAAK,EAAE,CAAC;YACR,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;gBAC5C,IAAI,CAAC,GAAG,QAAQ;oBAAE,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,MAAM;IACb,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAC1B,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QACjC,EAAE,CAAC,IAAI,CAAC;;;;;;;;KAQP,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,CAAC,wBAAwB;IACvC,CAAC;AACH,CAAC;AAED,mCAAmC;AACnC,SAAS,WAAW,CAAC,EAAY;IAC/B,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAI,EAAE,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC,GAAG,CAAC,aAAa,CAA+B,EAAE,CAAC,CAAC;IAC/G,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;IACrD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO;IAE3B,EAAE,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;qCACW,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,EAAE,GAAG,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAQ,CAAC,CAAC,CAAC,IAAI,EACpF,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACnG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;AACvF,CAAC;AAED,SAAS,UAAU,CAAC,CAA0B;IAC5C,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAA6B;QAChF,QAAQ,EAAE,CAAC,CAAC,QAAmC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;QACzF,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAA+B;QACvE,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QAClD,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;KAC7E,CAAC;AACJ,CAAC;AAED,0CAA0C;AAC1C,SAAS,UAAU,CAAC,IAAgB;IAClC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;aACpD,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;aACzG,CAAC;YAAC,OAAO,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;IACnF,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC;AAC7E,CAAC;AAED,mCAAmC;AACnC,SAAS,UAAU,KAAa,OAAO,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;AAE7D;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAgB,EAAE,KAAa,EAAE,OAAmB,EAAE;IAC9E,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;IACzB,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,IAAI,KAAK,MAAM,IAAI,UAAU,EAAE,GAAG,iBAAiB;QAAE,OAAO,IAAI,CAAC;IAErE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;IACpB,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IACrB,IAAI,CAAC;QACH,WAAW,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,IAA+B,CAAC;QACpC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,qDAAqD;YACrD,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,+BAA+B,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAA8B,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,IAAI,GAAG,EAAE,CAAC,OAAO,CACf;iCACyB,KAAK,CAAC,GAAG,6BAA6B,CAChE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,GAAG,CAA8B,CAAC;QAClE,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACxC,OAAO,aAAa,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,CAAC,kBAAkB;IACjC,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC"}
|
|
@@ -11,6 +11,10 @@ export interface SearchOpts {
|
|
|
11
11
|
export declare function tokenize(s: string): string[];
|
|
12
12
|
/** 对给定条目集按 query 做 BM25 + tag 过滤 + salience 加权排序。纯函数(便于测试/组合)。 */
|
|
13
13
|
export declare function searchEntries(entries: MemoryEntry[], query: string, opts?: SearchOpts): SearchHit[];
|
|
14
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* 跨多个作用域检索(如 worker = project + global)。
|
|
16
|
+
* 门控:store 大且 node:sqlite 可用 → 走 FTS5 预筛(设计 §4 第二级);否则/异常 → 纯内存全扫。
|
|
17
|
+
* 两条路都用 searchEntries 精排,结果一致。
|
|
18
|
+
*/
|
|
15
19
|
export declare function recall(refs: ScopeRef[], query: string, opts?: SearchOpts): SearchHit[];
|
|
16
20
|
//# sourceMappingURL=retrieve.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.d.ts","sourceRoot":"","sources":["../../../src/core/memory2/retrieve.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"retrieve.d.ts","sourceRoot":"","sources":["../../../src/core/memory2/retrieve.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AACD,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,wBAAgB,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAK5C;AAKD,kEAAkE;AAClE,wBAAgB,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAe,GAAG,SAAS,EAAE,CA6CvG;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAe,GAAG,SAAS,EAAE,CAK1F"}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @module core/memory2/retrieve
|
|
3
3
|
* @description 纯内存相关性检索(修旧系统 C1 的零相关性)。BM25 + tag 过滤 + salience 加权。
|
|
4
|
-
* 零 DB、零依赖,任何 Node 可跑(设计 §4);store
|
|
4
|
+
* 零 DB、零依赖,任何 Node 可跑(设计 §4);store 大且 node:sqlite 可用时,recall() 自动
|
|
5
|
+
* 切 FTS5 预筛(见 ftsIndex.ts),精排仍用本文件的 searchEntries —— 结果一致、只更快。
|
|
5
6
|
* 中文按单字分词 + 拉丁按词——小规模够用。
|
|
6
7
|
* @layer core
|
|
7
8
|
* @boundedContext memory
|
|
8
9
|
*/
|
|
9
10
|
import { listEntries } from './store.js';
|
|
11
|
+
import { ftsRecall } from './ftsIndex.js';
|
|
10
12
|
/** 分词:拉丁词 + 数字 + 单个 CJK 字。 */
|
|
11
13
|
export function tokenize(s) {
|
|
12
14
|
const lower = s.toLowerCase();
|
|
@@ -62,8 +64,15 @@ export function searchEntries(entries, query, opts = {}) {
|
|
|
62
64
|
.sort((a, b) => b.score - a.score)
|
|
63
65
|
.slice(0, limit);
|
|
64
66
|
}
|
|
65
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* 跨多个作用域检索(如 worker = project + global)。
|
|
69
|
+
* 门控:store 大且 node:sqlite 可用 → 走 FTS5 预筛(设计 §4 第二级);否则/异常 → 纯内存全扫。
|
|
70
|
+
* 两条路都用 searchEntries 精排,结果一致。
|
|
71
|
+
*/
|
|
66
72
|
export function recall(refs, query, opts = {}) {
|
|
73
|
+
const viaFts = ftsRecall(refs, query, opts); // null = 未启用/不可用 → 回退内存
|
|
74
|
+
if (viaFts)
|
|
75
|
+
return viaFts;
|
|
67
76
|
const all = refs.flatMap((r) => listEntries(r));
|
|
68
77
|
return searchEntries(all, query, opts);
|
|
69
78
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../../src/core/memory2/retrieve.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"retrieve.js","sourceRoot":"","sources":["../../../src/core/memory2/retrieve.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAY1C,8BAA8B;AAC9B,MAAM,UAAU,QAAQ,CAAC,CAAS;IAChC,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxC,OAAO,CAAC,GAAG,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,kEAAkE;AAClE,MAAM,UAAU,aAAa,CAAC,OAAsB,EAAE,KAAa,EAAE,OAAmB,EAAE;IACxF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAE9B,IAAI,IAAI,GAAG,OAAO,CAAC;IACnB,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,yBAAyB;QACzB,OAAO,IAAI;aACR,KAAK,EAAE;aACP,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;aAC7E,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,KAAK,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;YAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAElF,MAAM,IAAI,GAAgB,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACnB,MAAM,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,KAAK,MAAM,CAAC,IAAI,CAAC;YAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACzB,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACpD,EAAE,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI;SACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,IAAgB,EAAE,KAAa,EAAE,OAAmB,EAAE;IAC3E,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,wBAAwB;IACrE,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,OAAO,aAAa,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AACzC,CAAC"}
|