dsh-layered-memory 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +118 -66
- package/README.md +131 -98
- package/assets/img/EmbeddingSource.png +0 -0
- package/assets/img/MemoryTools.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ToolTrajectory.png +0 -0
- package/dist/client.js +11 -2
- package/dist/config.d.ts +26 -1
- package/dist/config.js +9 -3
- package/dist/hooks/recall.d.ts +11 -8
- package/dist/hooks/recall.js +70 -51
- package/dist/index.d.ts +16 -0
- package/dist/index.js +5 -1
- package/dist/llm.d.ts +13 -0
- package/dist/llm.js +17 -0
- package/dist/pipeline/l1.js +5 -4
- package/dist/pipeline/l2.js +7 -2
- package/dist/pipeline/l3.js +7 -2
- package/dist/pipeline/runner.d.ts +36 -5
- package/dist/pipeline/runner.js +148 -35
- package/dist/pipeline/trigger.d.ts +38 -0
- package/dist/pipeline/trigger.js +64 -0
- package/dist/runtime-package-lock.json +982 -0
- package/dist/store/bm25.js +2 -1
- package/dist/store/embedding.d.ts +12 -6
- package/dist/store/embedding.js +10 -7
- package/dist/store/l0.d.ts +2 -0
- package/dist/store/l0.js +11 -4
- package/dist/store/l1.d.ts +2 -0
- package/dist/store/l1.js +9 -6
- package/dist/store/pending.d.ts +27 -6
- package/dist/store/pending.js +49 -9
- package/dist/store/runtime-installer.d.ts +6 -1
- package/dist/store/runtime-installer.js +65 -26
- package/dist/store/search-utils.js +3 -2
- package/dist/store/session-modes.d.ts +4 -0
- package/dist/store/session-modes.js +16 -0
- package/dist/store/sqlite.d.ts +27 -0
- package/dist/store/sqlite.js +186 -6
- package/dist/util/recall-budget.d.ts +32 -0
- package/dist/util/recall-budget.js +85 -0
- package/dist/util/text.d.ts +8 -3
- package/dist/util/text.js +41 -15
- package/dist/util/tokenizer.d.ts +14 -0
- package/dist/util/tokenizer.js +53 -0
- package/package.json +2 -1
package/dist/store/bm25.js
CHANGED
|
@@ -10,9 +10,14 @@ export interface EmbeddingProviderInfo {
|
|
|
10
10
|
model: string;
|
|
11
11
|
dimensions: number;
|
|
12
12
|
}
|
|
13
|
+
/** 单次嵌入调用的可选参数:timeoutMs 只允许缩短服务配置的超时(内层钳制),
|
|
14
|
+
* 永不放大——召回路径用它给 FTS 降级留时间(规格 A 节)。本地实现可忽略。 */
|
|
15
|
+
export interface EmbedCallOptions {
|
|
16
|
+
timeoutMs?: number;
|
|
17
|
+
}
|
|
13
18
|
export interface EmbeddingService {
|
|
14
|
-
embed(text: string): Promise<Float32Array>;
|
|
15
|
-
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
19
|
+
embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
|
|
20
|
+
embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
|
|
16
21
|
getDimensions(): number;
|
|
17
22
|
getProviderInfo(): EmbeddingProviderInfo;
|
|
18
23
|
isReady(): boolean;
|
|
@@ -46,8 +51,8 @@ export declare class RemoteEmbeddingService implements EmbeddingService {
|
|
|
46
51
|
getDimensions(): number;
|
|
47
52
|
getProviderInfo(): EmbeddingProviderInfo;
|
|
48
53
|
isReady(): boolean;
|
|
49
|
-
embed(text: string): Promise<Float32Array>;
|
|
50
|
-
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
54
|
+
embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
|
|
55
|
+
embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
|
|
51
56
|
}
|
|
52
57
|
/**
|
|
53
58
|
* 嵌入调用降级助手(L0/L1 Store 共用):查询向量失败 → undefined(调用方降级 FTS);
|
|
@@ -62,8 +67,9 @@ export declare class EmbedHelper {
|
|
|
62
67
|
/** 活切换嵌入源(D4/D5):换掉底层服务并复位一次性告警(新服务重新获得告警机会)。 */
|
|
63
68
|
setService(svc: EmbeddingService): void;
|
|
64
69
|
vectorReady(): boolean;
|
|
65
|
-
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
66
|
-
|
|
70
|
+
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
71
|
+
* timeoutMs 为内层钳制(仅缩短服务超时),召回路径使用。 */
|
|
72
|
+
query(text: string, timeoutMs?: number): Promise<Float32Array | undefined>;
|
|
67
73
|
/** 批量嵌入;服务未就绪或失败时返回全 undefined(不阻断元数据/FTS 写入)。 */
|
|
68
74
|
batch(texts: string[]): Promise<Array<Float32Array | undefined>>;
|
|
69
75
|
private warn;
|
package/dist/store/embedding.js
CHANGED
|
@@ -34,15 +34,17 @@ export class RemoteEmbeddingService {
|
|
|
34
34
|
isReady() {
|
|
35
35
|
return true;
|
|
36
36
|
}
|
|
37
|
-
async embed(text) {
|
|
38
|
-
const [vec] = await this.embedBatch([text]);
|
|
37
|
+
async embed(text, callOpts) {
|
|
38
|
+
const [vec] = await this.embedBatch([text], callOpts);
|
|
39
39
|
return vec;
|
|
40
40
|
}
|
|
41
|
-
async embedBatch(texts) {
|
|
41
|
+
async embedBatch(texts, callOpts) {
|
|
42
42
|
if (texts.length === 0)
|
|
43
43
|
return [];
|
|
44
44
|
const input = texts.map((t) => t.slice(0, this.opts.maxInputChars));
|
|
45
45
|
const base = this.opts.baseUrl.replace(/\/+$/, '');
|
|
46
|
+
// 内层钳制只缩短不放大:召回路径传短超时,给 FTS 降级留时间
|
|
47
|
+
const timeoutMs = Math.min(this.opts.timeoutMs, callOpts?.timeoutMs ?? this.opts.timeoutMs);
|
|
46
48
|
const res = await fetch(`${base}/embeddings`, {
|
|
47
49
|
method: 'POST',
|
|
48
50
|
headers: {
|
|
@@ -50,7 +52,7 @@ export class RemoteEmbeddingService {
|
|
|
50
52
|
authorization: `Bearer ${this.opts.apiKey}`,
|
|
51
53
|
},
|
|
52
54
|
body: JSON.stringify({ model: this.opts.model, input }),
|
|
53
|
-
signal: AbortSignal.timeout(
|
|
55
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
54
56
|
});
|
|
55
57
|
if (!res.ok) {
|
|
56
58
|
const body = await res.text().catch(() => '');
|
|
@@ -103,10 +105,11 @@ export class EmbedHelper {
|
|
|
103
105
|
vectorReady() {
|
|
104
106
|
return this.embed.isReady();
|
|
105
107
|
}
|
|
106
|
-
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
107
|
-
|
|
108
|
+
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
109
|
+
* timeoutMs 为内层钳制(仅缩短服务超时),召回路径使用。 */
|
|
110
|
+
async query(text, timeoutMs) {
|
|
108
111
|
try {
|
|
109
|
-
const vec = await this.embed.embed(text);
|
|
112
|
+
const vec = await this.embed.embed(text, timeoutMs != null ? { timeoutMs } : undefined);
|
|
110
113
|
return vec.length > 0 ? vec : undefined;
|
|
111
114
|
}
|
|
112
115
|
catch (err) {
|
package/dist/store/l0.d.ts
CHANGED
|
@@ -15,6 +15,8 @@ export declare class L0Store {
|
|
|
15
15
|
append(sessionId: string, messages: ConversationMessage[]): Promise<void>;
|
|
16
16
|
/** 今日已捕获消息数(SQL 计数,不再读整文件)。 */
|
|
17
17
|
countToday(): Promise<number>;
|
|
18
|
+
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
19
|
+
recentBySession(sessionId: string, limit: number): Promise<ConversationMessage[]>;
|
|
18
20
|
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
19
21
|
search(query: string, limit: number): Promise<L0MessageRecord[]>;
|
|
20
22
|
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
package/dist/store/l0.js
CHANGED
|
@@ -99,6 +99,10 @@ export class L0Store {
|
|
|
99
99
|
const d = new Date();
|
|
100
100
|
return this.db.countL0Since(new Date(d.getFullYear(), d.getMonth(), d.getDate()).toISOString());
|
|
101
101
|
}
|
|
102
|
+
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
103
|
+
async recentBySession(sessionId, limit) {
|
|
104
|
+
return this.db.recentL0BySession(sessionId, limit);
|
|
105
|
+
}
|
|
102
106
|
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
103
107
|
async search(query, limit) {
|
|
104
108
|
const caps = this.db.getCapabilities();
|
|
@@ -154,17 +158,20 @@ export class L0Store {
|
|
|
154
158
|
opts?.onProgress?.(done, total);
|
|
155
159
|
continue;
|
|
156
160
|
}
|
|
161
|
+
const pending = [];
|
|
157
162
|
chunk.forEach((c, j) => {
|
|
158
163
|
if (isZeroVector(vecs[j])) {
|
|
159
164
|
skipped++;
|
|
160
165
|
skippedNow.push(c.id);
|
|
161
166
|
return;
|
|
162
167
|
}
|
|
163
|
-
|
|
164
|
-
written++;
|
|
165
|
-
else
|
|
166
|
-
failed++;
|
|
168
|
+
pending.push({ id: c.id, embedding: vecs[j] });
|
|
167
169
|
});
|
|
170
|
+
if (pending.length > 0) {
|
|
171
|
+
const ok = this.db.updateL0VecBatch(pending, '');
|
|
172
|
+
written += ok;
|
|
173
|
+
failed += pending.length - ok;
|
|
174
|
+
}
|
|
168
175
|
done += chunk.length;
|
|
169
176
|
opts?.onProgress?.(done, total);
|
|
170
177
|
}
|
package/dist/store/l1.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export interface L1SearchOptions {
|
|
|
10
10
|
/** 分数阈值(仅召回路径传;keyword/embedding 策略生效,FTS 含小语料例外;
|
|
11
11
|
* hybrid 按官方语义在 RRF 融合前不过滤)。 */
|
|
12
12
|
scoreThreshold?: number;
|
|
13
|
+
/** 嵌入查询内层钳制(ms,只缩短不放大;召回路径传入给 FTS 降级留时间)。 */
|
|
14
|
+
embeddingTimeoutMs?: number;
|
|
13
15
|
}
|
|
14
16
|
export declare class L1Store {
|
|
15
17
|
private readonly db;
|
package/dist/store/l1.js
CHANGED
|
@@ -135,7 +135,7 @@ export class L1Store {
|
|
|
135
135
|
return this.postProcess(applyFtsThreshold(fts, threshold, limit), opts?.type, limit);
|
|
136
136
|
}
|
|
137
137
|
if (strategy === 'embedding') {
|
|
138
|
-
const vec = await this.helper.query(query);
|
|
138
|
+
const vec = await this.helper.query(query, opts?.embeddingTimeoutMs);
|
|
139
139
|
if (!vec) {
|
|
140
140
|
// embedding 调用失败:降级 FTS,不阻断
|
|
141
141
|
const fts = this.db.searchL1Fts(query, candidateK, opts?.family);
|
|
@@ -148,7 +148,7 @@ export class L1Store {
|
|
|
148
148
|
// → 融合分归一化:rank1 双列表命中 = 1.0,单列表命中 ≤ 0.5,保持 0~1 语义
|
|
149
149
|
const [ftsList, vecRaw] = await Promise.all([
|
|
150
150
|
Promise.resolve(this.db.searchL1Fts(query, candidateK, opts?.family)),
|
|
151
|
-
this.helper.query(query),
|
|
151
|
+
this.helper.query(query, opts?.embeddingTimeoutMs),
|
|
152
152
|
]);
|
|
153
153
|
const vecList = vecRaw ? this.db.searchL1Vector(vecRaw, candidateK, opts?.family) : [];
|
|
154
154
|
const merged = rrfMerge([ftsList, vecList], (h) => h.id);
|
|
@@ -221,17 +221,20 @@ export class L1Store {
|
|
|
221
221
|
opts?.onProgress?.(done, total);
|
|
222
222
|
continue;
|
|
223
223
|
}
|
|
224
|
+
const pending = [];
|
|
224
225
|
chunk.forEach((c, j) => {
|
|
225
226
|
if (isZeroVector(vecs[j])) {
|
|
226
227
|
skipped++;
|
|
227
228
|
skippedNow.push(c.id);
|
|
228
229
|
return;
|
|
229
230
|
}
|
|
230
|
-
|
|
231
|
-
written++;
|
|
232
|
-
else
|
|
233
|
-
failed++;
|
|
231
|
+
pending.push({ id: c.id, embedding: vecs[j] });
|
|
234
232
|
});
|
|
233
|
+
if (pending.length > 0) {
|
|
234
|
+
const ok = this.db.updateL1VecBatch(pending);
|
|
235
|
+
written += ok;
|
|
236
|
+
failed += pending.length - ok;
|
|
237
|
+
}
|
|
235
238
|
done += chunk.length;
|
|
236
239
|
opts?.onProgress?.(done, total);
|
|
237
240
|
}
|
package/dist/store/pending.d.ts
CHANGED
|
@@ -1,15 +1,36 @@
|
|
|
1
1
|
import type { ConversationMessage, ExtractMode, MemoryLogger } from '../types.js';
|
|
2
|
+
/** 带会话标识的未蒸馏消息(会话切片的成员;CONTEXT.md「会话切片」「捕获档位」)。 */
|
|
3
|
+
export interface PendingMessage extends ConversationMessage {
|
|
4
|
+
/** 捕获该消息的会话;旧格式数据无此字段,加载时归 legacy 组一次性蒸馏。 */
|
|
5
|
+
sessionId: string;
|
|
6
|
+
}
|
|
2
7
|
/** 三档蒸馏缓冲桶(off 在捕获侧已被拦截,永远不到这里)。 */
|
|
3
8
|
export interface PendingBuckets {
|
|
4
|
-
auto:
|
|
5
|
-
chat:
|
|
6
|
-
work:
|
|
9
|
+
auto: PendingMessage[];
|
|
10
|
+
chat: PendingMessage[];
|
|
11
|
+
work: PendingMessage[];
|
|
7
12
|
}
|
|
13
|
+
/** 旧格式(无 sessionId 字段)条目加载时归属的会话组。 */
|
|
14
|
+
export declare const LEGACY_SESSION = "legacy";
|
|
15
|
+
/** 各档位桶的渐进阈值状态(0 = 已毕业用稳态值;1 = 爬坡起点;ADR-0003)。 */
|
|
16
|
+
export type WarmupState = Record<ExtractMode, number>;
|
|
17
|
+
/** 全新起步:三档都从 1 爬坡(首轮即触发抽取)。 */
|
|
18
|
+
export declare function freshWarmup(): WarmupState;
|
|
8
19
|
export declare function emptyPending(): PendingBuckets;
|
|
9
|
-
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
10
|
-
|
|
20
|
+
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
21
|
+
* 旧格式条目(无 sessionId)归 legacy 组;warmup 缺省 = 全新起步。 */
|
|
22
|
+
export declare function loadPending(file: string, logger?: MemoryLogger): Promise<{
|
|
23
|
+
buckets: PendingBuckets;
|
|
24
|
+
warmup: WarmupState;
|
|
25
|
+
}>;
|
|
26
|
+
/** 按会话分组(会话切片):组按首条时间排序、组内按时间稳定排序——
|
|
27
|
+
* 蒸馏的一切触发都以切片为单位,切片内永不跨会话混装(ADR-0003)。 */
|
|
28
|
+
export declare function groupPendingBySession(messages: PendingMessage[]): Array<{
|
|
29
|
+
sessionId: string;
|
|
30
|
+
messages: PendingMessage[];
|
|
31
|
+
}>;
|
|
11
32
|
/** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
|
|
12
|
-
export declare function savePending(file: string, buckets: PendingBuckets): Promise<void>;
|
|
33
|
+
export declare function savePending(file: string, buckets: PendingBuckets, warmup?: WarmupState): Promise<void>;
|
|
13
34
|
export declare function pendingPathFor(dataDir: string): string;
|
|
14
35
|
/** 三档 key(调度/遍历用)。 */
|
|
15
36
|
export declare const PENDING_MODES: readonly ExtractMode[];
|
package/dist/store/pending.js
CHANGED
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import * as path from 'node:path';
|
|
9
9
|
import { atomicWriteJson, readJsonIfExists } from './io.js';
|
|
10
|
+
/** 旧格式(无 sessionId 字段)条目加载时归属的会话组。 */
|
|
11
|
+
export const LEGACY_SESSION = 'legacy';
|
|
12
|
+
/** 全新起步:三档都从 1 爬坡(首轮即触发抽取)。 */
|
|
13
|
+
export function freshWarmup() {
|
|
14
|
+
return { auto: 1, chat: 1, work: 1 };
|
|
15
|
+
}
|
|
10
16
|
export function emptyPending() {
|
|
11
17
|
return { auto: [], chat: [], work: [] };
|
|
12
18
|
}
|
|
@@ -16,7 +22,8 @@ function isMessage(m) {
|
|
|
16
22
|
const r = m;
|
|
17
23
|
return typeof r.id === 'string' && typeof r.content === 'string' && (r.role === 'user' || r.role === 'assistant');
|
|
18
24
|
}
|
|
19
|
-
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
25
|
+
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
26
|
+
* 旧格式条目(无 sessionId)归 legacy 组;warmup 缺省 = 全新起步。 */
|
|
20
27
|
export async function loadPending(file, logger) {
|
|
21
28
|
const out = emptyPending();
|
|
22
29
|
let raw;
|
|
@@ -26,27 +33,60 @@ export async function loadPending(file, logger) {
|
|
|
26
33
|
catch {
|
|
27
34
|
raw = undefined;
|
|
28
35
|
}
|
|
29
|
-
if (!raw || typeof raw !== 'object' || !raw.buckets || typeof raw.buckets !== 'object')
|
|
30
|
-
return out;
|
|
36
|
+
if (!raw || typeof raw !== 'object' || !raw.buckets || typeof raw.buckets !== 'object') {
|
|
37
|
+
return { buckets: out, warmup: freshWarmup() };
|
|
38
|
+
}
|
|
31
39
|
let dropped = 0;
|
|
40
|
+
let legacy = 0;
|
|
32
41
|
for (const key of ['auto', 'chat', 'work']) {
|
|
33
42
|
const arr = raw.buckets[key];
|
|
34
43
|
if (!Array.isArray(arr))
|
|
35
44
|
continue;
|
|
36
45
|
for (const m of arr) {
|
|
37
|
-
if (isMessage(m))
|
|
38
|
-
out[key].push(m);
|
|
39
|
-
else
|
|
46
|
+
if (!isMessage(m)) {
|
|
40
47
|
dropped++;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const sid = m.sessionId;
|
|
51
|
+
if (typeof sid === 'string' && sid) {
|
|
52
|
+
out[key].push({ ...m, sessionId: sid });
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
legacy++;
|
|
56
|
+
out[key].push({ ...m, sessionId: LEGACY_SESSION });
|
|
57
|
+
}
|
|
41
58
|
}
|
|
42
59
|
}
|
|
43
60
|
if (dropped > 0)
|
|
44
61
|
logger?.warn(`[memory] 未蒸馏缓冲文件含 ${dropped} 条坏记录,已丢弃`);
|
|
45
|
-
|
|
62
|
+
if (legacy > 0)
|
|
63
|
+
logger?.info(`[memory] 未蒸馏缓冲含 ${legacy} 条旧格式条目,归入 legacy 会话组`);
|
|
64
|
+
const warmup = freshWarmup();
|
|
65
|
+
for (const key of ['auto', 'chat', 'work']) {
|
|
66
|
+
const w = raw.warmup?.[key];
|
|
67
|
+
if (typeof w === 'number' && Number.isFinite(w) && w >= 0)
|
|
68
|
+
warmup[key] = Math.floor(w);
|
|
69
|
+
}
|
|
70
|
+
return { buckets: out, warmup };
|
|
71
|
+
}
|
|
72
|
+
/** 按会话分组(会话切片):组按首条时间排序、组内按时间稳定排序——
|
|
73
|
+
* 蒸馏的一切触发都以切片为单位,切片内永不跨会话混装(ADR-0003)。 */
|
|
74
|
+
export function groupPendingBySession(messages) {
|
|
75
|
+
const groups = new Map();
|
|
76
|
+
for (const m of messages) {
|
|
77
|
+
const g = groups.get(m.sessionId);
|
|
78
|
+
if (g)
|
|
79
|
+
g.push(m);
|
|
80
|
+
else
|
|
81
|
+
groups.set(m.sessionId, [m]);
|
|
82
|
+
}
|
|
83
|
+
return [...groups.entries()]
|
|
84
|
+
.map(([sessionId, msgs]) => ({ sessionId, messages: [...msgs].sort((a, b) => a.timestamp - b.timestamp) }))
|
|
85
|
+
.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
|
46
86
|
}
|
|
47
87
|
/** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
|
|
48
|
-
export async function savePending(file, buckets) {
|
|
49
|
-
const payload = { version: 1, buckets };
|
|
88
|
+
export async function savePending(file, buckets, warmup) {
|
|
89
|
+
const payload = { version: 1, buckets, ...(warmup ? { warmup } : {}) };
|
|
50
90
|
await atomicWriteJson(file, payload);
|
|
51
91
|
}
|
|
52
92
|
export function pendingPathFor(dataDir) {
|
|
@@ -28,14 +28,17 @@ export declare class RuntimeInstaller {
|
|
|
28
28
|
private readonly target;
|
|
29
29
|
private readonly logger?;
|
|
30
30
|
private readonly spawnImpl;
|
|
31
|
+
/** 随包 lockfile 路径(测试可注入;默认取 dist 根下构建期拷入的资产)。 */
|
|
32
|
+
private readonly lockfileSource;
|
|
31
33
|
private progress;
|
|
32
34
|
private child;
|
|
33
35
|
private current;
|
|
34
|
-
/** 安装超时(npm 卡死不罕见:registry 停滞即永挂,applyBusy
|
|
36
|
+
/** 安装超时(npm 卡死不罕见:registry 停滞即永挂,applyBusy 会被锁死)。每次 spawn 独立计时。 */
|
|
35
37
|
private static INSTALL_TIMEOUT_MS;
|
|
36
38
|
constructor(dataDir: string, targetVersion: string, opts?: {
|
|
37
39
|
logger?: MemoryLogger;
|
|
38
40
|
spawnImpl?: SpawnImpl;
|
|
41
|
+
lockfileSource?: string;
|
|
39
42
|
});
|
|
40
43
|
/** 包内模块名(与钉死版本一起构成安装目标)。 */
|
|
41
44
|
static packageName: string;
|
|
@@ -56,5 +59,7 @@ export declare class RuntimeInstaller {
|
|
|
56
59
|
/** 从 runtime 目录解析已安装的 transformers 模块(LocalEmbeddingService 用)。 */
|
|
57
60
|
resolveModule(): unknown;
|
|
58
61
|
private pushLine;
|
|
62
|
+
/** 跑一次 npm 子进程(采集尾行 + 超时 kill),返回退出码(null = 被杀死/启动失败)。 */
|
|
63
|
+
private runNpm;
|
|
59
64
|
private installOnce;
|
|
60
65
|
}
|
|
@@ -4,7 +4,11 @@
|
|
|
4
4
|
* 插件 npm 包本体不带重依赖,不用本地嵌入的用户零成本。
|
|
5
5
|
*
|
|
6
6
|
* - 安装位置:数据目录 runtime/(自带 package.json 锚定,防 npm 向上层目录逃逸安装);
|
|
7
|
-
* - 子进程 npm
|
|
7
|
+
* - 子进程 npm ci --ignore-scripts --no-audit --no-fund,钉死精确版本 + 随包 lockfile
|
|
8
|
+
* (resources/runtime-package-lock.json,构建期拷入 dist/)锁定完整传递依赖树——
|
|
9
|
+
* 纯 install 只锁直接依赖的精确版本,传递依赖按 semver 浮动解析,registry 端
|
|
10
|
+
* 后续发布/投毒会随安装时间漂移;lockfile 把树冻结在作者侧。ci 失败(lock 与
|
|
11
|
+
* package.json 漂移等)自动回退 npm install 精确版本(可用性优先,树不再受锁);
|
|
8
12
|
* - 进度(用户硬性要求:不能傻等):npm 非交互模式无百分比 API,采用不确定进度——
|
|
9
13
|
* 已耗时 + 子进程 stdout/stderr 尾行实时流出 + 可 kill;
|
|
10
14
|
* - 幂等:已装版本 == 目标版本直接就绪;版本漂移(插件升级换了钉死版本)重装覆盖。
|
|
@@ -13,6 +17,7 @@ import { spawn } from 'node:child_process';
|
|
|
13
17
|
import { createRequire } from 'node:module';
|
|
14
18
|
import { promises as fs } from 'node:fs';
|
|
15
19
|
import * as path from 'node:path';
|
|
20
|
+
import { fileURLToPath } from 'node:url';
|
|
16
21
|
/** 钉死的 transformers.js 版本(D8:精确版本,升级插件时在此变更并重装运行时)。 */
|
|
17
22
|
export const PINNED_TRANSFORMERS_VERSION = '4.2.0';
|
|
18
23
|
export class RuntimeInstaller {
|
|
@@ -20,15 +25,19 @@ export class RuntimeInstaller {
|
|
|
20
25
|
target;
|
|
21
26
|
logger;
|
|
22
27
|
spawnImpl;
|
|
28
|
+
/** 随包 lockfile 路径(测试可注入;默认取 dist 根下构建期拷入的资产)。 */
|
|
29
|
+
lockfileSource;
|
|
23
30
|
progress;
|
|
24
31
|
child = null;
|
|
25
32
|
current = null;
|
|
26
|
-
/** 安装超时(npm 卡死不罕见:registry 停滞即永挂,applyBusy
|
|
33
|
+
/** 安装超时(npm 卡死不罕见:registry 停滞即永挂,applyBusy 会被锁死)。每次 spawn 独立计时。 */
|
|
27
34
|
static INSTALL_TIMEOUT_MS = 10 * 60_000;
|
|
28
35
|
constructor(dataDir, targetVersion, opts) {
|
|
29
36
|
this.runtimeDir = path.join(dataDir, 'runtime');
|
|
30
37
|
this.target = targetVersion;
|
|
31
38
|
this.logger = opts?.logger;
|
|
39
|
+
this.lockfileSource =
|
|
40
|
+
opts?.lockfileSource ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'runtime-package-lock.json');
|
|
32
41
|
this.spawnImpl =
|
|
33
42
|
opts?.spawnImpl ??
|
|
34
43
|
((command, args, cwd) => {
|
|
@@ -127,41 +136,71 @@ export class RuntimeInstaller {
|
|
|
127
136
|
if (lines.length > 5)
|
|
128
137
|
lines.splice(0, lines.length - 5);
|
|
129
138
|
}
|
|
139
|
+
/** 跑一次 npm 子进程(采集尾行 + 超时 kill),返回退出码(null = 被杀死/启动失败)。 */
|
|
140
|
+
async runNpm(args) {
|
|
141
|
+
const child = this.spawnImpl('npm', args, this.runtimeDir);
|
|
142
|
+
this.child = child;
|
|
143
|
+
child.onStdout((l) => this.pushLine(l));
|
|
144
|
+
child.onStderr((l) => this.pushLine(l));
|
|
145
|
+
const timeout = setTimeout(() => {
|
|
146
|
+
this.pushLine('安装超时(10 分钟),终止子进程');
|
|
147
|
+
child.kill();
|
|
148
|
+
}, RuntimeInstaller.INSTALL_TIMEOUT_MS);
|
|
149
|
+
const code = await child.exited;
|
|
150
|
+
clearTimeout(timeout);
|
|
151
|
+
this.child = null;
|
|
152
|
+
return code;
|
|
153
|
+
}
|
|
130
154
|
async installOnce() {
|
|
131
|
-
// 锚定 package.json
|
|
155
|
+
// 锚定 package.json(带精确依赖):没有它 npm 会向上层目录找最近的 package.json 安装(逃逸事故);
|
|
156
|
+
// npm ci 还要求它与 lockfile 根条目一致——每次安装都写规范化形状,覆盖历史遗留/被 npm 改写的副本。
|
|
132
157
|
await fs.mkdir(this.runtimeDir, { recursive: true });
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
158
|
+
const manifest = {
|
|
159
|
+
name: 'dsh-memory-runtime',
|
|
160
|
+
private: true,
|
|
161
|
+
dependencies: { [RuntimeInstaller.packageName]: this.target },
|
|
162
|
+
};
|
|
163
|
+
await fs.writeFile(path.join(this.runtimeDir, 'package.json'), JSON.stringify(manifest, null, 2));
|
|
140
164
|
this.progress = {
|
|
141
165
|
phase: 'installing',
|
|
142
166
|
targetVersion: this.target,
|
|
143
167
|
installedVersion: await this.installedVersion(),
|
|
144
168
|
startedAt: Date.now(),
|
|
145
169
|
elapsedMs: 0,
|
|
146
|
-
lastLines: [
|
|
170
|
+
lastLines: [],
|
|
147
171
|
};
|
|
148
|
-
const child = this.spawnImpl('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--loglevel', 'notice', `${RuntimeInstaller.packageName}@${this.target}`], this.runtimeDir);
|
|
149
|
-
this.child = child;
|
|
150
|
-
child.onStdout((l) => this.pushLine(l));
|
|
151
|
-
child.onStderr((l) => this.pushLine(l));
|
|
152
|
-
const timeout = setTimeout(() => {
|
|
153
|
-
this.pushLine('安装超时(10 分钟),终止子进程');
|
|
154
|
-
child.kill();
|
|
155
|
-
}, RuntimeInstaller.INSTALL_TIMEOUT_MS);
|
|
156
172
|
this.logger?.info(`[memory] 运行时安装开始: ${RuntimeInstaller.packageName}@${this.target} → ${this.runtimeDir}`);
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
173
|
+
// 首选 npm ci + 随包 lockfile(传递依赖树冻结在作者侧);lockfile 资产缺失(npm 包被裁剪等)
|
|
174
|
+
// 或 ci 失败(lock 与锚定版本漂移等)回退 npm install 精确版本——可用性优先。
|
|
175
|
+
let code = null;
|
|
176
|
+
let usedCi = false;
|
|
177
|
+
try {
|
|
178
|
+
const lock = await fs.readFile(this.lockfileSource, 'utf8');
|
|
179
|
+
await fs.writeFile(path.join(this.runtimeDir, 'package-lock.json'), lock);
|
|
180
|
+
this.pushLine(`npm ci(随包 lockfile 锁定依赖树,@${this.target},--ignore-scripts)`);
|
|
181
|
+
usedCi = true;
|
|
182
|
+
code = await this.runNpm(['ci', '--ignore-scripts', '--no-audit', '--no-fund', '--loglevel', 'notice']);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
/* 无随包 lockfile,直接走 install 回退 */
|
|
186
|
+
}
|
|
187
|
+
if (!usedCi || code !== 0) {
|
|
188
|
+
if (usedCi) {
|
|
189
|
+
this.pushLine('npm ci 失败(lockfile 与钉死版本漂移?),回退 npm install');
|
|
190
|
+
this.logger?.warn('[memory] 运行时 npm ci 失败,回退 npm install(传递依赖不再受随包 lockfile 锁定)');
|
|
191
|
+
}
|
|
192
|
+
this.pushLine(`npm install ${RuntimeInstaller.packageName}@${this.target}(--ignore-scripts)`);
|
|
193
|
+
code = await this.runNpm([
|
|
194
|
+
'install',
|
|
195
|
+
'--ignore-scripts',
|
|
196
|
+
'--no-audit',
|
|
197
|
+
'--no-fund',
|
|
198
|
+
'--loglevel',
|
|
199
|
+
'notice',
|
|
200
|
+
`${RuntimeInstaller.packageName}@${this.target}`,
|
|
201
|
+
]);
|
|
202
|
+
}
|
|
163
203
|
this.progress.elapsedMs = Date.now() - this.progress.startedAt;
|
|
164
|
-
this.child = null;
|
|
165
204
|
const version = await this.installedVersion();
|
|
166
205
|
this.progress.installedVersion = version;
|
|
167
206
|
if (this.progress.phase === 'cancelled') {
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
* - rrfMerge:RRF(Reciprocal Rank Fusion,k=60)多路结果融合,hybrid 检索用;
|
|
4
4
|
* - bm25RankToScore:FTS5 bm25 rank(负值=更相关)转 0~1 分数;
|
|
5
5
|
* - buildFtsQuery / tokenizeForFts:FTS5 查询构造与写入侧分词。
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* 分词走 util/text.ts 的 tokenize(jieba 词 + CJK 二元组并集,@node-rs/jieba
|
|
7
|
+
* 预编译二进制,加载失败自动回退纯二元组),读写两侧共用同一分词器,
|
|
8
|
+
* 保证查询 token 与索引 token 对齐;FTS 索引按分词器版本戳自动重建(sqlite.ts)。
|
|
8
9
|
*/
|
|
9
10
|
import { tokenize } from '../util/text.js';
|
|
10
11
|
/** 标准 RRF 常数(原论文值);k 越大越偏向低排名项(分布更平滑)。 */
|
|
@@ -7,6 +7,8 @@ export declare class SessionModeStore {
|
|
|
7
7
|
private readonly entries;
|
|
8
8
|
private readonly loaded;
|
|
9
9
|
private persistFailed;
|
|
10
|
+
/** 档位切换回调(index.ts 装配 runner 的同步动作:切片落袋/挂起,ADR-0003)。 */
|
|
11
|
+
private onModeChange?;
|
|
10
12
|
/** 串行化持久化写(避免并发原子写撞临时文件名)。 */
|
|
11
13
|
private writeChain;
|
|
12
14
|
constructor(dataDir: string, defaultMode: Extract<MemoryMode, 'auto' | 'chat' | 'work'>, logger?: MemoryLogger | undefined);
|
|
@@ -15,6 +17,8 @@ export declare class SessionModeStore {
|
|
|
15
17
|
get default(): MemoryMode;
|
|
16
18
|
/** 同步读取:未设置过的会话返回默认档。 */
|
|
17
19
|
get(sessionId: string): MemoryMode;
|
|
20
|
+
/** 注册档位切换回调(同步调用;回调异常只记日志不阻断写穿)。 */
|
|
21
|
+
setModeChangeHandler(cb: (sessionId: string, oldMode: MemoryMode, newMode: MemoryMode) => void): void;
|
|
18
22
|
/** 设置会话档位(写穿持久化;持久化失败保持内存态生效)。 */
|
|
19
23
|
set(sessionId: string, mode: MemoryMode): void;
|
|
20
24
|
/** 等待在途持久化写完成(测试/停机用)。 */
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* 存储失败只降级为内存态(warn 不崩),与插件的存储降级不变量一致。
|
|
6
6
|
*/
|
|
7
7
|
import * as path from 'node:path';
|
|
8
|
+
import { errDetail } from '../util/filelog.js';
|
|
8
9
|
import { atomicWriteJson, ensureDir, readJsonIfExists } from './io.js';
|
|
9
10
|
const MODES = ['auto', 'chat', 'work', 'off'];
|
|
10
11
|
const PRUNE_MS = 90 * 24 * 3600_000;
|
|
@@ -19,6 +20,8 @@ export class SessionModeStore {
|
|
|
19
20
|
entries = new Map();
|
|
20
21
|
loaded;
|
|
21
22
|
persistFailed = false;
|
|
23
|
+
/** 档位切换回调(index.ts 装配 runner 的同步动作:切片落袋/挂起,ADR-0003)。 */
|
|
24
|
+
onModeChange;
|
|
22
25
|
/** 串行化持久化写(避免并发原子写撞临时文件名)。 */
|
|
23
26
|
writeChain = Promise.resolve();
|
|
24
27
|
constructor(dataDir, defaultMode, logger) {
|
|
@@ -52,10 +55,23 @@ export class SessionModeStore {
|
|
|
52
55
|
get(sessionId) {
|
|
53
56
|
return this.entries.get(sessionId)?.mode ?? this.loaded;
|
|
54
57
|
}
|
|
58
|
+
/** 注册档位切换回调(同步调用;回调异常只记日志不阻断写穿)。 */
|
|
59
|
+
setModeChangeHandler(cb) {
|
|
60
|
+
this.onModeChange = cb;
|
|
61
|
+
}
|
|
55
62
|
/** 设置会话档位(写穿持久化;持久化失败保持内存态生效)。 */
|
|
56
63
|
set(sessionId, mode) {
|
|
64
|
+
const old = this.get(sessionId);
|
|
57
65
|
this.entries.set(sessionId, { mode, updatedAt: Date.now() });
|
|
58
66
|
this.writeChain = this.writeChain.then(() => this.persist());
|
|
67
|
+
if (old !== mode && this.onModeChange) {
|
|
68
|
+
try {
|
|
69
|
+
this.onModeChange(sessionId, old, mode);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
this.logger?.warn(`[memory] 档位切换回调失败: ${errDetail(err)}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
59
75
|
}
|
|
60
76
|
/** 等待在途持久化写完成(测试/停机用)。 */
|
|
61
77
|
flush() {
|
package/dist/store/sqlite.d.ts
CHANGED
|
@@ -34,6 +34,8 @@ export declare class MemoryDb {
|
|
|
34
34
|
private readonly logger?;
|
|
35
35
|
private stmtUpsertL1;
|
|
36
36
|
private stmtGetL1;
|
|
37
|
+
/** 主表存在性点查(防御性 FTS 删除的前置判断,走主键索引)。 */
|
|
38
|
+
private stmtL1Exists;
|
|
37
39
|
private stmtDeleteL1Meta;
|
|
38
40
|
private stmtDeleteL1Vec?;
|
|
39
41
|
private stmtInsertL1Vec?;
|
|
@@ -44,6 +46,8 @@ export declare class MemoryDb {
|
|
|
44
46
|
private stmtL1FtsSearchFamily;
|
|
45
47
|
private stmtUpsertL0;
|
|
46
48
|
private stmtGetL0;
|
|
49
|
+
/** 主表存在性点查(同 L1:防御性 FTS 删除的前置判断)。 */
|
|
50
|
+
private stmtL0Exists;
|
|
47
51
|
private stmtDeleteL0Vec?;
|
|
48
52
|
private stmtInsertL0Vec?;
|
|
49
53
|
private stmtSearchL0Vec?;
|
|
@@ -81,8 +85,13 @@ export declare class MemoryDb {
|
|
|
81
85
|
private hasColumn;
|
|
82
86
|
/** 重建后的 l1_fts 从 l1_records 全量回灌(仅在 drop 重建时调用)。 */
|
|
83
87
|
private backfillL1Fts;
|
|
88
|
+
/** 重建后的 l0_fts 从 l0_conversations 全量回灌(仅 drop 重建时调用;iterate 流式防大库内存峰值)。 */
|
|
89
|
+
private backfillL0Fts;
|
|
84
90
|
private readEmbeddingMeta;
|
|
85
91
|
private writeEmbeddingMeta;
|
|
92
|
+
/** 通用字符串 kv(embedding_meta 表兼作元数据 kv 存储,如 FTS 分词器版本戳)。 */
|
|
93
|
+
private readMetaString;
|
|
94
|
+
private writeMetaString;
|
|
86
95
|
/**
|
|
87
96
|
* 持久化 embedding meta(语义:物理向量表当前对应的 provider/维度)。
|
|
88
97
|
* 活切换在 swapProvider 成功后即写(表已是新维度);启动/补齐链在
|
|
@@ -139,6 +148,9 @@ export declare class MemoryDb {
|
|
|
139
148
|
countL0(): number;
|
|
140
149
|
/** 统计 recorded_at >= iso 的消息数(状态面板"今日捕获"用)。 */
|
|
141
150
|
countL0Since(iso: string): number;
|
|
151
|
+
/** 按会话取最近消息(时间升序返回;走 idx_l0_session_id 索引)。
|
|
152
|
+
* 蒸馏背景参考专用——按会话现查替代全局内存数组(ADR-0003)。 */
|
|
153
|
+
recentL0BySession(sessionId: string, limit: number): L0MessageRecord[];
|
|
142
154
|
/** L0 全量列举(重建快照用;按时间升序,事务一致性避开 JSONL 追加竞态)。 */
|
|
143
155
|
listL0All(): L0MessageRecord[];
|
|
144
156
|
/** 重建成本预估(一次全表聚合:会话数 / 消息数 / 字符量)。 */
|
|
@@ -176,6 +188,21 @@ export declare class MemoryDb {
|
|
|
176
188
|
/** 只更新向量行(重嵌入用)。 */
|
|
177
189
|
updateL1Vec(id: string, embedding: Float32Array): boolean;
|
|
178
190
|
updateL0Vec(id: string, embedding: Float32Array, recordedAt: string): boolean;
|
|
191
|
+
/**
|
|
192
|
+
* 批量更新 L1 向量行(重嵌入热路径):单事务写入整批,替代逐条裸写——
|
|
193
|
+
* 逐条每行一次隐式事务,批量场景(万级记录重嵌)开销集中在 fsync 上。
|
|
194
|
+
* 整批失败回退逐条:好行照常入库,坏行只丢自身(向量行 id 寻址,无顺序依赖)。
|
|
195
|
+
* 返回成功写入的行数(零向量行防御性跳过、不计入)。
|
|
196
|
+
*/
|
|
197
|
+
updateL1VecBatch(items: Array<{
|
|
198
|
+
id: string;
|
|
199
|
+
embedding: Float32Array;
|
|
200
|
+
}>): number;
|
|
201
|
+
/** L0 版 updateL1VecBatch(语义同:单事务 + 失败回退逐条)。recordedAt 整批统一。 */
|
|
202
|
+
updateL0VecBatch(items: Array<{
|
|
203
|
+
id: string;
|
|
204
|
+
embedding: Float32Array;
|
|
205
|
+
}>, recordedAt: string): number;
|
|
179
206
|
close(): void;
|
|
180
207
|
}
|
|
181
208
|
/** 全零向量(cosine 未定义,不可入向量表)。reindex 侧用它区分"不可嵌入"与"写入失败"。 */
|