dsh-layered-memory 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +101 -81
- package/README.md +88 -80
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/dist/client.js +1100 -402
- package/dist/config.d.ts +12 -0
- package/dist/config.js +20 -17
- package/dist/hooks/capture.d.ts +17 -1
- package/dist/hooks/capture.js +45 -11
- package/dist/hooks/recall.d.ts +7 -0
- package/dist/hooks/recall.js +20 -4
- package/dist/index.d.ts +8 -0
- package/dist/index.js +107 -44
- package/dist/pipeline/rebuild.d.ts +2 -0
- package/dist/pipeline/rebuild.js +7 -0
- package/dist/pipeline/runner.d.ts +7 -1
- package/dist/pipeline/runner.js +22 -7
- package/dist/settings.js +79 -20
- package/dist/stats.d.ts +7 -1
- package/dist/stats.js +131 -14
- package/dist/store/download-queue.d.ts +71 -0
- package/dist/store/download-queue.js +313 -0
- package/dist/store/embedding-source.d.ts +160 -0
- package/dist/store/embedding-source.js +421 -0
- package/dist/store/embedding.d.ts +3 -1
- package/dist/store/embedding.js +5 -0
- package/dist/store/l0.d.ts +13 -5
- package/dist/store/l0.js +33 -6
- package/dist/store/l1.d.ts +15 -5
- package/dist/store/l1.js +40 -15
- package/dist/store/local-embedding.d.ts +64 -0
- package/dist/store/local-embedding.js +120 -0
- package/dist/store/model-catalog.d.ts +45 -0
- package/dist/store/model-catalog.js +78 -0
- package/dist/store/runtime-installer.d.ts +60 -0
- package/dist/store/runtime-installer.js +181 -0
- package/dist/store/sqlite.d.ts +53 -7
- package/dist/store/sqlite.js +339 -72
- package/dist/tools/index.js +6 -4
- package/dist/util/filelog.d.ts +2 -0
- package/dist/util/filelog.js +20 -3
- package/package.json +1 -1
- package/assets/readme/hero.svg +0 -58
package/dist/pipeline/runner.js
CHANGED
|
@@ -32,6 +32,8 @@ export class MemoryRunner {
|
|
|
32
32
|
live;
|
|
33
33
|
tasks = [];
|
|
34
34
|
draining = false;
|
|
35
|
+
/** 停止标志(dispose 序置位):不再取新任务;进行中任务自然收尾,其 DB 写入失败由各层兜底捕获。 */
|
|
36
|
+
stopped = false;
|
|
35
37
|
pending = emptyPending();
|
|
36
38
|
pendingFile;
|
|
37
39
|
background = [];
|
|
@@ -104,7 +106,13 @@ export class MemoryRunner {
|
|
|
104
106
|
runRebuildTurn(sessionId, messages) {
|
|
105
107
|
return this.runTurn(sessionId, messages, 'auto', { noBufferCap: true });
|
|
106
108
|
}
|
|
109
|
+
/** 停止取新任务(插件 dispose 序调用;进行中任务照常跑完但不 await——LLM 慢调用不拖住宿主卸载)。 */
|
|
110
|
+
stop() {
|
|
111
|
+
this.stopped = true;
|
|
112
|
+
}
|
|
107
113
|
pushTask(task) {
|
|
114
|
+
if (this.stopped)
|
|
115
|
+
return;
|
|
108
116
|
this.tasks.push(task);
|
|
109
117
|
void this.drain();
|
|
110
118
|
}
|
|
@@ -113,7 +121,7 @@ export class MemoryRunner {
|
|
|
113
121
|
return;
|
|
114
122
|
this.draining = true;
|
|
115
123
|
try {
|
|
116
|
-
while (this.tasks.length > 0) {
|
|
124
|
+
while (!this.stopped && this.tasks.length > 0) {
|
|
117
125
|
const [task] = this.tasks.splice(pickNextTaskIndex(this.tasks), 1);
|
|
118
126
|
try {
|
|
119
127
|
await task.run();
|
|
@@ -127,9 +135,18 @@ export class MemoryRunner {
|
|
|
127
135
|
this.draining = false;
|
|
128
136
|
}
|
|
129
137
|
}
|
|
130
|
-
/** 缓冲落盘(每次蒸馏尝试后调用;失败只告警不阻断管线)。
|
|
131
|
-
|
|
138
|
+
/** 缓冲落盘(每次蒸馏尝试后调用;失败只告警不阻断管线)。
|
|
139
|
+
* 非重建轮持久化前按桶截断到上限:重建取消后的大桶不至于在后续每次
|
|
140
|
+
* 蒸馏尝试时反复整量序列化落盘(多 MB 级 IO);重建轮豁免维持。 */
|
|
141
|
+
async persistPending(noBufferCap = false) {
|
|
132
142
|
try {
|
|
143
|
+
if (!noBufferCap) {
|
|
144
|
+
for (const key of PENDING_MODES) {
|
|
145
|
+
const bucket = this.pending[key];
|
|
146
|
+
if (bucket.length > PENDING_BUCKET_CAP)
|
|
147
|
+
this.pending[key] = bucket.slice(-PENDING_BUCKET_CAP);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
133
150
|
await savePending(this.pendingFile, this.pending);
|
|
134
151
|
}
|
|
135
152
|
catch (err) {
|
|
@@ -161,13 +178,11 @@ export class MemoryRunner {
|
|
|
161
178
|
this.logger.info(`[memory] L1 阶段完成(${Date.now() - t}ms)`);
|
|
162
179
|
}
|
|
163
180
|
catch (err) {
|
|
164
|
-
// 保留 pending
|
|
165
|
-
if (!opts?.noBufferCap && this.pending[mode].length > PENDING_BUCKET_CAP)
|
|
166
|
-
this.pending[mode] = [];
|
|
181
|
+
// 保留 pending 下次重试(runTurn 入口已裁到 ≤200,防无限堆积;重建轮不裁,量被会话规模约束)
|
|
167
182
|
this.logger.warn(`[memory] L1 抽取失败(mode=${mode},pending=${this.pending[mode].length}): ${errDetail(err)}`);
|
|
168
183
|
}
|
|
169
184
|
// 缓冲每次尝试后立即落盘:进程中途退出不丢待重试/攒阈值状态
|
|
170
|
-
await this.persistPending();
|
|
185
|
+
await this.persistPending(opts?.noBufferCap);
|
|
171
186
|
// L1 计数推进后立即落盘:L2/L3 失败或进程中途退出不得回滚阈值进度
|
|
172
187
|
// (记录已入库但计数丢失会让该族 L2 永远差一截,state 与 DB 脱节)
|
|
173
188
|
try {
|
package/dist/settings.js
CHANGED
|
@@ -2,6 +2,23 @@ import Schema from '@deepseek-ai/schemastery';
|
|
|
2
2
|
import { settingsNamespace } from '@deepseek-ai/dsh-settings';
|
|
3
3
|
const NS = settingsNamespace('dsh-memory');
|
|
4
4
|
const ALWAYS_ON = { enabled: true, capture: true, distill: true, recall: true, reasoningEffort: '' };
|
|
5
|
+
/**
|
|
6
|
+
* 进程内 scope 复用(fiber 重启重挂)。
|
|
7
|
+
* dsh-settings 的 register 把注册挂在其**服务自身 ctx** 的 effect 上
|
|
8
|
+
* (node_modules/@deepseek-ai/dsh-settings/lib/index.js:`this.ctx.effect(...)`),
|
|
9
|
+
* 不随本插件 fiber 销毁——fiber 重启后二次 register 会抛
|
|
10
|
+
* `settings namespace "dsh-memory" is already registered`。模块级状态在
|
|
11
|
+
* fiber 重启间存活:复用上次注册的 scope 并重挂 watcher,否则开关读写停在
|
|
12
|
+
* stub、用户已存开关被静默忽略直到重启进程。
|
|
13
|
+
*
|
|
14
|
+
* 同时按服务实例(cachedSvc)判活:settings 服务自身重启时其注册随服务 ctx
|
|
15
|
+
* 销毁,旧 scope 变死引用(get 返回冻结旧值、update 抛 not registered、
|
|
16
|
+
* watcher 永不触发)——internal/service 携带的新实例与缓存不符时作废缓存、
|
|
17
|
+
* 向新实例重新注册(用户层由服务从磁盘重解析,已存开关不丢)。
|
|
18
|
+
*/
|
|
19
|
+
let cachedScope;
|
|
20
|
+
let cachedUnwatch;
|
|
21
|
+
let cachedSvc;
|
|
5
22
|
export function liveSettingsSchema() {
|
|
6
23
|
return Schema.object({
|
|
7
24
|
enabled: Schema.boolean().default(true),
|
|
@@ -18,28 +35,56 @@ export function registerLiveSettings(ctx, logger) {
|
|
|
18
35
|
get: () => ALWAYS_ON,
|
|
19
36
|
update: () => Promise.reject(new Error('settings 服务不可用')),
|
|
20
37
|
};
|
|
38
|
+
/** 挂接一个(新注册或复用的)scope:重挂前先摘旧 watcher,防跨重启累积。 */
|
|
39
|
+
const wireScope = (scope) => {
|
|
40
|
+
cachedUnwatch?.();
|
|
41
|
+
let current = resolveSettings(scope.get());
|
|
42
|
+
cachedUnwatch = scope.watch((next) => {
|
|
43
|
+
const prev = current;
|
|
44
|
+
current = resolveSettings(next);
|
|
45
|
+
logger.info(`[memory] 记忆模式开关更新:总=${current.enabled} 捕获=${current.capture} 蒸馏=${current.distill} 召回=${current.recall}` +
|
|
46
|
+
`,蒸馏思考=${current.reasoningEffort || '跟随配置'}(此前 总=${prev.enabled})`);
|
|
47
|
+
});
|
|
48
|
+
return {
|
|
49
|
+
supported: true,
|
|
50
|
+
get: () => current,
|
|
51
|
+
update: async (patch) => {
|
|
52
|
+
await scope.update(patch);
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
};
|
|
56
|
+
/** 作废进程内缓存(服务下线/实例替换时旧注册已随服务销毁)。 */
|
|
57
|
+
const invalidateCache = () => {
|
|
58
|
+
cachedScope = undefined;
|
|
59
|
+
cachedSvc = undefined;
|
|
60
|
+
cachedUnwatch?.();
|
|
61
|
+
cachedUnwatch = undefined;
|
|
62
|
+
};
|
|
21
63
|
const tryAttach = () => {
|
|
22
64
|
const settings = ctx.get('settings');
|
|
23
65
|
if (!settings)
|
|
24
66
|
return false;
|
|
67
|
+
// 仅当缓存来自同一服务实例时才可复用——换了实例(服务重启/替换)就重新注册
|
|
68
|
+
if (cachedScope && cachedSvc === settings) {
|
|
69
|
+
try {
|
|
70
|
+
inner = wireScope(cachedScope);
|
|
71
|
+
const c = inner.get();
|
|
72
|
+
logger.info(`[memory] 记忆模式开关重挂(复用进程内注册,当前:总=${c.enabled} 捕获=${c.capture} 蒸馏=${c.distill} 召回=${c.recall}` +
|
|
73
|
+
`,蒸馏思考=${c.reasoningEffort || '跟随配置'})`);
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
logger.warn(`[memory] 记忆模式开关缓存复用失败,改为重新注册: ${err instanceof Error ? err.message : String(err)}`);
|
|
78
|
+
invalidateCache();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
25
81
|
try {
|
|
26
82
|
const scope = settings.register(NS, liveSettingsSchema(), { applies: 'live' });
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
`,蒸馏思考=${current.reasoningEffort || '跟随配置'}(此前 总=${prev.enabled})`);
|
|
33
|
-
});
|
|
34
|
-
inner = {
|
|
35
|
-
supported: true,
|
|
36
|
-
get: () => current,
|
|
37
|
-
update: async (patch) => {
|
|
38
|
-
await scope.update(patch);
|
|
39
|
-
},
|
|
40
|
-
};
|
|
41
|
-
logger.info(`[memory] 记忆模式开关就绪(settings 命名空间 dsh-memory,当前:总=${current.enabled} 捕获=${current.capture} 蒸馏=${current.distill} 召回=${current.recall}` +
|
|
42
|
-
`,蒸馏思考=${current.reasoningEffort || '跟随配置'})`);
|
|
83
|
+
cachedScope = scope;
|
|
84
|
+
cachedSvc = settings;
|
|
85
|
+
inner = wireScope(scope);
|
|
86
|
+
logger.info(`[memory] 记忆模式开关就绪(settings 命名空间 dsh-memory,当前:总=${inner.get().enabled} 捕获=${inner.get().capture} 蒸馏=${inner.get().distill} 召回=${inner.get().recall}` +
|
|
87
|
+
`,蒸馏思考=${inner.get().reasoningEffort || '跟随配置'})`);
|
|
43
88
|
return true;
|
|
44
89
|
}
|
|
45
90
|
catch (err) {
|
|
@@ -49,11 +94,25 @@ export function registerLiveSettings(ctx, logger) {
|
|
|
49
94
|
};
|
|
50
95
|
if (!tryAttach()) {
|
|
51
96
|
logger.warn('[memory] settings 服务未就绪,记忆模式开关暂不可用(保持全开,等待服务上线)');
|
|
52
|
-
ctx.on('internal/service', (name) => {
|
|
53
|
-
if (name === 'settings' && !inner.supported)
|
|
54
|
-
tryAttach();
|
|
55
|
-
});
|
|
56
97
|
}
|
|
98
|
+
// 无论初始是否成功都监听服务迁移:下线 → 作废缓存;换实例 → 作废后立即重挂
|
|
99
|
+
// (此前监听仅在初始失败时注册,服务替换场景下没有任何自愈路径)
|
|
100
|
+
ctx.on('internal/service', (name, impl) => {
|
|
101
|
+
if (name !== 'settings')
|
|
102
|
+
return;
|
|
103
|
+
if (!impl) {
|
|
104
|
+
if (cachedSvc !== undefined) {
|
|
105
|
+
invalidateCache();
|
|
106
|
+
logger.warn('[memory] settings 服务下线,开关缓存已作废(期间读数为冻结值,恢复后自动重挂)');
|
|
107
|
+
}
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
// 实例变了才作废缓存;但 tryAttach 无条件执行(幂等)——同一事件会广播到
|
|
111
|
+
// 所有存活 fiber 的监听器,后跑的那个也必须修好自己闭包里的 inner
|
|
112
|
+
if (impl !== cachedSvc)
|
|
113
|
+
invalidateCache();
|
|
114
|
+
tryAttach();
|
|
115
|
+
});
|
|
57
116
|
return {
|
|
58
117
|
get supported() {
|
|
59
118
|
return inner.supported;
|
package/dist/stats.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type { L1Store } from './store/l1.js';
|
|
|
7
7
|
import type { PersonaStore } from './store/persona.js';
|
|
8
8
|
import type { SceneStore } from './store/scenes.js';
|
|
9
9
|
import type { SessionModeStore } from './store/session-modes.js';
|
|
10
|
+
import type { EmbeddingManager } from './store/embedding-source.js';
|
|
10
11
|
import type { StateStore } from './store/state.js';
|
|
11
12
|
import type { MemoryFamily, MemoryLogger } from './types.js';
|
|
12
13
|
export declare const PLUGIN_VERSION: string;
|
|
@@ -36,6 +37,11 @@ export interface MemoryStats {
|
|
|
36
37
|
memoriesSinceL3: number;
|
|
37
38
|
pendingExtract: number;
|
|
38
39
|
message: string;
|
|
40
|
+
/** 实际生效的阈值(概览进度分母用,避免 UI 硬编码与部署配置脱节)。 */
|
|
41
|
+
thresholds: {
|
|
42
|
+
l2MinNewMemories: number;
|
|
43
|
+
l3Interval: number;
|
|
44
|
+
};
|
|
39
45
|
}
|
|
40
46
|
/** 注册状态 RPC(web 侧 connection 服务可选,缺失时跳过,不影响插件主体)。 */
|
|
41
47
|
export declare function registerMemoryRpc(ctx: Context, cfg: MemoryConfig, stores: {
|
|
@@ -44,4 +50,4 @@ export declare function registerMemoryRpc(ctx: Context, cfg: MemoryConfig, store
|
|
|
44
50
|
scenes: Record<MemoryFamily, SceneStore>;
|
|
45
51
|
persona: Record<MemoryFamily, PersonaStore>;
|
|
46
52
|
state: StateStore;
|
|
47
|
-
}, logger: MemoryLogger, status?: MemoryStatusSource, live?: LiveSettingsHandle, modes?: SessionModeStore, dataDir?: string, rebuild?: RebuildController): void;
|
|
53
|
+
}, logger: MemoryLogger, status?: MemoryStatusSource, live?: LiveSettingsHandle, modes?: SessionModeStore, dataDir?: string, rebuild?: RebuildController, embedManager?: EmbeddingManager): void;
|
package/dist/stats.js
CHANGED
|
@@ -3,18 +3,21 @@
|
|
|
3
3
|
* Client 设置页通过 ctx.connection.rpc.call('/rpc', 'dsh-memory/stats') 拉取。
|
|
4
4
|
*
|
|
5
5
|
* connection 是可选服务且可能晚于本插件就绪:先探测一次,未就绪则监听
|
|
6
|
-
* internal/service
|
|
6
|
+
* internal/service(事件携带 (name, impl),impl=undefined 即下线),服务
|
|
7
|
+
* 上线、下线、替换实例三种迁移都会正确释放/重挂 RPC 注册。
|
|
7
8
|
*/
|
|
8
9
|
import { createRequire } from 'node:module';
|
|
9
|
-
import {
|
|
10
|
+
import { closeSync, openSync, readSync, statSync } from 'node:fs';
|
|
10
11
|
import { join } from 'node:path';
|
|
11
12
|
import { resolveDataDir } from './config.js';
|
|
12
13
|
const require = createRequire(import.meta.url);
|
|
13
14
|
export const PLUGIN_VERSION = require('../package.json').version;
|
|
14
15
|
/** 注册状态 RPC(web 侧 connection 服务可选,缺失时跳过,不影响插件主体)。 */
|
|
15
|
-
export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes, dataDir, rebuild) {
|
|
16
|
+
export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes, dataDir, rebuild, embedManager) {
|
|
16
17
|
/** 当前是否持有一段有效注册(dispose 完成后清空,允许服务重上线时重注册)。 */
|
|
17
18
|
let holding = false;
|
|
19
|
+
/** 当前 handle 绑定的 connection 实例(internal/service 第二参;用于识别实例替换)。 */
|
|
20
|
+
let registeredImpl;
|
|
18
21
|
const tryRegister = () => {
|
|
19
22
|
if (holding)
|
|
20
23
|
return;
|
|
@@ -35,6 +38,7 @@ export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes,
|
|
|
35
38
|
dataDir: dataDir ?? resolveDataDir(cfg),
|
|
36
39
|
logger,
|
|
37
40
|
rebuild,
|
|
41
|
+
embedManager,
|
|
38
42
|
});
|
|
39
43
|
return { ok: true, value };
|
|
40
44
|
}
|
|
@@ -45,6 +49,7 @@ export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes,
|
|
|
45
49
|
};
|
|
46
50
|
}
|
|
47
51
|
}, { authority: 'loopback' });
|
|
52
|
+
registeredImpl = connection;
|
|
48
53
|
if (!active) {
|
|
49
54
|
void dispose();
|
|
50
55
|
return;
|
|
@@ -56,17 +61,35 @@ export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes,
|
|
|
56
61
|
void dispose();
|
|
57
62
|
});
|
|
58
63
|
};
|
|
64
|
+
/** 释放全部持有注册(handle 随旧服务实例失效,holding 复位以允许重挂)。 */
|
|
65
|
+
const release = () => {
|
|
66
|
+
for (const dispose of disposers.splice(0))
|
|
67
|
+
dispose();
|
|
68
|
+
};
|
|
59
69
|
const disposers = [];
|
|
60
70
|
ctx.effect(() => {
|
|
61
71
|
tryRegister();
|
|
62
|
-
const off = ctx.on('internal/service', (name) => {
|
|
63
|
-
if (name
|
|
64
|
-
|
|
72
|
+
const off = ctx.on('internal/service', (name, impl) => {
|
|
73
|
+
if (name !== 'connection')
|
|
74
|
+
return;
|
|
75
|
+
if (!impl) {
|
|
76
|
+
// 服务下线:旧 handle 已随旧服务实例失效——主动释放并复位,
|
|
77
|
+
// 服务恢复时本事件再触发即可重挂(否则 holding 恒真 → RPC 永久失联)
|
|
78
|
+
release();
|
|
79
|
+
registeredImpl = undefined;
|
|
80
|
+
logger.debug?.('[memory] connection 服务下线,RPC 注册已释放(待恢复重挂)');
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
if (impl !== registeredImpl) {
|
|
84
|
+
// 实例替换:旧 handle 失效,换新实例重挂
|
|
85
|
+
release();
|
|
86
|
+
registeredImpl = undefined;
|
|
87
|
+
}
|
|
88
|
+
tryRegister();
|
|
65
89
|
});
|
|
66
90
|
return () => {
|
|
67
91
|
off();
|
|
68
|
-
|
|
69
|
-
dispose();
|
|
92
|
+
release();
|
|
70
93
|
};
|
|
71
94
|
});
|
|
72
95
|
}
|
|
@@ -102,10 +125,11 @@ async function buildStats(cfg, stores, status) {
|
|
|
102
125
|
memoriesSinceL3: chat.memoriesSinceL3 + work.memoriesSinceL3,
|
|
103
126
|
pendingExtract: status?.pending() ?? 0,
|
|
104
127
|
message: degraded ? 'degraded:存储不可用,记忆功能已停用' : 'running',
|
|
128
|
+
thresholds: { l2MinNewMemories: cfg.l2.minNewMemories, l3Interval: cfg.l3.interval },
|
|
105
129
|
};
|
|
106
130
|
}
|
|
107
131
|
async function handleEndpoint(endpoint, payload, deps) {
|
|
108
|
-
const { cfg, stores, status, live, modes, dataDir, rebuild } = deps;
|
|
132
|
+
const { cfg, stores, status, live, modes, dataDir, rebuild, embedManager } = deps;
|
|
109
133
|
switch (endpoint) {
|
|
110
134
|
case 'dsh-memory/stats':
|
|
111
135
|
return buildStats(cfg, stores, status);
|
|
@@ -172,14 +196,19 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
172
196
|
const p = (payload ?? {});
|
|
173
197
|
const limit = Math.min(Math.max(Number(p.limit) || 50, 1), 200);
|
|
174
198
|
const offset = Math.max(Number(p.offset) || 0, 0);
|
|
175
|
-
// 关键词路径:复用检索唯一缝(与召回同源),取回后做场景过滤 +
|
|
199
|
+
// 关键词路径:复用检索唯一缝(与召回同源),取回后做场景过滤 + 手工分页。
|
|
200
|
+
// 检索侧单次上限 200:分页窗口触达上限时显式标记 truncated(结果可能不完整),
|
|
201
|
+
// 不再静默返回空结果让用户误以为"没有更多"等于"不存在更多"。
|
|
176
202
|
if (p.query && p.query.trim()) {
|
|
177
|
-
const
|
|
203
|
+
const SEARCH_CAP = 200;
|
|
204
|
+
const wanted = offset + limit + 1;
|
|
205
|
+
const hits = await stores.l1.search(p.query, Math.min(wanted, SEARCH_CAP), { type: p.type || undefined });
|
|
178
206
|
const filtered = p.scene ? hits.filter((h) => h.scene_name === p.scene) : hits;
|
|
179
207
|
return {
|
|
180
208
|
items: filtered.slice(offset, offset + limit).map(hitToUiRecord),
|
|
181
209
|
hasMore: filtered.length > offset + limit,
|
|
182
210
|
total: null,
|
|
211
|
+
truncated: wanted > SEARCH_CAP,
|
|
183
212
|
scenes: offset === 0 ? stores.l1.distinctScenes() : undefined,
|
|
184
213
|
};
|
|
185
214
|
}
|
|
@@ -188,6 +217,7 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
188
217
|
items: items.map(hitToUiRecord),
|
|
189
218
|
hasMore: offset + items.length < total,
|
|
190
219
|
total,
|
|
220
|
+
truncated: false,
|
|
191
221
|
scenes: offset === 0 ? stores.l1.distinctScenes() : undefined,
|
|
192
222
|
};
|
|
193
223
|
}
|
|
@@ -240,6 +270,60 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
240
270
|
throw new Error('重建控制器未初始化');
|
|
241
271
|
return rebuild.requestCancel();
|
|
242
272
|
}
|
|
273
|
+
// ── 嵌入源(远程/本地/关闭 三态)与模型管理 ──
|
|
274
|
+
case 'dsh-memory/embedding-state-get': {
|
|
275
|
+
if (!embedManager)
|
|
276
|
+
return { supported: false };
|
|
277
|
+
return { supported: true, ...(await embedManager.snapshot()) };
|
|
278
|
+
}
|
|
279
|
+
case 'dsh-memory/embedding-source-set': {
|
|
280
|
+
if (!embedManager)
|
|
281
|
+
throw new Error('嵌入管理器未初始化(存储不可用)');
|
|
282
|
+
const p = (payload ?? {});
|
|
283
|
+
if (p.source !== 'remote' && p.source !== 'local' && p.source !== 'off') {
|
|
284
|
+
throw new Error('source 必须是 remote | local | off');
|
|
285
|
+
}
|
|
286
|
+
const r = embedManager.requestSource({ source: p.source, activeModel: p.activeModel ?? null });
|
|
287
|
+
if (!r.accepted)
|
|
288
|
+
throw new Error(r.error ?? '切换请求被拒绝');
|
|
289
|
+
deps.logger.info(`[memory] 收到嵌入源切换指令(source=${p.source}${p.activeModel ? ',model=' + p.activeModel : ''})`);
|
|
290
|
+
return { accepted: true };
|
|
291
|
+
}
|
|
292
|
+
case 'dsh-memory/embedding-download-start': {
|
|
293
|
+
if (!embedManager)
|
|
294
|
+
throw new Error('嵌入管理器未初始化(存储不可用)');
|
|
295
|
+
const p = (payload ?? {});
|
|
296
|
+
if (typeof p.modelId !== 'string' || !p.modelId)
|
|
297
|
+
throw new Error('modelId 缺失');
|
|
298
|
+
const r = embedManager.startDownload(p.modelId);
|
|
299
|
+
if (!r.ok)
|
|
300
|
+
throw new Error(r.error ?? '下载请求被拒绝');
|
|
301
|
+
deps.logger.info(`[memory] 收到模型下载指令(${p.modelId})`);
|
|
302
|
+
return { accepted: true };
|
|
303
|
+
}
|
|
304
|
+
case 'dsh-memory/embedding-download-cancel': {
|
|
305
|
+
if (!embedManager)
|
|
306
|
+
throw new Error('嵌入管理器未初始化');
|
|
307
|
+
return { cancelled: embedManager.cancelDownload() };
|
|
308
|
+
}
|
|
309
|
+
case 'dsh-memory/embedding-model-delete': {
|
|
310
|
+
if (!embedManager)
|
|
311
|
+
throw new Error('嵌入管理器未初始化');
|
|
312
|
+
const p = (payload ?? {});
|
|
313
|
+
if (typeof p.modelId !== 'string' || !p.modelId)
|
|
314
|
+
throw new Error('modelId 缺失');
|
|
315
|
+
return embedManager.deleteModel(p.modelId);
|
|
316
|
+
}
|
|
317
|
+
case 'dsh-memory/embedding-runtime-cancel': {
|
|
318
|
+
if (!embedManager)
|
|
319
|
+
throw new Error('嵌入管理器未初始化');
|
|
320
|
+
return { cancelled: embedManager.cancelRuntimeInstall() };
|
|
321
|
+
}
|
|
322
|
+
case 'dsh-memory/embedding-reindex-cancel': {
|
|
323
|
+
if (!embedManager)
|
|
324
|
+
throw new Error('嵌入管理器未初始化');
|
|
325
|
+
return { cancelled: embedManager.cancelReindex() };
|
|
326
|
+
}
|
|
243
327
|
default:
|
|
244
328
|
throw new Error(`unknown endpoint: ${endpoint}`);
|
|
245
329
|
}
|
|
@@ -261,14 +345,47 @@ function hitToUiRecord(r) {
|
|
|
261
345
|
score: r.score ?? null,
|
|
262
346
|
};
|
|
263
347
|
}
|
|
264
|
-
/**
|
|
348
|
+
/**
|
|
349
|
+
* 从文件尾反向分块读取最后 N 行:不整读全文件(轮转上限 2MB,整读会
|
|
350
|
+
* 阻塞事件循环数毫秒)。原始 Buffer 拼接后再解码——分块边界可能切在
|
|
351
|
+
* UTF-8 多字节字符中间,先 toString 再拼接会产生乱码替换符。
|
|
352
|
+
*/
|
|
265
353
|
function readLogTail(logPath, maxLines) {
|
|
354
|
+
let fd;
|
|
266
355
|
try {
|
|
267
|
-
|
|
268
|
-
const
|
|
356
|
+
fd = openSync(logPath, 'r');
|
|
357
|
+
const { size } = statSync(logPath);
|
|
358
|
+
const CHUNK = 64 * 1024;
|
|
359
|
+
const bufs = [];
|
|
360
|
+
let newlines = 0;
|
|
361
|
+
let pos = size;
|
|
362
|
+
while (pos > 0) {
|
|
363
|
+
const read = Math.min(CHUNK, pos);
|
|
364
|
+
pos -= read;
|
|
365
|
+
const buf = Buffer.alloc(read);
|
|
366
|
+
readSync(fd, buf, 0, read, pos);
|
|
367
|
+
bufs.unshift(buf);
|
|
368
|
+
// \n 是完整单字节,绝不会出现在 UTF-8 续字节里——按字节计数跨块安全
|
|
369
|
+
for (let i = 0; i < buf.length; i++)
|
|
370
|
+
if (buf[i] === 0x0a)
|
|
371
|
+
newlines++;
|
|
372
|
+
if (newlines > maxLines)
|
|
373
|
+
break;
|
|
374
|
+
}
|
|
375
|
+
const lines = Buffer.concat(bufs).toString('utf8').split('\n').filter((l) => l.length > 0);
|
|
269
376
|
return lines.slice(-maxLines);
|
|
270
377
|
}
|
|
271
378
|
catch {
|
|
272
379
|
return [];
|
|
273
380
|
}
|
|
381
|
+
finally {
|
|
382
|
+
if (fd !== undefined) {
|
|
383
|
+
try {
|
|
384
|
+
closeSync(fd);
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
/* ignore */
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
274
391
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { MemoryLogger } from '../types.js';
|
|
2
|
+
import { type CatalogEntry } from './model-catalog.js';
|
|
3
|
+
export type DownloadPhase = 'downloading' | 'verifying' | 'done' | 'cancelled' | 'error';
|
|
4
|
+
export interface DownloadProgress {
|
|
5
|
+
modelId: string;
|
|
6
|
+
phase: DownloadPhase;
|
|
7
|
+
/** 当前文件序号(1-based)。 */
|
|
8
|
+
fileIndex: number;
|
|
9
|
+
fileCount: number;
|
|
10
|
+
/** 当前文件已收字节(含续传基线)。 */
|
|
11
|
+
fileReceived: number;
|
|
12
|
+
fileTotal: number;
|
|
13
|
+
/** 整模型累计字节(含已完成文件;分母 = 目录总大小)。 */
|
|
14
|
+
overallReceived: number;
|
|
15
|
+
overallTotal: number;
|
|
16
|
+
/** EMA 平滑速度(字节/秒)。 */
|
|
17
|
+
speedBps: number;
|
|
18
|
+
startedAt: number;
|
|
19
|
+
/** phase=error 时的原因。 */
|
|
20
|
+
error?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ModelStatus {
|
|
23
|
+
id: string;
|
|
24
|
+
/** none=未下载;partial=有断点/不完整;downloaded=全部文件就位且尺寸吻合。 */
|
|
25
|
+
state: 'none' | 'partial' | 'downloaded';
|
|
26
|
+
bytesOnDisk: number;
|
|
27
|
+
totalBytes: number;
|
|
28
|
+
}
|
|
29
|
+
type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
|
|
30
|
+
export interface DownloaderOptions {
|
|
31
|
+
/** 下载镜像根(默认 https://hf-mirror.com,可配回 https://huggingface.co)。 */
|
|
32
|
+
mirror: string;
|
|
33
|
+
logger?: MemoryLogger;
|
|
34
|
+
/** 测试注入;默认全局 fetch。 */
|
|
35
|
+
fetchImpl?: FetchLike;
|
|
36
|
+
/** 测试注入磁盘剩余字节;默认 statfs。 */
|
|
37
|
+
freeBytes?: () => Promise<number | null>;
|
|
38
|
+
}
|
|
39
|
+
export declare class ModelDownloadQueue {
|
|
40
|
+
private readonly dataDir;
|
|
41
|
+
private readonly opts;
|
|
42
|
+
private progress;
|
|
43
|
+
private busy;
|
|
44
|
+
private abort;
|
|
45
|
+
constructor(dataDir: string, opts: DownloaderOptions);
|
|
46
|
+
/** 当前进度快照(无任务时 null)。 */
|
|
47
|
+
getProgress(): DownloadProgress | null;
|
|
48
|
+
/** 是否有任务在跑(含校验阶段)。 */
|
|
49
|
+
isBusy(): boolean;
|
|
50
|
+
modelsDir(id: string): string;
|
|
51
|
+
/** 全目录状态扫描(设置页模型卡数据源)。 */
|
|
52
|
+
listStatus(): Promise<ModelStatus[]>;
|
|
53
|
+
/** 单模型是否已完整下载(尺寸口径,不做哈希复验——下载完成时已验过)。 */
|
|
54
|
+
isDownloaded(id: string): Promise<boolean>;
|
|
55
|
+
/** 删除已下载模型(切走后释放磁盘;正在使用/下载中的拒绝)。 */
|
|
56
|
+
deleteModel(id: string): Promise<{
|
|
57
|
+
ok: boolean;
|
|
58
|
+
error?: string;
|
|
59
|
+
}>;
|
|
60
|
+
/** 启动下载(串行队列:忙时直接拒绝)。resolve 在任务终态(done/error/cancelled)。 */
|
|
61
|
+
start(id: string): Promise<DownloadProgress>;
|
|
62
|
+
/** 按给定目录项启动(测试缝:合成目录项驱动状态机,不触网)。 */
|
|
63
|
+
startEntry(entry: CatalogEntry): Promise<DownloadProgress>;
|
|
64
|
+
/** 取消当前任务:中断 fetch,保留 .part 断点。 */
|
|
65
|
+
cancel(): boolean;
|
|
66
|
+
private run;
|
|
67
|
+
/** 下载单文件到最终路径(含续传与校验),返回该文件贡献的字节数。 */
|
|
68
|
+
private downloadFile;
|
|
69
|
+
private freeBytes;
|
|
70
|
+
}
|
|
71
|
+
export {};
|