dsh-layered-memory 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.en.md +101 -81
  2. package/README.md +91 -71
  3. package/assets/img/Hero.png +0 -0
  4. package/assets/img/Layers.png +0 -0
  5. package/assets/img/Modes.png +0 -0
  6. package/assets/img/ui-dark.jpg +0 -0
  7. package/assets/img/ui-light.jpg +0 -0
  8. package/assets/readme/flow.svg +189 -0
  9. package/assets/readme/storage.svg +115 -0
  10. package/dist/client.js +1242 -325
  11. package/dist/config.d.ts +12 -0
  12. package/dist/config.js +20 -17
  13. package/dist/hooks/capture.d.ts +17 -1
  14. package/dist/hooks/capture.js +45 -11
  15. package/dist/hooks/recall.d.ts +7 -0
  16. package/dist/hooks/recall.js +20 -4
  17. package/dist/index.d.ts +8 -0
  18. package/dist/index.js +112 -44
  19. package/dist/pipeline/rebuild.d.ts +80 -0
  20. package/dist/pipeline/rebuild.js +307 -0
  21. package/dist/pipeline/runner.d.ts +42 -4
  22. package/dist/pipeline/runner.js +125 -21
  23. package/dist/settings.js +79 -20
  24. package/dist/stats.d.ts +8 -1
  25. package/dist/stats.js +156 -14
  26. package/dist/store/download-queue.d.ts +71 -0
  27. package/dist/store/download-queue.js +313 -0
  28. package/dist/store/embedding-source.d.ts +160 -0
  29. package/dist/store/embedding-source.js +421 -0
  30. package/dist/store/embedding.d.ts +3 -1
  31. package/dist/store/embedding.js +5 -0
  32. package/dist/store/l0.d.ts +13 -5
  33. package/dist/store/l0.js +33 -6
  34. package/dist/store/l1.d.ts +15 -5
  35. package/dist/store/l1.js +40 -15
  36. package/dist/store/local-embedding.d.ts +64 -0
  37. package/dist/store/local-embedding.js +120 -0
  38. package/dist/store/model-catalog.d.ts +45 -0
  39. package/dist/store/model-catalog.js +78 -0
  40. package/dist/store/pending.d.ts +15 -0
  41. package/dist/store/pending.js +56 -0
  42. package/dist/store/runtime-installer.d.ts +60 -0
  43. package/dist/store/runtime-installer.js +181 -0
  44. package/dist/store/sqlite.d.ts +68 -7
  45. package/dist/store/sqlite.js +414 -71
  46. package/dist/store/state.d.ts +6 -0
  47. package/dist/store/state.js +11 -0
  48. package/dist/tools/index.js +6 -4
  49. package/dist/util/filelog.d.ts +2 -0
  50. package/dist/util/filelog.js +20 -3
  51. package/package.json +1 -1
  52. package/assets/readme/hero.svg +0 -58
package/dist/store/l1.js CHANGED
@@ -12,6 +12,7 @@ import { familyForType } from '../types.js';
12
12
  import { EmbedHelper, NoopEmbeddingService } from './embedding.js';
13
13
  import { appendJsonl, dayKey, ensureDir, readJsonl } from './io.js';
14
14
  import { RRF_K, rrfMerge } from './search-utils.js';
15
+ import { isZeroVector } from './sqlite.js';
15
16
  /** 官方过度召回倍数:候选池 = limit × 3(官方 tool 路径同款)。 */
16
17
  const CANDIDATE_MULTIPLIER = 3;
17
18
  export class L1Store {
@@ -41,13 +42,10 @@ export class L1Store {
41
42
  return;
42
43
  try {
43
44
  const records = await readJsonl(this.legacyFile);
45
+ const valid = records.filter((r) => r && typeof r.id === 'string' && r.content);
44
46
  let n = 0;
45
- for (const r of records) {
46
- if (r && typeof r.id === 'string' && r.content) {
47
- if (this.db.upsertL1(r))
48
- n++;
49
- }
50
- }
47
+ if (valid.length > 0 && this.db.upsertL1Batch(valid))
48
+ n = valid.length;
51
49
  // 只有确实导入成功(或文件为空)才改名,避免把未入库的数据改名带走
52
50
  if (n === records.length) {
53
51
  const renamed = await fs
@@ -98,9 +96,8 @@ export class L1Store {
98
96
  await appendJsonl(path.join(this.recordsDir, `${day}.jsonl`), list);
99
97
  }
100
98
  const vecs = await this.helper.batch(records.map((r) => r.content));
101
- for (let i = 0; i < records.length; i++) {
102
- this.db.upsertL1(records[i], vecs[i]);
103
- }
99
+ // 单事务批量写:逐条开事务在 WAL FULL 下每条一次 fsync
100
+ this.db.upsertL1Batch(records, vecs);
104
101
  }
105
102
  /** 去重 update/merge 产出的记录:只更新检索库(JSONL 事实源不改写,官方语义)。 */
106
103
  async upsert(record) {
@@ -109,6 +106,11 @@ export class L1Store {
109
106
  const vec = (await this.helper.batch([record.content]))[0];
110
107
  this.db.upsertL1(record, vec);
111
108
  }
109
+ /** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
110
+ setEmbeddingService(svc) {
111
+ this.embedSvc = svc;
112
+ this.helper.setService(svc);
113
+ }
112
114
  async deleteBatch(ids) {
113
115
  this.db.deleteL1Batch(ids);
114
116
  }
@@ -185,17 +187,29 @@ export class L1Store {
185
187
  return this.db.getL1ByIds(fts.map((h) => h.id));
186
188
  }
187
189
  /**
188
- * 全量重嵌入(embedding 配置变化 / 周期性补齐用)。
189
- * 返回写入数与失败数——failed > 0 时调用方不应标记 meta 同步完成。
190
+ * 增量重嵌入(embedding 配置变化 / 周期性补齐用):只处理缺失向量的记录,
191
+ * 排除已判定"当前 provider 不可嵌入"的 skip 集。返回写入/失败/跳过数——
192
+ * failed > 0 时调用方不应标记 meta 同步完成;skipped(零向量)不算失败、
193
+ * 不阻塞同步标记(否则补齐判据永不收敛,每 30 分钟全量重嵌死循环)。
194
+ * onProgress/shouldCancel 供活切换(D5)的进度展示与取消。
190
195
  */
191
- async reindex() {
196
+ async reindex(opts) {
192
197
  if (!this.helper.vectorReady())
193
- return { written: 0, failed: 0 };
194
- const items = this.db.getL1ForReindex();
198
+ return { written: 0, failed: 0, skipped: 0 };
199
+ const items = this.db.getL1ForReindex(this.db.getVecSkipSet('l1'));
200
+ const total = items.length;
201
+ let done = 0;
195
202
  let written = 0;
196
203
  let failed = 0;
204
+ let skipped = 0;
205
+ let cancelled = false;
206
+ const skippedNow = [];
197
207
  const CHUNK = 16;
198
208
  for (let i = 0; i < items.length; i += CHUNK) {
209
+ if (opts?.shouldCancel?.()) {
210
+ cancelled = true;
211
+ break;
212
+ }
199
213
  const chunk = items.slice(i, i + CHUNK);
200
214
  let vecs;
201
215
  try {
@@ -203,16 +217,27 @@ export class L1Store {
203
217
  }
204
218
  catch {
205
219
  failed += chunk.length;
220
+ done += chunk.length;
221
+ opts?.onProgress?.(done, total);
206
222
  continue;
207
223
  }
208
224
  chunk.forEach((c, j) => {
225
+ if (isZeroVector(vecs[j])) {
226
+ skipped++;
227
+ skippedNow.push(c.id);
228
+ return;
229
+ }
209
230
  if (this.db.updateL1Vec(c.id, vecs[j]))
210
231
  written++;
211
232
  else
212
233
  failed++;
213
234
  });
235
+ done += chunk.length;
236
+ opts?.onProgress?.(done, total);
214
237
  }
215
- return { written, failed };
238
+ if (skippedNow.length > 0)
239
+ this.db.addVecSkippedIds('l1', skippedNow);
240
+ return { written, failed, skipped, cancelled };
216
241
  }
217
242
  postProcess(hits, type, limit) {
218
243
  const filtered = type ? hits.filter((h) => h.type === type) : hits;
@@ -0,0 +1,64 @@
1
+ /**
2
+ * 本地嵌入服务(D1/D6 决策):transformers.js + ONNX,进程内 CPU 推理。
3
+ *
4
+ * - 懒加载(D6):首次嵌入调用才加载模型(下载完成后的预热也走这里),
5
+ * 加载后常驻,close() 释放(嵌入源切走/关闭时调用);
6
+ * - 模型从数据目录 models/<id>/ 本地加载(env.allowRemoteModels=false 杜绝联网);
7
+ * - loader 可注入(测试缝):不依赖真实 transformers.js/模型文件即可测状态机;
8
+ * - 池化方式来自模型目录(BGE 系 CLS / Gemma 系 MEAN),normalize 交给 pipeline
9
+ * 内建 L2 归一(与远程路径的 sanitizeAndNormalize 语义一致)。
10
+ *
11
+ * 注意:真实模型路径的池化正确性(尤其 embeddinggemma 的 mean)属于实现期
12
+ * 待验证项——下载预热后应做一次相似度 sanity 检查(见 PR 验证指引)。
13
+ */
14
+ import type { MemoryLogger } from '../types.js';
15
+ import type { CatalogEntry } from './model-catalog.js';
16
+ import type { EmbeddingProviderInfo, EmbeddingService } from './embedding.js';
17
+ /** transformers.js 的 feature-extraction pipeline 最小面(结构化注入缝)。 */
18
+ export interface ExtractorLike {
19
+ (texts: string[], opts: {
20
+ pooling: 'cls' | 'mean';
21
+ normalize: boolean;
22
+ }): Promise<Array<{
23
+ data: number[] | Float32Array;
24
+ }>>;
25
+ dispose?: () => Promise<void> | void;
26
+ }
27
+ export interface TransformersModuleLike {
28
+ pipeline: (task: string, model: string, opts?: Record<string, unknown>) => Promise<ExtractorLike>;
29
+ env?: {
30
+ allowLocalModels?: boolean;
31
+ allowRemoteModels?: boolean;
32
+ };
33
+ }
34
+ /** 模块加载器(默认走 RuntimeInstaller.resolveModule;测试注入假模块)。 */
35
+ export type ModuleLoader = () => Promise<TransformersModuleLike>;
36
+ type LocalState = 'idle' | 'loading' | 'ready' | 'failed' | 'terminated';
37
+ export declare class LocalEmbeddingService implements EmbeddingService {
38
+ private state;
39
+ private extractor;
40
+ private loadPromise;
41
+ private loadError;
42
+ private readonly modelDir;
43
+ private readonly entry;
44
+ private readonly loader;
45
+ private readonly logger?;
46
+ constructor(entry: CatalogEntry, modelDir: string, loader: ModuleLoader, logger?: MemoryLogger);
47
+ getDimensions(): number;
48
+ getProviderInfo(): EmbeddingProviderInfo;
49
+ isReady(): boolean;
50
+ /** 状态(进度展示用)。 */
51
+ getState(): LocalState;
52
+ getLoadError(): string | null;
53
+ /** 后台预热:下载完成后验证可加载性(幂等;失败态可重试)。 */
54
+ startWarmup(): void;
55
+ /** 等待预热完成(测试用)。 */
56
+ waitForReady(): Promise<void>;
57
+ embed(text: string): Promise<Float32Array>;
58
+ embedBatch(texts: string[]): Promise<Float32Array[]>;
59
+ /** 释放模型(嵌入源切走/关闭时调用;幂等)。terminated 后不可再复用——
60
+ * 防止插件卸载/切走后残留的重嵌循环把模型重新加载常驻(内存泄漏)。 */
61
+ close(): void;
62
+ private ensureLoaded;
63
+ }
64
+ export {};
@@ -0,0 +1,120 @@
1
+ export class LocalEmbeddingService {
2
+ state = 'idle';
3
+ extractor = null;
4
+ loadPromise = null;
5
+ loadError = null;
6
+ modelDir;
7
+ entry;
8
+ loader;
9
+ logger;
10
+ constructor(entry, modelDir, loader, logger) {
11
+ this.entry = entry;
12
+ this.modelDir = modelDir;
13
+ this.loader = loader;
14
+ this.logger = logger;
15
+ }
16
+ getDimensions() {
17
+ return this.entry.dims;
18
+ }
19
+ getProviderInfo() {
20
+ // dimensions 语义与远程路径一致(sqlite.ts 的 providerInfo 比对含维度)
21
+ return { provider: 'local', model: this.entry.id, dimensions: this.entry.dims };
22
+ }
23
+ isReady() {
24
+ return this.state === 'ready' && this.extractor !== null;
25
+ }
26
+ /** 状态(进度展示用)。 */
27
+ getState() {
28
+ return this.state;
29
+ }
30
+ getLoadError() {
31
+ return this.loadError;
32
+ }
33
+ /** 后台预热:下载完成后验证可加载性(幂等;失败态可重试)。 */
34
+ startWarmup() {
35
+ void this.ensureLoaded();
36
+ }
37
+ /** 等待预热完成(测试用)。 */
38
+ async waitForReady() {
39
+ await this.ensureLoaded();
40
+ }
41
+ async embed(text) {
42
+ const [vec] = await this.embedBatch([text]);
43
+ return vec;
44
+ }
45
+ async embedBatch(texts) {
46
+ if (texts.length === 0)
47
+ return [];
48
+ if (this.state === 'terminated') {
49
+ throw new Error('本地嵌入服务已释放(嵌入源已切换);本实例不可复用');
50
+ }
51
+ // 首次调用触发懒加载(D6:加载期间调用方等待而非失败——召回只慢这一次);
52
+ // 只有明确失败过的服务才抛错(EmbedHelper 捕获后降级 FTS)
53
+ if (this.state !== 'ready') {
54
+ if (this.state === 'failed') {
55
+ throw new Error(`本地嵌入模型加载失败: ${this.loadError ?? '未知原因'}(重启插件或重新下载模型可重试)`);
56
+ }
57
+ await this.ensureLoaded();
58
+ }
59
+ const out = [];
60
+ // CPU 推理逐条最稳(ORT session 内部并行),量级为本插件写入/重嵌批(≤16)没问题
61
+ for (const text of texts) {
62
+ const result = await this.extractor([text.slice(0, 5000)], { pooling: this.entry.pooling, normalize: true });
63
+ out.push(new Float32Array(result[0].data));
64
+ }
65
+ return out;
66
+ }
67
+ /** 释放模型(嵌入源切走/关闭时调用;幂等)。terminated 后不可再复用——
68
+ * 防止插件卸载/切走后残留的重嵌循环把模型重新加载常驻(内存泄漏)。 */
69
+ close() {
70
+ const ext = this.extractor;
71
+ this.extractor = null;
72
+ this.loadPromise = null;
73
+ this.state = 'terminated';
74
+ this.loadError = null;
75
+ try {
76
+ const dispose = ext?.dispose;
77
+ if (dispose)
78
+ void Promise.resolve(dispose.call(ext));
79
+ }
80
+ catch {
81
+ /* 释放失败不影响状态复位 */
82
+ }
83
+ }
84
+ async ensureLoaded() {
85
+ if (this.state === 'ready')
86
+ return;
87
+ if (this.state === 'loading' && this.loadPromise)
88
+ return this.loadPromise;
89
+ this.state = 'loading';
90
+ this.loadError = null;
91
+ this.loadPromise = (async () => {
92
+ try {
93
+ const mod = await this.loader();
94
+ if (mod.env) {
95
+ mod.env.allowRemoteModels = false;
96
+ if (mod.env.allowLocalModels !== undefined)
97
+ mod.env.allowLocalModels = true;
98
+ }
99
+ const extractor = await mod.pipeline('feature-extraction', this.modelDir, { dtype: 'q8' });
100
+ // loading 中被 close()(terminated):不得覆写状态复活(review C)——直接释放丢弃
101
+ if (this.state === 'terminated') {
102
+ const dispose = extractor.dispose;
103
+ if (dispose)
104
+ void Promise.resolve(dispose.call(extractor)).catch(() => { });
105
+ throw new Error('加载期间服务已释放');
106
+ }
107
+ this.extractor = extractor;
108
+ this.state = 'ready';
109
+ this.logger?.info(`[memory] 本地嵌入模型就绪: ${this.entry.id}(dims=${this.entry.dims},pooling=${this.entry.pooling})`);
110
+ }
111
+ catch (err) {
112
+ this.state = 'failed';
113
+ this.loadError = err instanceof Error ? err.message : String(err);
114
+ this.logger?.warn(`[memory] 本地嵌入模型加载失败(${this.entry.id}): ${this.loadError}`);
115
+ throw err;
116
+ }
117
+ })();
118
+ return this.loadPromise;
119
+ }
120
+ }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * 本地嵌入模型目录(D2 决策):插件内置白名单,不提供任意 repo 下载。
3
+ * 每款锁定 revision(commit sha)+ 每文件 sha256——目录即完整性契约,
4
+ * 升级模型 = 改这里(维度变化会触发使用方全量重嵌入)。
5
+ *
6
+ * 数据来源(2026-08-17 采集):
7
+ * - revision 与 LFS 大文件 sha256:HF tree API(LFS oid 即 sha256);
8
+ * - 小文件 sha256:从镜像按锁定 revision 下载后本地实测。
9
+ * embeddinggemma 是 ONNX 外部权重格式:model_quantized.onnx(图,~0.5MB)与
10
+ * model_quantized.onnx_data(权重,~294MB)必须成对存在,onnxruntime 同目录自动加载。
11
+ */
12
+ export interface CatalogFile {
13
+ /** 仓库内相对路径(同时是 models/<id>/ 下的落盘路径)。 */
14
+ path: string;
15
+ /** 字节数(磁盘检查的进度分母)。 */
16
+ size: number;
17
+ /** sha256 hex(64 位)。 */
18
+ sha256: string;
19
+ }
20
+ export interface CatalogEntry {
21
+ /** 稳定 id:嵌入源状态文件(embedding-source.json)引用它。 */
22
+ id: string;
23
+ /** 展示名。 */
24
+ name: string;
25
+ /** HF 仓库 repo id。 */
26
+ repo: string;
27
+ /** 锁定的 revision。 */
28
+ revision: string;
29
+ /** 向量维度(vec0 建表与切换重嵌的判据)。 */
30
+ dims: number;
31
+ /** 上下文窗口(token),仅 UI 展示。 */
32
+ contextTokens: number;
33
+ /** 池化方式:BGE 系 CLS;embeddinggemma 系(Gemma 解码器底座)MEAN。 */
34
+ pooling: 'cls' | 'mean';
35
+ /** UI 标签。 */
36
+ tags: string[];
37
+ /** 一句话说明(设置页模型卡)。 */
38
+ description: string;
39
+ files: CatalogFile[];
40
+ }
41
+ export declare const MODEL_CATALOG: CatalogEntry[];
42
+ /** 按 id 取目录项。 */
43
+ export declare function catalogById(id: string): CatalogEntry | undefined;
44
+ /** 模型总字节数(磁盘检查与整体进度分母)。 */
45
+ export declare function catalogTotalBytes(entry: CatalogEntry): number;
@@ -0,0 +1,78 @@
1
+ /**
2
+ * 本地嵌入模型目录(D2 决策):插件内置白名单,不提供任意 repo 下载。
3
+ * 每款锁定 revision(commit sha)+ 每文件 sha256——目录即完整性契约,
4
+ * 升级模型 = 改这里(维度变化会触发使用方全量重嵌入)。
5
+ *
6
+ * 数据来源(2026-08-17 采集):
7
+ * - revision 与 LFS 大文件 sha256:HF tree API(LFS oid 即 sha256);
8
+ * - 小文件 sha256:从镜像按锁定 revision 下载后本地实测。
9
+ * embeddinggemma 是 ONNX 外部权重格式:model_quantized.onnx(图,~0.5MB)与
10
+ * model_quantized.onnx_data(权重,~294MB)必须成对存在,onnxruntime 同目录自动加载。
11
+ */
12
+ export const MODEL_CATALOG = [
13
+ {
14
+ id: 'bge-small-zh-v1.5',
15
+ name: 'BGE small 中文',
16
+ repo: 'Xenova/bge-small-zh-v1.5',
17
+ revision: '75c43b069aac4d136ba6bc1122f995fedcfd2781',
18
+ dims: 512,
19
+ contextTokens: 512,
20
+ pooling: 'cls',
21
+ tags: ['中文', '轻量'],
22
+ description: '中文专用小模型:约 25MB,CPU 嵌入最快,适合先体验语义检索。',
23
+ files: [
24
+ { path: 'config.json', size: 716, sha256: 'd4193ead3a810fd694fa8a31d7fc72fbaebc0668b603e398734bf2f6538ff42f' },
25
+ { path: 'tokenizer_config.json', size: 367, sha256: 'e6f3b96db926a37d4039995fbf5ad17de158dfb8f6343d607e4dbaad18d75f5a' },
26
+ { path: 'special_tokens_map.json', size: 125, sha256: 'b6d346be366a7d1d48332dbc9fdf3bf8960b5d879522b7799ddba59e76237ee3' },
27
+ { path: 'tokenizer.json', size: 439125, sha256: '48cea5d44424912a6fd1ea647bf4fe50b55ab8b1e5879c3275f80e339e8fae26' },
28
+ { path: 'onnx/model_quantized.onnx', size: 24010842, sha256: '15b717c382bcb518ba457b93ea6850ede7f4f1cd8937454aa06972366cd19bcc' },
29
+ ],
30
+ },
31
+ {
32
+ id: 'embeddinggemma-300m',
33
+ name: 'EmbeddingGemma 300M',
34
+ repo: 'onnx-community/embeddinggemma-300m-ONNX',
35
+ revision: '5090578d9565bb06545b4552f76e6bc2c93e4a66',
36
+ dims: 768,
37
+ contextTokens: 2048,
38
+ pooling: 'mean',
39
+ tags: ['多语言', '均衡'],
40
+ description: 'Google 官方嵌入模型:100+ 语言含中文,约 295MB,质量与开销均衡(上游 MemoryCore 同款)。',
41
+ files: [
42
+ { path: 'config.json', size: 1765, sha256: '6e1f06404b7163e0325ed2ea3e6781cde50f4a50b31780a95ad0d30e8404d77b' },
43
+ { path: 'added_tokens.json', size: 35, sha256: '50b2f405ba56a26d4913fd772089992252d7f942123cc0a034d96424221ba946' },
44
+ { path: 'special_tokens_map.json', size: 662, sha256: '2f7b0adf4fb469770bb1490e3e35df87b1dc578246c5e7e6fc76ecf33213a397' },
45
+ { path: 'tokenizer_config.json', size: 1156830, sha256: '3ca953eea6c3c9fcda9cf3df22949ff18b216f7c74bd6459230f3f1013953f3a' },
46
+ { path: 'generation_config.json', size: 133, sha256: '1fb1efd221c1ca88a736b1b36cb47d754c177677e222acb3b1e5424c5d664870' },
47
+ { path: 'tokenizer.json', size: 20323312, sha256: '4dda02faaf32bc91031dc8c88457ac272b00c1016cc679757d1c441b248b9c47' },
48
+ { path: 'onnx/model_quantized.onnx', size: 567874, sha256: '172efde319fe1542dc41f31be6154910b05b78f7a861c265c4600eec906bd6d8' },
49
+ { path: 'onnx/model_quantized.onnx_data', size: 308890624, sha256: '705626e28e4c23c82ade34566b4197d97f534c12275fa406dfb71e9937d388c0' },
50
+ ],
51
+ },
52
+ {
53
+ id: 'bge-m3',
54
+ name: 'BGE-M3 大杯',
55
+ repo: 'Xenova/bge-m3',
56
+ revision: '4de13258303883538bd53b696b452bf8099f0858',
57
+ dims: 1024,
58
+ contextTokens: 8192,
59
+ pooling: 'cls',
60
+ tags: ['中文', '长上下文'],
61
+ description: '中文质量最强:8192 token 上下文,约 543MB,CPU 嵌入较慢(单条可达秒级)。',
62
+ files: [
63
+ { path: 'config.json', size: 770, sha256: '734a79bf12d388c1467a4e3ab625f45de7f6906cffcfb93a1eca1787504bed95' },
64
+ { path: 'special_tokens_map.json', size: 964, sha256: '8c785abebea9ae3257b61681b4e6fd8365ceafde980c21970d001e834cf10835' },
65
+ { path: 'tokenizer_config.json', size: 1173, sha256: '7e4c1cc848840aeccdd763458c18dd525eb0f795c992e00ebe9c28554e7db2d4' },
66
+ { path: 'tokenizer.json', size: 17082821, sha256: '6710678b12670bc442b99edc952c4d996ae309a7020c1fa0096dd245c2faf790' },
67
+ { path: 'onnx/model_quantized.onnx', size: 569694530, sha256: '0826f8c1ab9edf1801db86c61919d4d108e8bfc0b809ec823ad366882ff0b77d' },
68
+ ],
69
+ },
70
+ ];
71
+ /** 按 id 取目录项。 */
72
+ export function catalogById(id) {
73
+ return MODEL_CATALOG.find((m) => m.id === id);
74
+ }
75
+ /** 模型总字节数(磁盘检查与整体进度分母)。 */
76
+ export function catalogTotalBytes(entry) {
77
+ return entry.files.reduce((sum, f) => sum + f.size, 0);
78
+ }
@@ -0,0 +1,15 @@
1
+ import type { ConversationMessage, ExtractMode, MemoryLogger } from '../types.js';
2
+ /** 三档蒸馏缓冲桶(off 在捕获侧已被拦截,永远不到这里)。 */
3
+ export interface PendingBuckets {
4
+ auto: ConversationMessage[];
5
+ chat: ConversationMessage[];
6
+ work: ConversationMessage[];
7
+ }
8
+ export declare function emptyPending(): PendingBuckets;
9
+ /** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。 */
10
+ export declare function loadPending(file: string, logger?: MemoryLogger): Promise<PendingBuckets>;
11
+ /** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
12
+ export declare function savePending(file: string, buckets: PendingBuckets): Promise<void>;
13
+ export declare function pendingPathFor(dataDir: string): string;
14
+ /** 三档 key(调度/遍历用)。 */
15
+ export declare const PENDING_MODES: readonly ExtractMode[];
@@ -0,0 +1,56 @@
1
+ /**
2
+ * 未蒸馏缓冲持久化(pending.json):
3
+ * 按档分桶的"已捕获但尚未成功蒸馏"消息——既包括抽取失败的待重试消息,
4
+ * 也包括攒够触发阈值之前的消息。跨重启不丢(CONTEXT.md「未蒸馏缓冲」)。
5
+ *
6
+ * 与 state.json / session-modes.json 同款原子写;读取宽容(坏行丢弃、坏文件空桶起步)。
7
+ */
8
+ import * as path from 'node:path';
9
+ import { atomicWriteJson, readJsonIfExists } from './io.js';
10
+ export function emptyPending() {
11
+ return { auto: [], chat: [], work: [] };
12
+ }
13
+ function isMessage(m) {
14
+ if (!m || typeof m !== 'object')
15
+ return false;
16
+ const r = m;
17
+ return typeof r.id === 'string' && typeof r.content === 'string' && (r.role === 'user' || r.role === 'assistant');
18
+ }
19
+ /** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。 */
20
+ export async function loadPending(file, logger) {
21
+ const out = emptyPending();
22
+ let raw;
23
+ try {
24
+ raw = await readJsonIfExists(file);
25
+ }
26
+ catch {
27
+ raw = undefined;
28
+ }
29
+ if (!raw || typeof raw !== 'object' || !raw.buckets || typeof raw.buckets !== 'object')
30
+ return out;
31
+ let dropped = 0;
32
+ for (const key of ['auto', 'chat', 'work']) {
33
+ const arr = raw.buckets[key];
34
+ if (!Array.isArray(arr))
35
+ continue;
36
+ for (const m of arr) {
37
+ if (isMessage(m))
38
+ out[key].push(m);
39
+ else
40
+ dropped++;
41
+ }
42
+ }
43
+ if (dropped > 0)
44
+ logger?.warn(`[memory] 未蒸馏缓冲文件含 ${dropped} 条坏记录,已丢弃`);
45
+ return out;
46
+ }
47
+ /** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
48
+ export async function savePending(file, buckets) {
49
+ const payload = { version: 1, buckets };
50
+ await atomicWriteJson(file, payload);
51
+ }
52
+ export function pendingPathFor(dataDir) {
53
+ return path.join(dataDir, 'pending.json');
54
+ }
55
+ /** 三档 key(调度/遍历用)。 */
56
+ export const PENDING_MODES = ['auto', 'chat', 'work'];
@@ -0,0 +1,60 @@
1
+ import type { MemoryLogger } from '../types.js';
2
+ /** 钉死的 transformers.js 版本(D8:精确版本,升级插件时在此变更并重装运行时)。 */
3
+ export declare const PINNED_TRANSFORMERS_VERSION = "4.2.0";
4
+ export interface RuntimeProgress {
5
+ phase: 'idle' | 'installing' | 'ready' | 'error' | 'cancelled';
6
+ /** 插件钉死的目标版本。 */
7
+ targetVersion: string;
8
+ /** runtime 里实际就位的版本(未安装为 null)。 */
9
+ installedVersion: string | null;
10
+ startedAt: number;
11
+ /** 已运行毫秒(读时计算)。 */
12
+ elapsedMs: number;
13
+ /** npm 输出尾行(最近 5 行,让用户看到 npm 在干什么)。 */
14
+ lastLines: string[];
15
+ error?: string;
16
+ }
17
+ /** 子进程抽象(测试注入缝):注册输出行回调 + 终止 + 终态。 */
18
+ export interface SpawnedProcess {
19
+ onStdout(cb: (line: string) => void): void;
20
+ onStderr(cb: (line: string) => void): void;
21
+ kill(): void;
22
+ /** 终态 Promise:退出码(null = 被信号杀死)。 */
23
+ exited: Promise<number | null>;
24
+ }
25
+ export type SpawnImpl = (command: string, args: string[], cwd: string) => SpawnedProcess;
26
+ export declare class RuntimeInstaller {
27
+ readonly runtimeDir: string;
28
+ private readonly target;
29
+ private readonly logger?;
30
+ private readonly spawnImpl;
31
+ private progress;
32
+ private child;
33
+ private current;
34
+ /** 安装超时(npm 卡死不罕见:registry 停滞即永挂,applyBusy 会被锁死)。 */
35
+ private static INSTALL_TIMEOUT_MS;
36
+ constructor(dataDir: string, targetVersion: string, opts?: {
37
+ logger?: MemoryLogger;
38
+ spawnImpl?: SpawnImpl;
39
+ });
40
+ /** 包内模块名(与钉死版本一起构成安装目标)。 */
41
+ static packageName: string;
42
+ /** 进度快照。 */
43
+ getProgress(): RuntimeProgress;
44
+ private pkgJsonPath;
45
+ /** 已就位版本(读 package.json;未安装返回 null)。 */
46
+ installedVersion(): Promise<string | null>;
47
+ /** 是否就绪(版本精确匹配目标)。 */
48
+ isReady(): Promise<boolean>;
49
+ /**
50
+ * 确保运行时就位:版本匹配直接就绪;否则安装(忙时并 await 同一次任务)。
51
+ * 返回是否就绪(失败/取消返回 false 并在 progress.error 说明原因)。
52
+ */
53
+ ensure(): Promise<boolean>;
54
+ /** 取消安装(kill 子进程;node_modules 残留无害,npm 幂等重装)。 */
55
+ cancel(): boolean;
56
+ /** 从 runtime 目录解析已安装的 transformers 模块(LocalEmbeddingService 用)。 */
57
+ resolveModule(): unknown;
58
+ private pushLine;
59
+ private installOnce;
60
+ }