@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/CHANGELOG.md +92 -0
  2. package/README.en.md +225 -2
  3. package/README.md +180 -2
  4. package/lib/deploy-guard.mjs +463 -275
  5. package/lib/guard-http.mjs +59 -62
  6. package/lib/pitfall-form-map.mjs +94 -84
  7. package/lib/pitfall-score.mjs +262 -185
  8. package/lib/session-file-name.mjs +116 -100
  9. package/lib/zstd-frames.mjs +88 -67
  10. package/memory-db-tools/README.md +29 -1
  11. package/memory-db-tools/audit-analyze.mjs +1 -1
  12. package/memory-db-tools/audit-export.mjs +60 -16
  13. package/memory-db-tools/audit-struct-probe.mjs +29 -8
  14. package/memory-db-tools/lib/known-devices.mjs +137 -0
  15. package/memory-engine/README.md +58 -4
  16. package/memory-engine/bench/bench.mjs +1 -1
  17. package/memory-engine/cli/band-sentinel.mjs +164 -0
  18. package/memory-engine/cli/distill-rollback.mjs +112 -44
  19. package/memory-engine/cli/distill-sessions.mjs +237 -0
  20. package/memory-engine/cli/distill.mjs +196 -95
  21. package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
  22. package/memory-engine/cli/doc-index.mjs +47 -38
  23. package/memory-engine/cli/domain-tags.mjs +246 -162
  24. package/memory-engine/cli/layer-migrate.mjs +176 -0
  25. package/memory-engine/cli/layer-move.mjs +189 -0
  26. package/memory-engine/cli/pitfall-hook.mjs +136 -70
  27. package/memory-engine/cli/recall.mjs +243 -110
  28. package/memory-engine/cli/selfcheck.mjs +93 -44
  29. package/memory-engine/cli/serve-start.mjs +221 -140
  30. package/memory-engine/cli/sweep.mjs +83 -44
  31. package/memory-engine/cli/sync.mjs +93 -57
  32. package/memory-engine/config/audit-devices.json +6 -0
  33. package/memory-engine/config/bands.json +7 -0
  34. package/memory-engine/config/layers.json +7 -0
  35. package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
  36. package/memory-engine/dict/lexicon-config.json +11 -0
  37. package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
  38. package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
  39. package/memory-engine/lib/band-sentinel.mjs +417 -0
  40. package/memory-engine/lib/cli-strict.mjs +128 -0
  41. package/memory-engine/lib/cli-util.mjs +147 -125
  42. package/memory-engine/lib/config.mjs +83 -96
  43. package/memory-engine/lib/dirty-ledger.mjs +127 -94
  44. package/memory-engine/lib/distill-rollback.mjs +173 -133
  45. package/memory-engine/lib/distill.mjs +878 -585
  46. package/memory-engine/lib/doc-index-store.mjs +208 -235
  47. package/memory-engine/lib/doc-index.mjs +351 -350
  48. package/memory-engine/lib/doc-scan.mjs +178 -156
  49. package/memory-engine/lib/doc-summarize.mjs +220 -205
  50. package/memory-engine/lib/domain-tags.mjs +358 -277
  51. package/memory-engine/lib/embedder.mjs +175 -194
  52. package/memory-engine/lib/hashes.mjs +22 -7
  53. package/memory-engine/lib/layer-migrate.mjs +330 -0
  54. package/memory-engine/lib/layer-move.mjs +307 -0
  55. package/memory-engine/lib/layers.mjs +191 -0
  56. package/memory-engine/lib/lexicon-recall.mjs +473 -0
  57. package/memory-engine/lib/memory-source.mjs +78 -80
  58. package/memory-engine/lib/pitfall-source.mjs +115 -114
  59. package/memory-engine/lib/policy-source.mjs +279 -261
  60. package/memory-engine/lib/recall-log.mjs +285 -0
  61. package/memory-engine/lib/recall.mjs +565 -240
  62. package/memory-engine/lib/selfcheck.mjs +142 -106
  63. package/memory-engine/lib/session-distill.mjs +603 -0
  64. package/memory-engine/lib/session-source.mjs +421 -0
  65. package/memory-engine/lib/sweep.mjs +107 -130
  66. package/memory-engine/lib/sync.mjs +110 -140
  67. package/memory-engine/lib/tick-hook.mjs +123 -148
  68. package/memory-engine/lib/vectors-store.mjs +213 -188
  69. package/memory-engine/test/12-layering.test.mjs +539 -0
  70. package/memory-engine/test/13-session-distill.test.mjs +616 -0
  71. package/memory-engine/tools/bridge-smoke.mjs +94 -55
  72. package/memory-engine/tools/fetch-model.mjs +167 -87
  73. package/memory-engine/tools/smoke-recall.mjs +94 -38
  74. package/memory-engine/tools/verify-recall-chain.mjs +413 -0
  75. package/package.json +1 -1
@@ -1,305 +1,320 @@
1
- // 记忆引擎 v2 · 离线层 —— 索引条摘要生成(方案 §3.2 粒度 + §3.5 成本闸)
1
+ // 离线层 —— 索引条摘要生成(两种通路:规则的确定性抽取 / 可选的模型润色)
2
2
  //
3
- // 两条通路,**默认 rule(确定性抽取式,零外部调用)**:
4
- // · `rule` —— 从文档结构直接抽取(标题 + 关键章节 + 各节首句)。零成本、零外发、
5
- // 可复现(同一文档两次生成逐字节相同)⇒ 满足本卡「重建到与首跑一致」与「幂等」两条验收。
6
- // · `llm` —— 便宜模型批量润色(方案 §3.2 原语句「摘要生成:便宜模型批量(一次性成本)」)。
7
- // **输入是压缩视图(骨架 + 各节首段截断),不是全文** ⇒ token 可控;受 §3.5 同一成本闸
8
- // (调用 ≤20 次/日、token ≤150K/日、超限即停 + 日报一行)。
9
- // ⚠️ 家人相关数据不进第三方(方案 §3.5/§3.6 订正注:按 D2 从严读)——见 `assertNoFamilyContent`。
3
+ // 默认走规则通路:直接从文档骨架里抽「标题 + 关键章节 + 各节首句」。
4
+ // 零成本、零外发、同一份文档两次生成逐字节相同 ⇒ 才能拿来做「重建后与首跑一致」的验收。
5
+ // 模型通路是可选增益:喂进去的是**压缩视图(骨架 + 各节首段截断)**,绝不是全文,
6
+ // 因此 token 可控;调用前照样过每日成本闸(次数 / token 双上限,触顶即停不重试)。
7
+ // 家人相关内容一律不送出去(按从严口径读):命中哨兵就退回规则摘要并记一条失败。
10
8
  import fs from 'node:fs';
11
9
  import { textHash } from './hashes.mjs';
12
10
  import { SUMMARY_MAX_CHARS, firstSentence, pickParagraph, shortTitle, stripMd } from './doc-scan.mjs';
13
11
 
14
- /** §3.5 成本闸(与提炼 job 同一上限;先定,实测后调)。 */
12
+ // 每日成本闸(与提炼那条链同一档上限)
15
13
  export const LLM_LIMITS = { callsPerDay: 20, tokensPerDay: 150000 };
16
14
 
17
- /** 压缩视图上限(喂 LLM 的骨架文本;不是全文)。 */
15
+ // 喂模型的压缩视图上限(字符);单节在视图里只留开头这么多
18
16
  export const LLM_VIEW_MAX_CHARS = 4000;
19
- /** 单节在压缩视图里的截断长度。 */
20
17
  export const LLM_SECTION_PREVIEW_CHARS = 220;
21
18
 
22
- /** 规则摘要的片段预算(合计 ≤ SUMMARY_MAX_CHARS)。 */
23
- const BUDGET = {
24
- head: 90,
25
- chapters: 170,
26
- points: 240,
27
- };
19
+ // 规则摘要的字符预算:标题行 90 + 章节行 170,再加 40 余量给拼接处
20
+ const SCATTER_ROOM = 90 + 170 + 40;
28
21
 
29
- // ---------------------------------------------------------------- 成本闸
22
+ // 家族 / 隐私哨兵:压缩视图里出现这些词就停手申报
23
+ export const FAMILY_SENTINELS = ['家人档', '家人投喂', '家庭成员'];
30
24
 
31
- /** 当日成本台账的空态。 */
32
- export function emptyCost(date) {
33
- return { date, calls: 0, tokens: 0 };
34
- }
25
+ // ── 成本台账 ────────────────────────────────────────────────────────────────
35
26
 
36
- /** 跨日重置(口径:按**本机本地日期**分段)。 */
37
- export function rolloverCost(cost, date) {
38
- if (!cost || cost.date !== date) return emptyCost(date);
39
- return { date: cost.date, calls: Number(cost.calls) || 0, tokens: Number(cost.tokens) || 0 };
40
- }
27
+ export const emptyCost = (date) => ({ date, calls: 0, tokens: 0 });
41
28
 
42
- /**
43
- * 成本闸判定(超限即停,不重试——方案 §3.5)。
44
- * @returns {{allowed:boolean, reason:string}}
45
- */
29
+ // 跨日重置:按本机本地日期分段,日期一变当日计数归零
30
+ export const rolloverCost = (cost, date) =>
31
+ !cost || cost.date !== date
32
+ ? emptyCost(date)
33
+ : { date: cost.date, calls: Number(cost.calls) || 0, tokens: Number(cost.tokens) || 0 };
34
+
35
+ // 还能不能再调一次:次数与 token 任一触顶就拒;reason 给调用方直接落日志
46
36
  export function checkCostGate(cost, limits = LLM_LIMITS) {
47
- if (Number(cost?.calls) >= limits.callsPerDay) {
48
- return { allowed: false, reason: `调用次数达上限 ${limits.callsPerDay}/日` };
49
- }
50
- if (Number(cost?.tokens) >= limits.tokensPerDay) {
51
- return { allowed: false, reason: `token 达上限 ${limits.tokensPerDay}/日` };
52
- }
37
+ const usedCalls = Number(cost?.calls);
38
+ const usedTokens = Number(cost?.tokens);
39
+ if (usedCalls >= limits.callsPerDay) return { allowed: false, reason: `调用次数达上限 ${limits.callsPerDay}/日` };
40
+ if (usedTokens >= limits.tokensPerDay) return { allowed: false, reason: `token 达上限 ${limits.tokensPerDay}/日` };
53
41
  return { allowed: true, reason: '' };
54
42
  }
55
43
 
56
- // ---------------------------------------------------------------- 规则摘要(默认)
44
+ // ── 规则摘要(默认通路) ────────────────────────────────────────────────────
45
+
46
+ // 按顺序往一行里塞片段,塞不下就丢尾;第一段就超长则硬截
47
+ function packPieces(pieces, limit) {
48
+ let line = '';
49
+ for (const piece of pieces) {
50
+ if (!piece) continue;
51
+ const grown = line ? line + piece : piece;
52
+ if (grown.length > limit) return line || grown.slice(0, limit);
53
+ line = grown;
54
+ }
55
+ return line;
56
+ }
57
57
 
58
- /** 摘要片段拼装:按预算装,装不下就丢尾(保证 ≤500 字且截断可预期)。 */
59
- function joinWithin(parts, maxChars) {
60
- let out = '';
61
- for (const p of parts) {
62
- if (!p) continue;
63
- const next = out ? `${out}${p}` : p;
64
- if (next.length > maxChars) return out || next.slice(0, maxChars);
65
- out = next;
58
+ // 每节首句(最多 5 条,每条 ≤70 字),前缀小节名
59
+ function collectBullets(sections) {
60
+ const bullets = [];
61
+ for (const s of sections) {
62
+ const sentence = firstSentence(pickParagraph(s.body, 260), 70);
63
+ if (sentence) bullets.push(`${shortTitle(s.title, 20)}:${sentence}`);
66
64
  }
67
- return out;
65
+ return bullets.slice(0, 5);
68
66
  }
69
67
 
70
68
  /**
71
69
  * 抽取式摘要(确定性:同一输入两次调用逐字节相同)。
72
70
  * @param {{rel:string, structure:{title:string,intro:string,sections:Array}}} doc
73
- * @param {{key:string,index:number,total:number,sections:Array, intro:string}} row
71
+ * @param {{key:string,index:number,total:number,sections:Array,intro:string}} row
74
72
  */
75
73
  export function buildRuleSummary(doc, row) {
76
- const title = doc.structure.title || doc.rel;
77
- const head =
78
- row.total > 1
79
- ? `【${title}】(第 ${row.index + 1}/${row.total} 段)`
80
- : `【${title}】`;
81
-
82
- const introLine = row.intro ? firstSentence(pickParagraph(row.intro, 300), 110) : '';
83
- const chapters = row.sections.map((s) => shortTitle(s.title)).filter(Boolean);
84
- const chapterLine = chapters.length ? `关键章节:${chapters.join('、')}` : '';
85
-
86
- // 要点:每节首句(上限 5 条,每条 ≤70 字)
87
- const points = [];
88
- for (const s of row.sections) {
89
- if (points.length >= 5) break;
90
- const p = firstSentence(pickParagraph(s.body, 260), 70);
91
- if (p) points.push(`${shortTitle(s.title, 20)}:${p}`);
92
- }
74
+ const name = doc.structure.title || doc.rel;
75
+ const opening = row.total > 1 ? `【${name}】(第 ${row.index + 1}/${row.total} 段)` : `【${name}】`;
76
+ const lead = row.intro ? firstSentence(pickParagraph(row.intro, 300), 110) : '';
77
+ const chapterNames = row.sections.map((s) => shortTitle(s.title)).filter(Boolean);
78
+
79
+ const scaffold = [
80
+ opening,
81
+ lead ? `说明:${lead}` : '',
82
+ chapterNames.length ? `关键章节:${chapterNames.join('、')}` : '',
83
+ ];
84
+ let brief = packPieces(scaffold, SCATTER_ROOM).trim();
93
85
 
94
- const parts = [head];
95
- if (introLine) parts.push(`说明:${introLine}`);
96
- if (chapterLine) parts.push(chapterLine);
97
- let summary = joinWithin(parts, BUDGET.head + BUDGET.chapters + 40).trim();
98
- // 要点:**逐条装**——预算是硬的,但装不下只丢尾,绝不因为"整段超限"把要点全删
99
- //(实测踩过:6 个长章节名吃掉 142 字后,原先的整段装法直接 return,索引条里一条要点都没有)
100
- const remain = SUMMARY_MAX_CHARS - summary.length - 5;
101
- if (points.length && remain > 30) {
102
- let acc = '';
103
- for (const p of points) {
104
- const piece = `${acc ? ';' : ''}· ${p}`;
105
- if (acc.length + piece.length > remain) break;
106
- acc += piece;
86
+ // 要点逐条装:预算硬,但装不下只丢尾 —— 绝不因为「整段超限」把要点一条不留
87
+ const room = SUMMARY_MAX_CHARS - brief.length - 5;
88
+ if (room > 30) {
89
+ let tail = '';
90
+ for (const bullet of collectBullets(row.sections)) {
91
+ const piece = `${tail ? ';' : ''}· ${bullet}`;
92
+ if (tail.length + piece.length > room) break;
93
+ tail += piece;
107
94
  }
108
- if (acc) summary += ` 要点:${acc}`;
95
+ if (tail) brief += ` 要点:${tail}`;
109
96
  }
110
- if (summary.length > SUMMARY_MAX_CHARS) summary = `${summary.slice(0, SUMMARY_MAX_CHARS - 1)}…`;
111
- return summary.replace(/】(说明|关键章节|要点):/g, '】 $1:');
112
- }
113
97
 
114
- // ---------------------------------------------------------------- LLM 通路(可选)
98
+ if (brief.length > SUMMARY_MAX_CHARS) brief = `${brief.slice(0, SUMMARY_MAX_CHARS - 1)}…`;
99
+ // 标题行紧贴下一行时补个空格,别糊在一起
100
+ return brief.replace(/】(说明|关键章节|要点):/g, '】 $1:');
101
+ }
115
102
 
116
- /** 家族/隐私哨兵:输入里出现这些标记 ⇒ 停手申报(D2 从严读)。 */
117
- export const FAMILY_SENTINELS = ['家人档', '家人投喂', '家庭成员'];
103
+ // ── 压缩视图与哨兵扫描 ──────────────────────────────────────────────────────
118
104
 
119
- /**
120
- * 压缩视图(喂 LLM 的输入):标题 + 引言截断 + 各节标题 + 各节首段截断。
121
- * **绝不送全文**(token 闸的物理前提)。
122
- */
123
- export function buildLlmView(doc, { maxChars = LLM_VIEW_MAX_CHARS, previewChars = LLM_SECTION_PREVIEW_CHARS } = {}) {
105
+ // 压缩视图:标题 + 引言截断 + 各节标题 + 各节首段截断。**绝不送全文**。
106
+ export function buildLlmView(
107
+ doc,
108
+ { maxChars = LLM_VIEW_MAX_CHARS, previewChars = LLM_SECTION_PREVIEW_CHARS } = {},
109
+ ) {
124
110
  const { title, intro, sections } = doc.structure;
125
- const lines = [`文档:${doc.rel}`, `标题:${title}`];
126
- const introS = firstSentence(pickParagraph(intro, 600), 300);
127
- if (introS) lines.push(`引言:${introS}`);
128
- lines.push('章节:');
111
+ const head = [`文档:${doc.rel}`, `标题:${title}`];
112
+ const lead = firstSentence(pickParagraph(intro, 600), 300);
113
+ if (lead) head.push(`引言:${lead}`);
114
+ head.push('章节:');
129
115
  for (const s of sections) {
130
- const prev = firstSentence(pickParagraph(s.body, previewChars * 2), previewChars);
131
- lines.push(`- ${s.title}${prev ? `:${prev}` : ''}`);
116
+ const glimpse = firstSentence(pickParagraph(s.body, previewChars * 2), previewChars);
117
+ head.push(glimpse ? `- ${s.title}:${glimpse}` : `- ${s.title}`);
132
118
  }
133
- const text = lines.join('\n');
134
- return text.length > maxChars ? `${text.slice(0, maxChars)}\n…[截断]` : text;
119
+ const skeleton = head.join('\n');
120
+ return skeleton.length > maxChars ? `${skeleton.slice(0, maxChars)}\n…[截断]` : skeleton;
135
121
  }
136
122
 
137
- /** 压缩视图里的家人哨兵扫描(命中的路径名单;非空 ⇒ 调用方必须停手申报)。 */
123
+ // 哨兵扫描:返回命中的关键词列表;非空 ⇒ 调用方必须放弃外发
138
124
  export function assertNoFamilyContent(doc) {
139
- const hay = `${doc.rel}\n${doc.structure.title}\n${doc.structure.intro}\n${doc.structure.sections
140
- .map((s) => `${s.title}\n${s.body}`)
141
- .join('\n')}`;
142
- return FAMILY_SENTINELS.filter((k) => hay.includes(k));
125
+ const hay = [
126
+ doc.rel,
127
+ doc.structure.title,
128
+ doc.structure.intro,
129
+ ...doc.structure.sections.flatMap((s) => [s.title, s.body]),
130
+ ].join('\n');
131
+ return FAMILY_SENTINELS.filter((word) => hay.includes(word));
143
132
  }
144
133
 
145
- /** 提示词(要求 JSON 输出,逐条 ≤500 字)。 */
134
+ // 提示词:要求逐条 JSON,并给出每一段的序号占位
146
135
  export function buildLlmPrompt(doc, rows) {
147
- const view = buildLlmView(doc);
148
- const ask = rows
149
- .map((r) => ` {"index": ${r.index}, "summary": "…"}`)
150
- .join(',\n');
136
+ const skeleton = buildLlmView(doc);
137
+ const shape = rows.map((r) => ` {"index": ${r.index}, "summary": "…"}`).join(',\n');
151
138
  return [
152
139
  '你在为一个本地文档索引库生成检索摘要。只输出 JSON,不要解释、不要代码块围栏。',
153
140
  `文档路径:${doc.rel}`,
154
141
  `要求:为下面 ${rows.length} 个分段各写一条中文摘要,每条 ≤500 字,必须包含:这段覆盖的关键章节名 + 这些章节讲什么。`,
155
142
  '不要复述整篇文档;不要编造未出现的内容;不要输出 markdown 标题记号。',
156
- `输出格式:{"rows": [\n${ask}\n]}`,
143
+ `输出格式:{"rows": [\n${shape}\n]}`,
157
144
  '',
158
145
  '=== 文档骨架(压缩视图,非全文)===',
159
- view,
146
+ skeleton,
160
147
  ].join('\n');
161
148
  }
162
149
 
163
- /** 从模型回复里稳健地抽出 JSON(容忍 ```json 围栏与前后废话)。 */
150
+ // 从模型回复里稳健地抠 JSON:容忍 ```json 围栏,也容忍前后废话
164
151
  export function parseLlmRows(raw) {
165
- const s = String(raw ?? '');
166
- const fence = /```(?:json)?\s*([\s\S]*?)```/i.exec(s);
167
- const body = fence ? fence[1] : s;
168
- const start = body.indexOf('{');
169
- const end = body.lastIndexOf('}');
170
- if (start < 0 || end <= start) throw new Error('模型输出里找不到 JSON 对象');
171
- const obj = JSON.parse(body.slice(start, end + 1));
172
- const rows = Array.isArray(obj?.rows) ? obj.rows : null;
173
- if (!rows) throw new Error('模型输出缺 rows 数组');
174
- return rows.map((r) => ({ index: Number(r.index), summary: stripMd(String(r.summary ?? '')) }));
152
+ const text = String(raw ?? '');
153
+ const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text);
154
+ const body = fenced ? fenced[1] : text;
155
+ const first = body.indexOf('{');
156
+ const last = body.lastIndexOf('}');
157
+ if (first < 0 || last <= first) throw new Error('模型输出里找不到 JSON 对象');
158
+ const parsed = JSON.parse(body.slice(first, last + 1));
159
+ if (!Array.isArray(parsed?.rows)) throw new Error('模型输出缺 rows 数组');
160
+ return parsed.rows.map((r) => ({ index: Number(r.index), summary: stripMd(String(r.summary ?? '')) }));
175
161
  }
176
162
 
177
- /** 读凭证(`DEEPSEEK_API_KEY` 环境变量 > `~/.dsh/.credentials.yaml` 的简单解析;不打印密钥)。 */
178
- export function resolveApiKey({ env = process.env, credentialsFile } = {}) {
179
- if (env.DEEPSEEK_API_KEY && String(env.DEEPSEEK_API_KEY).trim()) return String(env.DEEPSEEK_API_KEY).trim();
163
+ // 凭证:环境变量优先,其次 `~/.dsh/.credentials.yaml` 里的简单键值(绝不打印密钥)
164
+ export const resolveApiKey = ({ env = process.env, credentialsFile } = {}) => {
165
+ const fromEnv = String(env.DEEPSEEK_API_KEY ?? '').trim();
166
+ if (fromEnv) return fromEnv;
180
167
  if (!credentialsFile) return null;
168
+ let txt;
181
169
  try {
182
- const txt = fs.readFileSync(credentialsFile, 'utf8');
183
- const m = /DEEPSEEK_API_KEY\s*:\s*["']?([^"'\r\n]+)["']?/.exec(txt);
184
- return m ? m[1].trim() : null;
170
+ txt = fs.readFileSync(credentialsFile, 'utf8');
185
171
  } catch {
186
172
  return null;
187
173
  }
188
- }
174
+ const hit = /DEEPSEEK_API_KEY\s*:\s*["']?([^"'\r\n]+)["']?/.exec(txt);
175
+ return hit ? hit[1].trim() : null;
176
+ };
177
+
178
+ // token 估算:模型没回 usage 时的兜底记账(中文按 1 字 ≈ 1 token 保守估)
179
+ export const estimateTokens = (text) => Math.ceil(String(text ?? '').length);
180
+
181
+ // 摘要长度闸(≤SUMMARY_MAX_CHARS;超长按字截断并补省略号)
182
+ export const clampSummary = (s) => {
183
+ const trimmed = String(s ?? '').trim();
184
+ return trimmed.length > SUMMARY_MAX_CHARS ? `${trimmed.slice(0, SUMMARY_MAX_CHARS - 1)}…` : trimmed;
185
+ };
186
+
187
+ // ── 模型调用 ────────────────────────────────────────────────────────────────
189
188
 
190
189
  /**
191
- * 调一次便宜模型(一篇文档一次调用);返回 { ok, rows, tokens, model, error }。
192
- * `deps.fetch` 可注入 ⇒ 单测零网络。
190
+ * 调一次便宜模型(一篇文档一次)。返回 {ok, rows, tokens, model, usage}
191
+ * 或 {ok:false, error, rows, tokens}。`deps.fetch` 可注入 ⇒ 单测零网络。
193
192
  */
194
193
  export async function callLlmOnce({ doc, rows, llm, deps = {} }) {
195
- const doFetch = deps.fetch ?? globalThis.fetch;
196
- const key = llm.apiKey;
197
- if (!key) return { ok: false, error: '缺少 DEEPSEEK_API_KEY(凭证文件或环境变量)', rows: [], tokens: 0 };
194
+ const send = deps.fetch ?? globalThis.fetch;
195
+ if (!llm.apiKey) return { ok: false, error: '缺少 DEEPSEEK_API_KEY(凭证文件或环境变量)', rows: [], tokens: 0 };
196
+
198
197
  const prompt = buildLlmPrompt(doc, rows);
199
- let res;
198
+ const endpoint = `${llm.baseUrl.replace(/\/$/, '')}/chat/completions`;
199
+ const payload = {
200
+ model: llm.model,
201
+ messages: [{ role: 'user', content: prompt }],
202
+ temperature: 0.2,
203
+ stream: false,
204
+ };
205
+
206
+ let reply;
200
207
  try {
201
- res = await doFetch(`${llm.baseUrl.replace(/\/$/, '')}/chat/completions`, {
208
+ reply = await send(endpoint, {
202
209
  method: 'POST',
203
- headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
204
- body: JSON.stringify({
205
- model: llm.model,
206
- messages: [{ role: 'user', content: prompt }],
207
- temperature: 0.2,
208
- stream: false,
209
- }),
210
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${llm.apiKey}` },
211
+ body: JSON.stringify(payload),
210
212
  signal: deps.signal,
211
213
  });
212
214
  } catch (e) {
213
215
  return { ok: false, error: `请求失败:${String((e && e.message) || e)}`, rows: [], tokens: 0 };
214
216
  }
215
- let body;
217
+
218
+ let envelope;
216
219
  try {
217
- body = await res.json();
218
- } catch (e) {
219
- return { ok: false, error: `响应非 JSON(HTTP ${res.status})`, rows: [], tokens: 0 };
220
+ envelope = await reply.json();
221
+ } catch {
222
+ return { ok: false, error: `响应非 JSON(HTTP ${reply.status})`, rows: [], tokens: 0 };
220
223
  }
221
- if (!res.ok) {
222
- const msg = body?.error?.message || body?.message || `HTTP ${res.status}`;
223
- return { ok: false, error: `模型报错:${msg}`, rows: [], tokens: 0 };
224
+
225
+ if (!reply.ok) {
226
+ const detail = envelope?.error?.message || envelope?.message || `HTTP ${reply.status}`;
227
+ return { ok: false, error: `模型报错:${detail}`, rows: [], tokens: 0 };
224
228
  }
225
- const raw = body?.choices?.[0]?.message?.content ?? '';
226
- const usage = body?.usage ?? {};
227
- const tokens = Number(usage.total_tokens ?? 0) || estimateTokens(prompt) + estimateTokens(raw);
229
+
230
+ const content = envelope?.choices?.[0]?.message?.content ?? '';
231
+ const usage = envelope?.usage ?? {};
232
+ const tokens = Number(usage.total_tokens ?? 0) || estimateTokens(prompt) + estimateTokens(content);
228
233
  try {
229
- const parsed = parseLlmRows(raw);
230
- return { ok: true, rows: parsed, tokens, model: body?.model ?? llm.model, usage };
234
+ return { ok: true, rows: parseLlmRows(content), tokens, model: envelope?.model ?? llm.model, usage };
231
235
  } catch (e) {
232
- return { ok: false, error: `解析失败:${String((e && e.message) || e)}`, rows: [], tokens, raw: String(raw).slice(0, 200) };
236
+ return { ok: false, error: `解析失败:${String((e && e.message) || e)}`, rows: [], tokens, raw: String(content).slice(0, 200) };
233
237
  }
234
238
  }
235
239
 
236
- /** token 估算(模型未返回 usage 时的兜底记账;中文按 1 字 ≈ 1 token 保守估)。 */
237
- export function estimateTokens(text) {
238
- return Math.ceil(String(text ?? '').length);
240
+ // ── 编排 ────────────────────────────────────────────────────────────────────
241
+
242
+ // 一串行 → 规则摘要写进 byKey(规则通路 / 失败降级 / 家人命中三处共用的收口)
243
+ function emitRuleSummaries(doc, planRows, byKey) {
244
+ for (const row of planRows) {
245
+ const summary = buildRuleSummary(doc, row);
246
+ byKey.set(row.key, { summary, textHash: textHash(summary) });
247
+ }
248
+ }
249
+
250
+ // 模型答了:按 index 对号入座;对不上的那条退回规则摘要
251
+ function adoptModelRows(doc, planRows, modelRows, byKey) {
252
+ for (const row of planRows) {
253
+ const picked = modelRows.find((m) => m.index === row.index);
254
+ const summary = clampSummary(picked?.summary || buildRuleSummary(doc, row));
255
+ byKey.set(row.key, { summary, textHash: textHash(summary) });
256
+ }
239
257
  }
240
258
 
241
259
  /**
242
- * 为一批文档计划生成摘要(逐篇调用;受成本闸约束)。
260
+ * 为一批文档计划生成摘要(逐篇调用,受成本闸约束)。
243
261
  * @returns {Promise<{byKey:Map<string,{summary:string,textHash:string}>, cost:object,
244
262
  * calls:number, tokens:number, stopped:string|null, failures:Array, skippedForFamily:string[]}>}
245
263
  */
246
264
  export async function summarizeAll({ docs, plans, mode = 'rule', cost, llm = null, deps = {}, date = '' }) {
247
- const byKey = new Map();
248
- const failures = [];
249
- const skippedForFamily = [];
250
- let calls = 0;
251
- let tokens = 0;
252
- let stopped = null;
253
- let working = rolloverCost(cost, date);
265
+ const ledger = {
266
+ byKey: new Map(),
267
+ cost: rolloverCost(cost, date),
268
+ calls: 0,
269
+ tokens: 0,
270
+ stopped: null,
271
+ failures: [],
272
+ skippedForFamily: [],
273
+ };
274
+ const useLlm = mode === 'llm';
254
275
 
255
276
  for (const doc of docs) {
256
- const rows = plans.get(doc.rel) ?? [];
257
- if (mode !== 'llm') {
258
- for (const r of rows) {
259
- const summary = buildRuleSummary(doc, r);
260
- byKey.set(r.key, { summary, textHash: textHash(summary) });
261
- }
277
+ const planRows = plans.get(doc.rel) ?? [];
278
+
279
+ // 规则通路(默认):逐条抽,零外部调用
280
+ if (!useLlm) {
281
+ emitRuleSummaries(doc, planRows, ledger.byKey);
262
282
  continue;
263
283
  }
264
- // LLM 通路:先过家人哨兵(D2 从严读),再过成本闸
265
- const family = assertNoFamilyContent(doc);
266
- if (family.length) {
267
- skippedForFamily.push(doc.rel);
268
- failures.push({ path: doc.rel, error: `家人相关标记命中(${family.join('/')})⇒ 按 D2 从严读不送第三方,改用规则摘要` });
269
- for (const r of rows) {
270
- const summary = buildRuleSummary(doc, r);
271
- byKey.set(r.key, { summary, textHash: textHash(summary) });
272
- }
284
+
285
+ // 模型通路:先过家人哨兵(收紧口径),再过成本闸
286
+ const familyHits = assertNoFamilyContent(doc);
287
+ if (familyHits.length) {
288
+ ledger.skippedForFamily.push(doc.rel);
289
+ ledger.failures.push({
290
+ path: doc.rel,
291
+ error: `命中家人相关标记(${familyHits.join('/')})⇒ 不送第三方,本轮回退规则摘要`,
292
+ });
293
+ emitRuleSummaries(doc, planRows, ledger.byKey);
273
294
  continue;
274
295
  }
275
- const gate = checkCostGate(working, LLM_LIMITS);
296
+
297
+ const gate = checkCostGate(ledger.cost, LLM_LIMITS);
276
298
  if (!gate.allowed) {
277
- stopped = gate.reason;
278
- break; // 超限即停(不重试)
299
+ ledger.stopped = gate.reason;
300
+ break; // 触顶即停,剩余文档本轮不做了
279
301
  }
280
- const res = await callLlmOnce({ doc, rows, llm, deps });
281
- calls += 1;
282
- tokens += Number(res.tokens) || 0;
283
- working = { date, calls: working.calls + 1, tokens: working.tokens + (Number(res.tokens) || 0) };
284
- if (!res.ok) {
285
- failures.push({ path: doc.rel, error: res.error });
286
- for (const r of rows) {
287
- const summary = buildRuleSummary(doc, r); // 失败降级到规则摘要(不报错给主人)
288
- byKey.set(r.key, { summary, textHash: textHash(summary) });
289
- }
302
+
303
+ const reply = await callLlmOnce({ doc, rows: planRows, llm, deps });
304
+ const spent = Number(reply.tokens) || 0;
305
+ ledger.calls += 1;
306
+ ledger.tokens += spent;
307
+ ledger.cost = { date, calls: ledger.cost.calls + 1, tokens: ledger.cost.tokens + spent };
308
+
309
+ if (!reply.ok) {
310
+ // 模型挂了也不报错给主人:降级成规则摘要,失败照实记一条
311
+ ledger.failures.push({ path: doc.rel, error: reply.error });
312
+ emitRuleSummaries(doc, planRows, ledger.byKey);
290
313
  continue;
291
314
  }
292
- for (const r of rows) {
293
- const hit = res.rows.find((x) => x.index === r.index);
294
- const summary = clampSummary(hit?.summary || buildRuleSummary(doc, r));
295
- byKey.set(r.key, { summary, textHash: textHash(summary) });
296
- }
315
+
316
+ adoptModelRows(doc, planRows, reply.rows, ledger.byKey);
297
317
  }
298
- return { byKey, cost: working, calls, tokens, stopped, failures, skippedForFamily };
299
- }
300
318
 
301
- /** 摘要长度闸(≤500 字;超长按字截断)。 */
302
- export function clampSummary(s) {
303
- const t = String(s ?? '').trim();
304
- return t.length > SUMMARY_MAX_CHARS ? `${t.slice(0, SUMMARY_MAX_CHARS - 1)}…` : t;
319
+ return ledger;
305
320
  }