@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -1,305 +1,320 @@
|
|
|
1
|
-
//
|
|
1
|
+
// 离线层 —— 索引条摘要生成(两种通路:规则的确定性抽取 / 可选的模型润色)
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
// (调用 ≤20 次/日、token ≤150K/日、超限即停 + 日报一行)。
|
|
9
|
-
// ⚠️ 家人相关数据不进第三方(方案 §3.5/§3.6 订正注:按 D2 从严读)——见 `assertNoFamilyContent`。
|
|
3
|
+
// 默认走规则通路:直接从文档骨架里抽「标题 + 关键章节 + 各节首句」。
|
|
4
|
+
// 零成本、零外发、同一份文档两次生成逐字节相同 ⇒ 才能拿来做「重建后与首跑一致」的验收。
|
|
5
|
+
// 模型通路是可选增益:喂进去的是**压缩视图(骨架 + 各节首段截断)**,绝不是全文,
|
|
6
|
+
// 因此 token 可控;调用前照样过每日成本闸(次数 / token 双上限,触顶即停不重试)。
|
|
7
|
+
// 家人相关内容一律不送出去(按从严口径读):命中哨兵就退回规则摘要并记一条失败。
|
|
10
8
|
import fs from 'node:fs';
|
|
11
9
|
import { textHash } from './hashes.mjs';
|
|
12
10
|
import { SUMMARY_MAX_CHARS, firstSentence, pickParagraph, shortTitle, stripMd } from './doc-scan.mjs';
|
|
13
11
|
|
|
14
|
-
|
|
12
|
+
// 每日成本闸(与提炼那条链同一档上限)
|
|
15
13
|
export const LLM_LIMITS = { callsPerDay: 20, tokensPerDay: 150000 };
|
|
16
14
|
|
|
17
|
-
|
|
15
|
+
// 喂模型的压缩视图上限(字符);单节在视图里只留开头这么多
|
|
18
16
|
export const LLM_VIEW_MAX_CHARS = 4000;
|
|
19
|
-
/** 单节在压缩视图里的截断长度。 */
|
|
20
17
|
export const LLM_SECTION_PREVIEW_CHARS = 220;
|
|
21
18
|
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
head: 90,
|
|
25
|
-
chapters: 170,
|
|
26
|
-
points: 240,
|
|
27
|
-
};
|
|
19
|
+
// 规则摘要的字符预算:标题行 90 + 章节行 170,再加 40 余量给拼接处
|
|
20
|
+
const SCATTER_ROOM = 90 + 170 + 40;
|
|
28
21
|
|
|
29
|
-
//
|
|
22
|
+
// 家族 / 隐私哨兵:压缩视图里出现这些词就停手申报
|
|
23
|
+
export const FAMILY_SENTINELS = ['家人档', '家人投喂', '家庭成员'];
|
|
30
24
|
|
|
31
|
-
|
|
32
|
-
export function emptyCost(date) {
|
|
33
|
-
return { date, calls: 0, tokens: 0 };
|
|
34
|
-
}
|
|
25
|
+
// ── 成本台账 ────────────────────────────────────────────────────────────────
|
|
35
26
|
|
|
36
|
-
|
|
37
|
-
export function rolloverCost(cost, date) {
|
|
38
|
-
if (!cost || cost.date !== date) return emptyCost(date);
|
|
39
|
-
return { date: cost.date, calls: Number(cost.calls) || 0, tokens: Number(cost.tokens) || 0 };
|
|
40
|
-
}
|
|
27
|
+
export const emptyCost = (date) => ({ date, calls: 0, tokens: 0 });
|
|
41
28
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
29
|
+
// 跨日重置:按本机本地日期分段,日期一变当日计数归零
|
|
30
|
+
export const rolloverCost = (cost, date) =>
|
|
31
|
+
!cost || cost.date !== date
|
|
32
|
+
? emptyCost(date)
|
|
33
|
+
: { date: cost.date, calls: Number(cost.calls) || 0, tokens: Number(cost.tokens) || 0 };
|
|
34
|
+
|
|
35
|
+
// 还能不能再调一次:次数与 token 任一触顶就拒;reason 给调用方直接落日志
|
|
46
36
|
export function checkCostGate(cost, limits = LLM_LIMITS) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
}
|
|
50
|
-
if (
|
|
51
|
-
return { allowed: false, reason: `token 达上限 ${limits.tokensPerDay}/日` };
|
|
52
|
-
}
|
|
37
|
+
const usedCalls = Number(cost?.calls);
|
|
38
|
+
const usedTokens = Number(cost?.tokens);
|
|
39
|
+
if (usedCalls >= limits.callsPerDay) return { allowed: false, reason: `调用次数达上限 ${limits.callsPerDay}/日` };
|
|
40
|
+
if (usedTokens >= limits.tokensPerDay) return { allowed: false, reason: `token 达上限 ${limits.tokensPerDay}/日` };
|
|
53
41
|
return { allowed: true, reason: '' };
|
|
54
42
|
}
|
|
55
43
|
|
|
56
|
-
//
|
|
44
|
+
// ── 规则摘要(默认通路) ────────────────────────────────────────────────────
|
|
45
|
+
|
|
46
|
+
// 按顺序往一行里塞片段,塞不下就丢尾;第一段就超长则硬截
|
|
47
|
+
function packPieces(pieces, limit) {
|
|
48
|
+
let line = '';
|
|
49
|
+
for (const piece of pieces) {
|
|
50
|
+
if (!piece) continue;
|
|
51
|
+
const grown = line ? line + piece : piece;
|
|
52
|
+
if (grown.length > limit) return line || grown.slice(0, limit);
|
|
53
|
+
line = grown;
|
|
54
|
+
}
|
|
55
|
+
return line;
|
|
56
|
+
}
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
for (const
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
if (next.length > maxChars) return out || next.slice(0, maxChars);
|
|
65
|
-
out = next;
|
|
58
|
+
// 每节首句(最多 5 条,每条 ≤70 字),前缀小节名
|
|
59
|
+
function collectBullets(sections) {
|
|
60
|
+
const bullets = [];
|
|
61
|
+
for (const s of sections) {
|
|
62
|
+
const sentence = firstSentence(pickParagraph(s.body, 260), 70);
|
|
63
|
+
if (sentence) bullets.push(`${shortTitle(s.title, 20)}:${sentence}`);
|
|
66
64
|
}
|
|
67
|
-
return
|
|
65
|
+
return bullets.slice(0, 5);
|
|
68
66
|
}
|
|
69
67
|
|
|
70
68
|
/**
|
|
71
69
|
* 抽取式摘要(确定性:同一输入两次调用逐字节相同)。
|
|
72
70
|
* @param {{rel:string, structure:{title:string,intro:string,sections:Array}}} doc
|
|
73
|
-
* @param {{key:string,index:number,total:number,sections:Array,
|
|
71
|
+
* @param {{key:string,index:number,total:number,sections:Array,intro:string}} row
|
|
74
72
|
*/
|
|
75
73
|
export function buildRuleSummary(doc, row) {
|
|
76
|
-
const
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const points = [];
|
|
88
|
-
for (const s of row.sections) {
|
|
89
|
-
if (points.length >= 5) break;
|
|
90
|
-
const p = firstSentence(pickParagraph(s.body, 260), 70);
|
|
91
|
-
if (p) points.push(`${shortTitle(s.title, 20)}:${p}`);
|
|
92
|
-
}
|
|
74
|
+
const name = doc.structure.title || doc.rel;
|
|
75
|
+
const opening = row.total > 1 ? `【${name}】(第 ${row.index + 1}/${row.total} 段)` : `【${name}】`;
|
|
76
|
+
const lead = row.intro ? firstSentence(pickParagraph(row.intro, 300), 110) : '';
|
|
77
|
+
const chapterNames = row.sections.map((s) => shortTitle(s.title)).filter(Boolean);
|
|
78
|
+
|
|
79
|
+
const scaffold = [
|
|
80
|
+
opening,
|
|
81
|
+
lead ? `说明:${lead}` : '',
|
|
82
|
+
chapterNames.length ? `关键章节:${chapterNames.join('、')}` : '',
|
|
83
|
+
];
|
|
84
|
+
let brief = packPieces(scaffold, SCATTER_ROOM).trim();
|
|
93
85
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
if (
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
let acc = '';
|
|
103
|
-
for (const p of points) {
|
|
104
|
-
const piece = `${acc ? ';' : ''}· ${p}`;
|
|
105
|
-
if (acc.length + piece.length > remain) break;
|
|
106
|
-
acc += piece;
|
|
86
|
+
// 要点逐条装:预算硬,但装不下只丢尾 —— 绝不因为「整段超限」把要点一条不留
|
|
87
|
+
const room = SUMMARY_MAX_CHARS - brief.length - 5;
|
|
88
|
+
if (room > 30) {
|
|
89
|
+
let tail = '';
|
|
90
|
+
for (const bullet of collectBullets(row.sections)) {
|
|
91
|
+
const piece = `${tail ? ';' : ''}· ${bullet}`;
|
|
92
|
+
if (tail.length + piece.length > room) break;
|
|
93
|
+
tail += piece;
|
|
107
94
|
}
|
|
108
|
-
if (
|
|
95
|
+
if (tail) brief += ` 要点:${tail}`;
|
|
109
96
|
}
|
|
110
|
-
if (summary.length > SUMMARY_MAX_CHARS) summary = `${summary.slice(0, SUMMARY_MAX_CHARS - 1)}…`;
|
|
111
|
-
return summary.replace(/】(说明|关键章节|要点):/g, '】 $1:');
|
|
112
|
-
}
|
|
113
97
|
|
|
114
|
-
|
|
98
|
+
if (brief.length > SUMMARY_MAX_CHARS) brief = `${brief.slice(0, SUMMARY_MAX_CHARS - 1)}…`;
|
|
99
|
+
// 标题行紧贴下一行时补个空格,别糊在一起
|
|
100
|
+
return brief.replace(/】(说明|关键章节|要点):/g, '】 $1:');
|
|
101
|
+
}
|
|
115
102
|
|
|
116
|
-
|
|
117
|
-
export const FAMILY_SENTINELS = ['家人档', '家人投喂', '家庭成员'];
|
|
103
|
+
// ── 压缩视图与哨兵扫描 ──────────────────────────────────────────────────────
|
|
118
104
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
105
|
+
// 压缩视图:标题 + 引言截断 + 各节标题 + 各节首段截断。**绝不送全文**。
|
|
106
|
+
export function buildLlmView(
|
|
107
|
+
doc,
|
|
108
|
+
{ maxChars = LLM_VIEW_MAX_CHARS, previewChars = LLM_SECTION_PREVIEW_CHARS } = {},
|
|
109
|
+
) {
|
|
124
110
|
const { title, intro, sections } = doc.structure;
|
|
125
|
-
const
|
|
126
|
-
const
|
|
127
|
-
if (
|
|
128
|
-
|
|
111
|
+
const head = [`文档:${doc.rel}`, `标题:${title}`];
|
|
112
|
+
const lead = firstSentence(pickParagraph(intro, 600), 300);
|
|
113
|
+
if (lead) head.push(`引言:${lead}`);
|
|
114
|
+
head.push('章节:');
|
|
129
115
|
for (const s of sections) {
|
|
130
|
-
const
|
|
131
|
-
|
|
116
|
+
const glimpse = firstSentence(pickParagraph(s.body, previewChars * 2), previewChars);
|
|
117
|
+
head.push(glimpse ? `- ${s.title}:${glimpse}` : `- ${s.title}`);
|
|
132
118
|
}
|
|
133
|
-
const
|
|
134
|
-
return
|
|
119
|
+
const skeleton = head.join('\n');
|
|
120
|
+
return skeleton.length > maxChars ? `${skeleton.slice(0, maxChars)}\n…[截断]` : skeleton;
|
|
135
121
|
}
|
|
136
122
|
|
|
137
|
-
|
|
123
|
+
// 哨兵扫描:返回命中的关键词列表;非空 ⇒ 调用方必须放弃外发
|
|
138
124
|
export function assertNoFamilyContent(doc) {
|
|
139
|
-
const hay =
|
|
140
|
-
.
|
|
141
|
-
.
|
|
142
|
-
|
|
125
|
+
const hay = [
|
|
126
|
+
doc.rel,
|
|
127
|
+
doc.structure.title,
|
|
128
|
+
doc.structure.intro,
|
|
129
|
+
...doc.structure.sections.flatMap((s) => [s.title, s.body]),
|
|
130
|
+
].join('\n');
|
|
131
|
+
return FAMILY_SENTINELS.filter((word) => hay.includes(word));
|
|
143
132
|
}
|
|
144
133
|
|
|
145
|
-
|
|
134
|
+
// 提示词:要求逐条 JSON,并给出每一段的序号占位
|
|
146
135
|
export function buildLlmPrompt(doc, rows) {
|
|
147
|
-
const
|
|
148
|
-
const
|
|
149
|
-
.map((r) => ` {"index": ${r.index}, "summary": "…"}`)
|
|
150
|
-
.join(',\n');
|
|
136
|
+
const skeleton = buildLlmView(doc);
|
|
137
|
+
const shape = rows.map((r) => ` {"index": ${r.index}, "summary": "…"}`).join(',\n');
|
|
151
138
|
return [
|
|
152
139
|
'你在为一个本地文档索引库生成检索摘要。只输出 JSON,不要解释、不要代码块围栏。',
|
|
153
140
|
`文档路径:${doc.rel}`,
|
|
154
141
|
`要求:为下面 ${rows.length} 个分段各写一条中文摘要,每条 ≤500 字,必须包含:这段覆盖的关键章节名 + 这些章节讲什么。`,
|
|
155
142
|
'不要复述整篇文档;不要编造未出现的内容;不要输出 markdown 标题记号。',
|
|
156
|
-
`输出格式:{"rows": [\n${
|
|
143
|
+
`输出格式:{"rows": [\n${shape}\n]}`,
|
|
157
144
|
'',
|
|
158
145
|
'=== 文档骨架(压缩视图,非全文)===',
|
|
159
|
-
|
|
146
|
+
skeleton,
|
|
160
147
|
].join('\n');
|
|
161
148
|
}
|
|
162
149
|
|
|
163
|
-
|
|
150
|
+
// 从模型回复里稳健地抠 JSON:容忍 ```json 围栏,也容忍前后废话
|
|
164
151
|
export function parseLlmRows(raw) {
|
|
165
|
-
const
|
|
166
|
-
const
|
|
167
|
-
const body =
|
|
168
|
-
const
|
|
169
|
-
const
|
|
170
|
-
if (
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
return rows.map((r) => ({ index: Number(r.index), summary: stripMd(String(r.summary ?? '')) }));
|
|
152
|
+
const text = String(raw ?? '');
|
|
153
|
+
const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text);
|
|
154
|
+
const body = fenced ? fenced[1] : text;
|
|
155
|
+
const first = body.indexOf('{');
|
|
156
|
+
const last = body.lastIndexOf('}');
|
|
157
|
+
if (first < 0 || last <= first) throw new Error('模型输出里找不到 JSON 对象');
|
|
158
|
+
const parsed = JSON.parse(body.slice(first, last + 1));
|
|
159
|
+
if (!Array.isArray(parsed?.rows)) throw new Error('模型输出缺 rows 数组');
|
|
160
|
+
return parsed.rows.map((r) => ({ index: Number(r.index), summary: stripMd(String(r.summary ?? '')) }));
|
|
175
161
|
}
|
|
176
162
|
|
|
177
|
-
|
|
178
|
-
export
|
|
179
|
-
|
|
163
|
+
// 凭证:环境变量优先,其次 `~/.dsh/.credentials.yaml` 里的简单键值(绝不打印密钥)
|
|
164
|
+
export const resolveApiKey = ({ env = process.env, credentialsFile } = {}) => {
|
|
165
|
+
const fromEnv = String(env.DEEPSEEK_API_KEY ?? '').trim();
|
|
166
|
+
if (fromEnv) return fromEnv;
|
|
180
167
|
if (!credentialsFile) return null;
|
|
168
|
+
let txt;
|
|
181
169
|
try {
|
|
182
|
-
|
|
183
|
-
const m = /DEEPSEEK_API_KEY\s*:\s*["']?([^"'\r\n]+)["']?/.exec(txt);
|
|
184
|
-
return m ? m[1].trim() : null;
|
|
170
|
+
txt = fs.readFileSync(credentialsFile, 'utf8');
|
|
185
171
|
} catch {
|
|
186
172
|
return null;
|
|
187
173
|
}
|
|
188
|
-
|
|
174
|
+
const hit = /DEEPSEEK_API_KEY\s*:\s*["']?([^"'\r\n]+)["']?/.exec(txt);
|
|
175
|
+
return hit ? hit[1].trim() : null;
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
// token 估算:模型没回 usage 时的兜底记账(中文按 1 字 ≈ 1 token 保守估)
|
|
179
|
+
export const estimateTokens = (text) => Math.ceil(String(text ?? '').length);
|
|
180
|
+
|
|
181
|
+
// 摘要长度闸(≤SUMMARY_MAX_CHARS;超长按字截断并补省略号)
|
|
182
|
+
export const clampSummary = (s) => {
|
|
183
|
+
const trimmed = String(s ?? '').trim();
|
|
184
|
+
return trimmed.length > SUMMARY_MAX_CHARS ? `${trimmed.slice(0, SUMMARY_MAX_CHARS - 1)}…` : trimmed;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
// ── 模型调用 ────────────────────────────────────────────────────────────────
|
|
189
188
|
|
|
190
189
|
/**
|
|
191
|
-
*
|
|
192
|
-
*
|
|
190
|
+
* 调一次便宜模型(一篇文档一次)。返回 {ok, rows, tokens, model, usage}
|
|
191
|
+
* 或 {ok:false, error, rows, tokens}。`deps.fetch` 可注入 ⇒ 单测零网络。
|
|
193
192
|
*/
|
|
194
193
|
export async function callLlmOnce({ doc, rows, llm, deps = {} }) {
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
194
|
+
const send = deps.fetch ?? globalThis.fetch;
|
|
195
|
+
if (!llm.apiKey) return { ok: false, error: '缺少 DEEPSEEK_API_KEY(凭证文件或环境变量)', rows: [], tokens: 0 };
|
|
196
|
+
|
|
198
197
|
const prompt = buildLlmPrompt(doc, rows);
|
|
199
|
-
|
|
198
|
+
const endpoint = `${llm.baseUrl.replace(/\/$/, '')}/chat/completions`;
|
|
199
|
+
const payload = {
|
|
200
|
+
model: llm.model,
|
|
201
|
+
messages: [{ role: 'user', content: prompt }],
|
|
202
|
+
temperature: 0.2,
|
|
203
|
+
stream: false,
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
let reply;
|
|
200
207
|
try {
|
|
201
|
-
|
|
208
|
+
reply = await send(endpoint, {
|
|
202
209
|
method: 'POST',
|
|
203
|
-
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${
|
|
204
|
-
body: JSON.stringify(
|
|
205
|
-
model: llm.model,
|
|
206
|
-
messages: [{ role: 'user', content: prompt }],
|
|
207
|
-
temperature: 0.2,
|
|
208
|
-
stream: false,
|
|
209
|
-
}),
|
|
210
|
+
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${llm.apiKey}` },
|
|
211
|
+
body: JSON.stringify(payload),
|
|
210
212
|
signal: deps.signal,
|
|
211
213
|
});
|
|
212
214
|
} catch (e) {
|
|
213
215
|
return { ok: false, error: `请求失败:${String((e && e.message) || e)}`, rows: [], tokens: 0 };
|
|
214
216
|
}
|
|
215
|
-
|
|
217
|
+
|
|
218
|
+
let envelope;
|
|
216
219
|
try {
|
|
217
|
-
|
|
218
|
-
} catch
|
|
219
|
-
return { ok: false, error: `响应非 JSON(HTTP ${
|
|
220
|
+
envelope = await reply.json();
|
|
221
|
+
} catch {
|
|
222
|
+
return { ok: false, error: `响应非 JSON(HTTP ${reply.status})`, rows: [], tokens: 0 };
|
|
220
223
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
+
|
|
225
|
+
if (!reply.ok) {
|
|
226
|
+
const detail = envelope?.error?.message || envelope?.message || `HTTP ${reply.status}`;
|
|
227
|
+
return { ok: false, error: `模型报错:${detail}`, rows: [], tokens: 0 };
|
|
224
228
|
}
|
|
225
|
-
|
|
226
|
-
const
|
|
227
|
-
const
|
|
229
|
+
|
|
230
|
+
const content = envelope?.choices?.[0]?.message?.content ?? '';
|
|
231
|
+
const usage = envelope?.usage ?? {};
|
|
232
|
+
const tokens = Number(usage.total_tokens ?? 0) || estimateTokens(prompt) + estimateTokens(content);
|
|
228
233
|
try {
|
|
229
|
-
|
|
230
|
-
return { ok: true, rows: parsed, tokens, model: body?.model ?? llm.model, usage };
|
|
234
|
+
return { ok: true, rows: parseLlmRows(content), tokens, model: envelope?.model ?? llm.model, usage };
|
|
231
235
|
} catch (e) {
|
|
232
|
-
return { ok: false, error: `解析失败:${String((e && e.message) || e)}`, rows: [], tokens, raw: String(
|
|
236
|
+
return { ok: false, error: `解析失败:${String((e && e.message) || e)}`, rows: [], tokens, raw: String(content).slice(0, 200) };
|
|
233
237
|
}
|
|
234
238
|
}
|
|
235
239
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
240
|
+
// ── 编排 ────────────────────────────────────────────────────────────────────
|
|
241
|
+
|
|
242
|
+
// 一串行 → 规则摘要写进 byKey(规则通路 / 失败降级 / 家人命中三处共用的收口)
|
|
243
|
+
function emitRuleSummaries(doc, planRows, byKey) {
|
|
244
|
+
for (const row of planRows) {
|
|
245
|
+
const summary = buildRuleSummary(doc, row);
|
|
246
|
+
byKey.set(row.key, { summary, textHash: textHash(summary) });
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// 模型答了:按 index 对号入座;对不上的那条退回规则摘要
|
|
251
|
+
function adoptModelRows(doc, planRows, modelRows, byKey) {
|
|
252
|
+
for (const row of planRows) {
|
|
253
|
+
const picked = modelRows.find((m) => m.index === row.index);
|
|
254
|
+
const summary = clampSummary(picked?.summary || buildRuleSummary(doc, row));
|
|
255
|
+
byKey.set(row.key, { summary, textHash: textHash(summary) });
|
|
256
|
+
}
|
|
239
257
|
}
|
|
240
258
|
|
|
241
259
|
/**
|
|
242
|
-
*
|
|
260
|
+
* 为一批文档计划生成摘要(逐篇调用,受成本闸约束)。
|
|
243
261
|
* @returns {Promise<{byKey:Map<string,{summary:string,textHash:string}>, cost:object,
|
|
244
262
|
* calls:number, tokens:number, stopped:string|null, failures:Array, skippedForFamily:string[]}>}
|
|
245
263
|
*/
|
|
246
264
|
export async function summarizeAll({ docs, plans, mode = 'rule', cost, llm = null, deps = {}, date = '' }) {
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
265
|
+
const ledger = {
|
|
266
|
+
byKey: new Map(),
|
|
267
|
+
cost: rolloverCost(cost, date),
|
|
268
|
+
calls: 0,
|
|
269
|
+
tokens: 0,
|
|
270
|
+
stopped: null,
|
|
271
|
+
failures: [],
|
|
272
|
+
skippedForFamily: [],
|
|
273
|
+
};
|
|
274
|
+
const useLlm = mode === 'llm';
|
|
254
275
|
|
|
255
276
|
for (const doc of docs) {
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
}
|
|
277
|
+
const planRows = plans.get(doc.rel) ?? [];
|
|
278
|
+
|
|
279
|
+
// 规则通路(默认):逐条抽,零外部调用
|
|
280
|
+
if (!useLlm) {
|
|
281
|
+
emitRuleSummaries(doc, planRows, ledger.byKey);
|
|
262
282
|
continue;
|
|
263
283
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
}
|
|
284
|
+
|
|
285
|
+
// 模型通路:先过家人哨兵(收紧口径),再过成本闸
|
|
286
|
+
const familyHits = assertNoFamilyContent(doc);
|
|
287
|
+
if (familyHits.length) {
|
|
288
|
+
ledger.skippedForFamily.push(doc.rel);
|
|
289
|
+
ledger.failures.push({
|
|
290
|
+
path: doc.rel,
|
|
291
|
+
error: `命中家人相关标记(${familyHits.join('/')})⇒ 不送第三方,本轮回退规则摘要`,
|
|
292
|
+
});
|
|
293
|
+
emitRuleSummaries(doc, planRows, ledger.byKey);
|
|
273
294
|
continue;
|
|
274
295
|
}
|
|
275
|
-
|
|
296
|
+
|
|
297
|
+
const gate = checkCostGate(ledger.cost, LLM_LIMITS);
|
|
276
298
|
if (!gate.allowed) {
|
|
277
|
-
stopped = gate.reason;
|
|
278
|
-
break; //
|
|
299
|
+
ledger.stopped = gate.reason;
|
|
300
|
+
break; // 触顶即停,剩余文档本轮不做了
|
|
279
301
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
}
|
|
302
|
+
|
|
303
|
+
const reply = await callLlmOnce({ doc, rows: planRows, llm, deps });
|
|
304
|
+
const spent = Number(reply.tokens) || 0;
|
|
305
|
+
ledger.calls += 1;
|
|
306
|
+
ledger.tokens += spent;
|
|
307
|
+
ledger.cost = { date, calls: ledger.cost.calls + 1, tokens: ledger.cost.tokens + spent };
|
|
308
|
+
|
|
309
|
+
if (!reply.ok) {
|
|
310
|
+
// 模型挂了也不报错给主人:降级成规则摘要,失败照实记一条
|
|
311
|
+
ledger.failures.push({ path: doc.rel, error: reply.error });
|
|
312
|
+
emitRuleSummaries(doc, planRows, ledger.byKey);
|
|
290
313
|
continue;
|
|
291
314
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
const summary = clampSummary(hit?.summary || buildRuleSummary(doc, r));
|
|
295
|
-
byKey.set(r.key, { summary, textHash: textHash(summary) });
|
|
296
|
-
}
|
|
315
|
+
|
|
316
|
+
adoptModelRows(doc, planRows, reply.rows, ledger.byKey);
|
|
297
317
|
}
|
|
298
|
-
return { byKey, cost: working, calls, tokens, stopped, failures, skippedForFamily };
|
|
299
|
-
}
|
|
300
318
|
|
|
301
|
-
|
|
302
|
-
export function clampSummary(s) {
|
|
303
|
-
const t = String(s ?? '').trim();
|
|
304
|
-
return t.length > SUMMARY_MAX_CHARS ? `${t.slice(0, SUMMARY_MAX_CHARS - 1)}…` : t;
|
|
319
|
+
return ledger;
|
|
305
320
|
}
|