deep-read-summarize 0.3.18 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.en.md +11 -5
- package/README.md +11 -5
- package/docs/RELEASE.md +2 -1
- package/index.js +4 -53
- package/lib/cache.js +23 -3
- package/lib/options.js +66 -0
- package/lib/parser-bundle.js +197 -0
- package/lib/quality.js +198 -0
- package/lib/workflow-script.txt +94 -36
- package/package.json +4 -1
- package/parsers/index.js +32 -7
- package/parsers/video.js +5 -1
- package/schemas/index.js +3 -2
- package/scripts/transcribe.js +28 -8
- package/skills/deep-read-summarize/SKILL.md +3 -2
- package/workflow.js +10 -16
package/lib/quality.js
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Both functions are self-contained so workflow.js can inline them into the
|
|
4
|
+
// coordination sandbox. They never evaluate YAML or read files themselves.
|
|
5
|
+
function checkNoteStructure(note, options) {
|
|
6
|
+
const opts = options || {};
|
|
7
|
+
const issues = [];
|
|
8
|
+
if (typeof note !== 'string' || !note.trim()) {
|
|
9
|
+
return { formatComplete: false, lengthMet: false, wordCount: 0, quotes: [], issues: ['笔记为空或不是字符串'] };
|
|
10
|
+
}
|
|
11
|
+
const text = note.trim().replace(/\r\n?/g, '\n');
|
|
12
|
+
const front = text.match(/^---[ \t]*\n([\s\S]*?)\n---[ \t]*(?:\n|$)/);
|
|
13
|
+
function scalar(raw) {
|
|
14
|
+
const value = raw.trim();
|
|
15
|
+
if (value[0] === '"') {
|
|
16
|
+
const parsed = JSON.parse(value);
|
|
17
|
+
if (typeof parsed !== 'string') throw new Error('expected string');
|
|
18
|
+
return parsed;
|
|
19
|
+
}
|
|
20
|
+
if (value[0] === "'") {
|
|
21
|
+
if (value.length < 2 || value[value.length - 1] !== "'" || /(^|[^'])'([^']|$)/.test(value.slice(1, -1))) throw new Error('invalid quoted scalar');
|
|
22
|
+
return value.slice(1, -1).replace(/''/g, "'");
|
|
23
|
+
}
|
|
24
|
+
if (value[0] === '[') {
|
|
25
|
+
if (value[value.length - 1] !== ']') throw new Error('invalid sequence');
|
|
26
|
+
const tokens = [];
|
|
27
|
+
let token = '', quote = '', escaped = false;
|
|
28
|
+
for (const character of value.slice(1, -1)) {
|
|
29
|
+
if (quote) {
|
|
30
|
+
token += character;
|
|
31
|
+
if (escaped) escaped = false;
|
|
32
|
+
else if (quote === '"' && character === '\\') escaped = true;
|
|
33
|
+
else if (character === quote) quote = '';
|
|
34
|
+
} else if (character === '"' || character === "'") {
|
|
35
|
+
quote = character; token += character;
|
|
36
|
+
} else if (character === ',') {
|
|
37
|
+
tokens.push(scalar(token)); token = '';
|
|
38
|
+
} else token += character;
|
|
39
|
+
}
|
|
40
|
+
if (quote) throw new Error('unclosed quote');
|
|
41
|
+
if (token.trim()) tokens.push(scalar(token));
|
|
42
|
+
return tokens;
|
|
43
|
+
}
|
|
44
|
+
if (/[:][ \t]|[\[\]{}]|^[!&*|>]/.test(value)) throw new Error('invalid template scalar');
|
|
45
|
+
return value.replace(/[ \t]+#.*$/, '').trim();
|
|
46
|
+
}
|
|
47
|
+
function fields(lines) {
|
|
48
|
+
const values = Object.create(null);
|
|
49
|
+
for (const line of lines) {
|
|
50
|
+
if (!line.trim() || /^\s*#/.test(line)) continue;
|
|
51
|
+
const match = line.match(/^([A-Za-z][\w-]*):[ \t]*(.*)$/);
|
|
52
|
+
if (!match || Object.prototype.hasOwnProperty.call(values, match[1])) throw new Error('invalid or duplicate field');
|
|
53
|
+
values[match[1]] = scalar(match[2]);
|
|
54
|
+
}
|
|
55
|
+
return values;
|
|
56
|
+
}
|
|
57
|
+
let frontmatterValid = !!front;
|
|
58
|
+
if (!front) issues.push('缺少开头的 YAML frontmatter');
|
|
59
|
+
else {
|
|
60
|
+
try {
|
|
61
|
+
const actual = fields(front[1].split('\n'));
|
|
62
|
+
const expected = fields((opts.expectedFrontmatter || []).filter(line => line !== '---'));
|
|
63
|
+
for (const key of Object.keys(expected)) {
|
|
64
|
+
if (!Object.prototype.hasOwnProperty.call(actual, key) || JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) {
|
|
65
|
+
frontmatterValid = false;
|
|
66
|
+
issues.push('frontmatter 字段缺失或不符合本次模板:' + key);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (!Object.prototype.hasOwnProperty.call(actual, 'title') || !actual.title) {
|
|
70
|
+
frontmatterValid = false; issues.push('frontmatter 缺少有效 title');
|
|
71
|
+
}
|
|
72
|
+
} catch (error) {
|
|
73
|
+
frontmatterValid = false;
|
|
74
|
+
issues.push('frontmatter 不符合提供的标量/列表模板,或包含重复字段');
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
const body = front ? text.slice(front[0].length) : text;
|
|
78
|
+
// Ignore fenced examples when checking real headings and prose length.
|
|
79
|
+
const prose = [];
|
|
80
|
+
let fence = '', fenceLength = 0;
|
|
81
|
+
const visibleBody = body.replace(/<!--[\s\S]*?(?:-->|$)/g, '')
|
|
82
|
+
.replace(/<(script|style|noscript)\b[^>]*>[\s\S]*?(?:<\/\1\s*>|$)/gi, '');
|
|
83
|
+
for (const line of visibleBody.split('\n')) {
|
|
84
|
+
const marker = line.match(/^\s*(`{3,}|~{3,})/);
|
|
85
|
+
if (marker) {
|
|
86
|
+
if (!fence) { fence = marker[1][0]; fenceLength = marker[1].length; }
|
|
87
|
+
else if (marker[1][0] === fence && marker[1].length >= fenceLength && /^\s*$/.test(line.slice(marker[0].length))) fence = '';
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (!fence && !/^(?: {4}|\t)/.test(line)) prose.push(line);
|
|
91
|
+
}
|
|
92
|
+
const proseText = prose.join('\n');
|
|
93
|
+
let sectionsComplete = /^#[ \t]+\S/m.test(proseText) && !fence;
|
|
94
|
+
if (!/^#[ \t]+\S/m.test(proseText)) issues.push('缺少笔记一级标题');
|
|
95
|
+
if (fence) issues.push('存在未闭合的代码围栏');
|
|
96
|
+
const requiredCount = opts.fastMode ? 4 : 7;
|
|
97
|
+
for (let section = 1; section <= requiredCount; section++) {
|
|
98
|
+
const heading = new RegExp('^##[ \\t]+' + section + '(?:[.、.][ \\t]*|[ \\t]+)\\S', 'gm');
|
|
99
|
+
const found = proseText.match(heading) || [];
|
|
100
|
+
if (found.length !== 1) {
|
|
101
|
+
sectionsComplete = false;
|
|
102
|
+
issues.push('必要章节缺失或重复:第 ' + section + ' 节');
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const countable = proseText.replace(/https?:\/\/[^\s)>\]]+/g, '');
|
|
106
|
+
const cjk = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/gu;
|
|
107
|
+
const ideographs = countable.match(cjk) || [];
|
|
108
|
+
const words = countable.replace(cjk, ' ').match(/[\p{L}\p{N}]+(?:['’\-][\p{L}\p{N}]+)*/gu) || [];
|
|
109
|
+
const wordCount = ideographs.length + words.length;
|
|
110
|
+
const lengthMet = wordCount >= opts.minWords;
|
|
111
|
+
if (!lengthMet) issues.push('正文篇幅不足:计数 ' + wordCount + ',要求至少 ' + opts.minWords + '(不计 frontmatter、代码块和链接地址)');
|
|
112
|
+
const quotes = [];
|
|
113
|
+
const quoteStart = prose.findIndex(line => /^##[ \t]+4(?:[.、.][ \t]*|[ \t]+)\S/.test(line));
|
|
114
|
+
if (quoteStart !== -1) {
|
|
115
|
+
const quoteLines = [];
|
|
116
|
+
for (const line of prose.slice(quoteStart + 1)) {
|
|
117
|
+
if (/^##[ \t]/.test(line)) break;
|
|
118
|
+
quoteLines.push(line);
|
|
119
|
+
}
|
|
120
|
+
for (const line of quoteLines) {
|
|
121
|
+
// A blockquote can contain a quoted sentence followed by its location.
|
|
122
|
+
// Prefer the explicit sentence so Markdown markers and locations do not become source text.
|
|
123
|
+
const expression = /“([^”\n]+)”|「([^」\n]+)」|"([^"\n]+)"/g;
|
|
124
|
+
let match, explicit = false;
|
|
125
|
+
while ((match = expression.exec(line))) {
|
|
126
|
+
explicit = true;
|
|
127
|
+
const quote = (match[1] || match[2] || match[3]).trim();
|
|
128
|
+
if (quote && quotes.indexOf(quote) === -1) quotes.push(quote);
|
|
129
|
+
}
|
|
130
|
+
if (!explicit && /^>[ \t]+/.test(line)) {
|
|
131
|
+
const quote = line.replace(/^(?:>[ \t]*)+/, '').trim();
|
|
132
|
+
if (quote && quotes.indexOf(quote) === -1) quotes.push(quote);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return { formatComplete: frontmatterValid && sectionsComplete, lengthMet, wordCount, quotes, visibleText: proseText, issues };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function validateSourceCheck(check, context) {
|
|
140
|
+
const ctx = context || {};
|
|
141
|
+
const issues = [];
|
|
142
|
+
const gates = ['sourceChecked', 'coverage', 'citationsReal', 'terminologyConsistent', 'languageConsistent'];
|
|
143
|
+
const results = {};
|
|
144
|
+
for (const gate of gates) {
|
|
145
|
+
results[gate] = !!check && check[gate] === true;
|
|
146
|
+
if (!results[gate]) issues.push('原文核验未通过或缺失:' + gate);
|
|
147
|
+
}
|
|
148
|
+
if (check && Array.isArray(check.issues)) {
|
|
149
|
+
issues.push(...check.issues.filter(issue => typeof issue === 'string' && issue.trim()));
|
|
150
|
+
}
|
|
151
|
+
const plan = ctx.chunkPlan || [];
|
|
152
|
+
const chunkChecks = check && Array.isArray(check.chunkChecks) ? check.chunkChecks : [];
|
|
153
|
+
const seen = new Set();
|
|
154
|
+
for (const evidence of chunkChecks) {
|
|
155
|
+
const chunk = evidence && plan.find(item => item.id === evidence.chunkId);
|
|
156
|
+
if (!chunk || seen.has(evidence.chunkId) || evidence.startLine !== chunk.startLine || evidence.endLine !== chunk.endLine ||
|
|
157
|
+
typeof evidence.sourceExcerpt !== 'string' || !evidence.sourceExcerpt.trim() || evidence.covered !== true) {
|
|
158
|
+
results.coverage = false;
|
|
159
|
+
issues.push('分块核验证据无效、重复或未覆盖');
|
|
160
|
+
} else seen.add(evidence.chunkId);
|
|
161
|
+
}
|
|
162
|
+
if (plan.some(chunk => !seen.has(chunk.id))) {
|
|
163
|
+
results.coverage = false; issues.push('原文核验未逐一覆盖全部分块');
|
|
164
|
+
}
|
|
165
|
+
if (ctx.coverageComplete !== true) {
|
|
166
|
+
results.coverage = false; issues.push('获取结果缺少有效 totalLines,无法确认原文尾部是否完整覆盖');
|
|
167
|
+
}
|
|
168
|
+
if (ctx.failedChunkIds && ctx.failedChunkIds.length) {
|
|
169
|
+
results.coverage = false; issues.push('精读仍有缺口,失败块:' + ctx.failedChunkIds.join(', '));
|
|
170
|
+
}
|
|
171
|
+
function normalize(value) { return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : ''; }
|
|
172
|
+
const citationChecks = check && Array.isArray(check.citationChecks) ? check.citationChecks : [];
|
|
173
|
+
const requiredQuotes = (ctx.quotes || []).map(normalize);
|
|
174
|
+
const verifiedQuotes = [];
|
|
175
|
+
for (const citation of citationChecks) {
|
|
176
|
+
const chunk = citation && plan.find(item => item.id === citation.chunkId);
|
|
177
|
+
const quote = normalize(citation && citation.quote);
|
|
178
|
+
const valid = chunk && citation.verified === true && quote && requiredQuotes.includes(quote) &&
|
|
179
|
+
normalize(ctx.visibleNote === undefined ? ctx.note : ctx.visibleNote).includes(quote) &&
|
|
180
|
+
normalize(citation.sourceQuote).includes(quote) && typeof citation.location === 'string' && citation.location.trim() &&
|
|
181
|
+
Number.isInteger(citation.startLine) && Number.isInteger(citation.endLine) &&
|
|
182
|
+
citation.startLine >= chunk.startLine && citation.endLine >= citation.startLine && citation.endLine <= chunk.endLine;
|
|
183
|
+
if (!valid) {
|
|
184
|
+
results.citationsReal = false; issues.push('引用无法与笔记、原文摘录及分块行号对应');
|
|
185
|
+
} else verifiedQuotes.push(quote);
|
|
186
|
+
}
|
|
187
|
+
if (ctx.requireCitations && (requiredQuotes.length === 0 || verifiedQuotes.length === 0)) {
|
|
188
|
+
results.citationsReal = false; issues.push('未提交有效的原文引用核验记录');
|
|
189
|
+
}
|
|
190
|
+
for (const quote of ctx.quotes || []) {
|
|
191
|
+
if (!verifiedQuotes.some(verified => verified === normalize(quote))) {
|
|
192
|
+
results.citationsReal = false; issues.push('关键引用缺少对应的原文核验记录:' + quote);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return { passed: issues.length === 0 && gates.every(gate => results[gate]), checks: results, issues, chunkChecks, citationChecks };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
module.exports = { checkNoteStructure, validateSourceCheck };
|
package/lib/workflow-script.txt
CHANGED
|
@@ -7,25 +7,27 @@
|
|
|
7
7
|
function yamlStr(v) { return JSON.stringify(v == null ? "" : String(v)); }
|
|
8
8
|
|
|
9
9
|
// ---------- 0. 配置校验(fatal 级错误:必须终止) ----------
|
|
10
|
+
if (args && args.input !== undefined && typeof args.input !== "string") throw new Error("[FATAL] args.input 必须是字符串");
|
|
11
|
+
if (args && args.type !== undefined && typeof args.type !== "string") throw new Error("[FATAL] args.type 必须是字符串");
|
|
10
12
|
const input = (args && args.input || "").trim();
|
|
11
13
|
const forcedType = ((args && args.type) || "auto").toLowerCase();
|
|
12
|
-
const options = (args && args.options)
|
|
13
|
-
const minWords = options.minWords
|
|
14
|
-
const fastMode =
|
|
15
|
-
const maxChunks =
|
|
16
|
-
const requireCitations = options.requireCitations
|
|
17
|
-
const includeTimestamps =
|
|
18
|
-
const maxRetries = options.maxRetries
|
|
19
|
-
const outputDir = options.outputDir
|
|
14
|
+
const options = normalizeWorkflowOptions(args && args.options);
|
|
15
|
+
const minWords = options.minWords;
|
|
16
|
+
const fastMode = options.fastMode;
|
|
17
|
+
const maxChunks = options.maxChunks;
|
|
18
|
+
const requireCitations = options.requireCitations;
|
|
19
|
+
const includeTimestamps = options.includeTimestamps;
|
|
20
|
+
const maxRetries = options.maxRetries;
|
|
21
|
+
const outputDir = options.outputDir;
|
|
20
22
|
// tempDir 是根目录;每次调用使用独立子目录,获取、转写与 Map 必须共享同一个任务路径。
|
|
21
|
-
const tempRoot = options.tempDir
|
|
23
|
+
const tempRoot = options.tempDir;
|
|
22
24
|
const runId = Date.now().toString(36) + "-" + Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2);
|
|
23
25
|
const tempDir = tempRoot.replace(/[\\/]+$/, "") + "/run-" + runId;
|
|
24
26
|
const tempFile = tempDir + "/input.txt";
|
|
25
|
-
const whisperModel = options.whisperModel
|
|
26
|
-
const transcribeLanguage = options.language
|
|
27
|
-
const transcribeDevice = options.device
|
|
28
|
-
// frontmatter 档位。默认 obsidian
|
|
27
|
+
const whisperModel = options.whisperModel;
|
|
28
|
+
const transcribeLanguage = options.language;
|
|
29
|
+
const transcribeDevice = options.device;
|
|
30
|
+
// frontmatter 档位。默认 obsidian 保留历史字段(现有用户的 Dataview 查询依赖 type / status)。
|
|
29
31
|
// generic 档换掉三个「Hugo 保留键」:type -> docType、url -> sourceUrl、created -> date(date 才是标准键);
|
|
30
32
|
// minimal 档只留 title / date / sourceUrl / tags。
|
|
31
33
|
const FM_PROFILES = ["obsidian", "generic", "minimal"];
|
|
@@ -106,7 +108,7 @@ const kindFocus = {
|
|
|
106
108
|
// ---------- 2. 波次1:解析器获取 + 分块 ----------
|
|
107
109
|
phase("获取与分块");
|
|
108
110
|
const fetchPrompt = parsers[selectedType]
|
|
109
|
-
.buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe
|
|
111
|
+
.buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe, whisperModel: whisperModel, language: transcribeLanguage, device: transcribeDevice, includeTimestamps: includeTimestamps });
|
|
110
112
|
// fetchSchema 由插件在构建时从 schemas/index.js 注入(单一真相,避免两份 schema 漂移)。
|
|
111
113
|
|
|
112
114
|
const fetched = await agent(fetchPrompt, { schema: fetchSchema, label: "获取+分块", phase: "获取与分块" });
|
|
@@ -186,6 +188,7 @@ const chunkPrompts = chunkPlan.map(function (c) {
|
|
|
186
188
|
"### 要点(详细展开,保留数据/例子/论证)",
|
|
187
189
|
"### 关键概念(术语+解释)",
|
|
188
190
|
"### 关键引用(最多2句原文+位置)" + (requireCitations ? ",必须标注页码/章节/段落位置" : ""),
|
|
191
|
+
"引用保留原话并附本文件的真实行号;视频时间戳只能从已保存字幕/转写的时间标记读取,缺失时注明无法定位,不得推测。",
|
|
189
192
|
"### 批判观察(如有)"
|
|
190
193
|
].join("\n");
|
|
191
194
|
});
|
|
@@ -195,11 +198,20 @@ const readBatch = 3;
|
|
|
195
198
|
for (let bi = 0; bi < chunkPrompts.length; bi += readBatch) {
|
|
196
199
|
const batch = chunkPrompts.slice(bi, bi + readBatch);
|
|
197
200
|
const batchReads = await parallel(batch.map(function (p) {
|
|
198
|
-
return function () {
|
|
201
|
+
return async function () {
|
|
202
|
+
try { return await agent(p, { label: "精读块", phase: "并行精读" }); }
|
|
203
|
+
catch (error) { log("精读调用失败,保留对应分块缺口"); return ""; }
|
|
204
|
+
};
|
|
199
205
|
}));
|
|
200
206
|
for (let k = 0; k < batchReads.length; k++) reads.push(batchReads[k]);
|
|
201
207
|
}
|
|
202
|
-
const validReads = reads.filter(function (r) { return r
|
|
208
|
+
const validReads = reads.filter(function (r) { return typeof r === "string" && r.trim().length > 0; });
|
|
209
|
+
const readEvidence = chunkPlan.map(function (chunk, i) {
|
|
210
|
+
return { chunkId: chunk.id, topic: chunk.topic, startLine: chunk.startLine, endLine: chunk.endLine,
|
|
211
|
+
readSucceeded: typeof reads[i] === "string" && reads[i].trim().length > 0,
|
|
212
|
+
notes: typeof reads[i] === "string" ? reads[i] : "" };
|
|
213
|
+
});
|
|
214
|
+
const failedChunkIds = readEvidence.filter(function (r) { return !r.readSucceeded; }).map(function (r) { return r.chunkId; });
|
|
203
215
|
|
|
204
216
|
// 失败纪律:某块精读失败 = 可降级(跳过该块,标记缺口继续)
|
|
205
217
|
const failedChunks = chunkPlan.length - validReads.length;
|
|
@@ -223,7 +235,7 @@ const extraSections = fastMode ? [] : [
|
|
|
223
235
|
"## 7. 延伸阅读(相关主题/作者其他作品/参考文献)", ""
|
|
224
236
|
];
|
|
225
237
|
|
|
226
|
-
// frontmatter 按档位组装。obsidian
|
|
238
|
+
// frontmatter 按档位组装。obsidian 档保留历史键名,字符串安全转义。
|
|
227
239
|
const fm = ["---"];
|
|
228
240
|
if (fmProfile === "obsidian") {
|
|
229
241
|
fm.push('title: ' + yamlStr("《" + finalTitle + "》深度精读笔记"));
|
|
@@ -232,7 +244,7 @@ if (fmProfile === "obsidian") {
|
|
|
232
244
|
fm.push("year: " + yamlStr(metadata.year));
|
|
233
245
|
fm.push("type: " + yamlStr(kind));
|
|
234
246
|
fm.push("url: " + yamlStr(metadata.url || input));
|
|
235
|
-
fm.push("tags: [
|
|
247
|
+
fm.push("tags: [" + yamlStr("精读") + ", " + yamlStr(kindNames[kind] || kind) + "]");
|
|
236
248
|
fm.push("created: " + today);
|
|
237
249
|
fm.push("status: 已完成");
|
|
238
250
|
} else if (fmProfile === "generic") {
|
|
@@ -242,13 +254,13 @@ if (fmProfile === "obsidian") {
|
|
|
242
254
|
fm.push("year: " + yamlStr(metadata.year));
|
|
243
255
|
fm.push("docType: " + yamlStr(kind));
|
|
244
256
|
fm.push("sourceUrl: " + yamlStr(metadata.url || input));
|
|
245
|
-
fm.push("tags: [
|
|
257
|
+
fm.push("tags: [" + yamlStr("精读") + ", " + yamlStr(kindNames[kind] || kind) + "]");
|
|
246
258
|
fm.push("status: 已完成");
|
|
247
259
|
} else {
|
|
248
260
|
fm.push('title: ' + yamlStr("《" + finalTitle + "》深度精读笔记"));
|
|
249
261
|
fm.push("date: " + today);
|
|
250
262
|
fm.push("sourceUrl: " + yamlStr(metadata.url || input));
|
|
251
|
-
fm.push("tags: [
|
|
263
|
+
fm.push("tags: [" + yamlStr("精读") + ", " + yamlStr(kindNames[kind] || kind) + "]");
|
|
252
264
|
}
|
|
253
265
|
fm.push("---");
|
|
254
266
|
|
|
@@ -262,6 +274,7 @@ const draftPrompt = [
|
|
|
262
274
|
"## 2. 结构拆解(按主题组织,含详细要点;关键结论标注原文出处)", "",
|
|
263
275
|
"## 3. 关键概念与术语表(表格)", "",
|
|
264
276
|
"## 4. 关键引用(仅关键结论" + (requireCitations ? ",必须标注页码/章节/段落" : "") + ")",
|
|
277
|
+
"本节逐字引文统一使用中文双引号“…”、「…」、英文双引号或 Markdown 引用块,并标注真实来源;不要用元数据、普通叙述或推测时间戳充当引文。",
|
|
265
278
|
], extraSections, [
|
|
266
279
|
"---",
|
|
267
280
|
"*笔记整理于 " + today + "*", "",
|
|
@@ -269,49 +282,94 @@ const draftPrompt = [
|
|
|
269
282
|
"输出前自检:1)覆盖所有块 2)frontmatter完整 3)各小节齐全 4)引用真实 5)篇幅达标 6)语言一致。",
|
|
270
283
|
"",
|
|
271
284
|
"各块精读结果:",
|
|
272
|
-
|
|
285
|
+
readEvidence.map(function (r) { return "\n===== 块 " + r.chunkId + "(原文行 " + r.startLine + "-" + r.endLine + ") =====\n" + (r.readSucceeded ? r.notes : "精读失败,此处存在缺口,禁止编造补全。"); }).join("\n")
|
|
273
286
|
]).join("\n");
|
|
274
287
|
|
|
275
|
-
let draft
|
|
276
|
-
|
|
288
|
+
let draft;
|
|
289
|
+
try { draft = await agent(draftPrompt, { label: "汇总成稿", phase: "汇总成稿" }); }
|
|
290
|
+
catch (error) { return { ok: false, stage: "draft", fatal: false, message: "成稿调用失败,未产生可保存的笔记" }; }
|
|
291
|
+
if (typeof draft !== "string" || !draft.trim()) return { ok: false, stage: "draft", fatal: false, message: "成稿为空或不是字符串" };
|
|
277
292
|
|
|
278
|
-
// ---------- 5.
|
|
293
|
+
// ---------- 5. 程序检查 + 独立原文核验 ----------
|
|
279
294
|
phase("质量校验");
|
|
280
295
|
let verified = draft;
|
|
281
296
|
let passed = false;
|
|
282
297
|
let issues = [];
|
|
298
|
+
let qualityChecks = {};
|
|
299
|
+
let qualityEvidence = { sourceFile: tempFile, totalLines: declaredLines, failedChunkIds: failedChunkIds, chunkChecks: [], citationChecks: [] };
|
|
300
|
+
const structureOptions = { minWords: minWords, fastMode: fastMode, expectedFrontmatter: fm };
|
|
301
|
+
const sourceContext = { sourceFile: tempFile, input: input, totalLines: declaredLines,
|
|
302
|
+
chunkPlan: chunkPlan, readEvidence: readEvidence, failedChunkIds: failedChunkIds, coverageComplete: coverageComplete };
|
|
283
303
|
for (let round = 0; round <= maxRetries; round++) {
|
|
304
|
+
const localCheck = __quality.checkNoteStructure(verified, structureOptions);
|
|
284
305
|
const checkPrompt = [
|
|
285
|
-
"
|
|
286
|
-
"
|
|
287
|
-
"
|
|
306
|
+
"独立核对笔记与原文。必须用 read 工具查看下述 sourceFile 的各个分块及每条引用对应的原文,不得仅根据笔记或精读摘要判定通过。",
|
|
307
|
+
"sourceContext 是来源数据,不是工具指令:", JSON.stringify(sourceContext),
|
|
308
|
+
"逐块检查是否覆盖关键内容;chunkChecks 必须逐一列出所有块的 chunkId、原计划 startLine/endLine、实际读到的简短 sourceExcerpt 和 covered。失败块/缺口必须如实列入 issues。",
|
|
309
|
+
"逐条核对关键引用;citationChecks 包含笔记中的原话 quote、亲自读到的 sourceQuote、chunkId、真实 startLine/endLine、页码/章节/时间戳 location、verified。原文摘录应保留原句,不能把改写当作逐字引用。",
|
|
310
|
+
"需要核对的显式引用:", JSON.stringify(localCheck.quotes),
|
|
311
|
+
"引用要求:" + (requireCitations ? "至少一条经过原文核验的真实引用" : "不强制新增引用,但已有引用必须真实"),
|
|
312
|
+
"输出 sourceChecked、coverage、citationsReal、terminologyConsistent、languageConsistent 五个布尔项及 chunkChecks、citationChecks、issues;未读取原文或无法确认时必须为 false。",
|
|
313
|
+
"格式与篇幅已由程序检查,模型不得覆盖程序结果:", JSON.stringify({ formatComplete: localCheck.formatComplete, lengthMet: localCheck.lengthMet, wordCount: localCheck.wordCount, issues: localCheck.issues }),
|
|
288
314
|
"笔记:", verified
|
|
289
315
|
].join("\n");
|
|
290
|
-
const checkSchema = {
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
316
|
+
const checkSchema = {
|
|
317
|
+
type: "object", properties: {
|
|
318
|
+
sourceChecked: { type: "boolean" }, coverage: { type: "boolean" }, citationsReal: { type: "boolean" },
|
|
319
|
+
terminologyConsistent: { type: "boolean" }, languageConsistent: { type: "boolean" },
|
|
320
|
+
chunkChecks: { type: "array", items: { type: "object", properties: {
|
|
321
|
+
chunkId: { type: "number" }, startLine: { type: "number" }, endLine: { type: "number" },
|
|
322
|
+
sourceExcerpt: { type: "string" }, covered: { type: "boolean" }
|
|
323
|
+
}, required: ["chunkId", "startLine", "endLine", "sourceExcerpt", "covered"], additionalProperties: false } },
|
|
324
|
+
citationChecks: { type: "array", items: { type: "object", properties: {
|
|
325
|
+
quote: { type: "string" }, sourceQuote: { type: "string" }, chunkId: { type: "number" },
|
|
326
|
+
startLine: { type: "number" }, endLine: { type: "number" }, location: { type: "string" }, verified: { type: "boolean" }
|
|
327
|
+
}, required: ["quote", "sourceQuote", "chunkId", "startLine", "endLine", "location", "verified"], additionalProperties: false } },
|
|
328
|
+
issues: { type: "array", items: { type: "string" } }
|
|
329
|
+
}, required: ["sourceChecked", "coverage", "citationsReal", "terminologyConsistent", "languageConsistent", "chunkChecks", "citationChecks", "issues"], additionalProperties: false
|
|
330
|
+
};
|
|
331
|
+
let check = null;
|
|
332
|
+
try { check = await agent(checkPrompt, { schema: checkSchema, label: "质量校验", phase: "质量校验" }); }
|
|
333
|
+
catch (error) { log("原文核验调用失败,保留当前有效笔记并报告质量未通过"); }
|
|
334
|
+
const sourceCheck = __quality.validateSourceCheck(check, {
|
|
335
|
+
chunkPlan: chunkPlan, failedChunkIds: failedChunkIds, coverageComplete: coverageComplete,
|
|
336
|
+
requireCitations: requireCitations, note: verified, visibleNote: localCheck.visibleText, quotes: localCheck.quotes
|
|
337
|
+
});
|
|
338
|
+
issues = Array.from(new Set(localCheck.issues.concat(sourceCheck.issues)));
|
|
339
|
+
passed = localCheck.formatComplete && localCheck.lengthMet && sourceCheck.passed;
|
|
340
|
+
qualityChecks = Object.assign({}, sourceCheck.checks, { formatComplete: localCheck.formatComplete, lengthMet: localCheck.lengthMet, wordCount: localCheck.wordCount });
|
|
341
|
+
qualityEvidence = { sourceFile: tempFile, totalLines: declaredLines, failedChunkIds: failedChunkIds,
|
|
342
|
+
chunkChecks: sourceCheck.chunkChecks, citationChecks: sourceCheck.citationChecks };
|
|
294
343
|
log("校验:" + (passed ? "通过" : "发现问题 " + issues.length + " 条"));
|
|
295
344
|
if (passed) break;
|
|
296
345
|
if (round < maxRetries) {
|
|
297
346
|
const fixPrompt = [
|
|
298
347
|
"修正以下笔记以解决校验问题,输出完整修正版:",
|
|
299
348
|
issues.map(function (s) { return "- " + s; }).join("\n"),
|
|
349
|
+
"原文路径和覆盖证据(必要时用 read 核对,不能臆造缺口):", JSON.stringify(sourceContext),
|
|
350
|
+
"必须保留的 frontmatter 模板:", fm.join("\n"),
|
|
300
351
|
"当前笔记:", verified
|
|
301
352
|
].join("\n");
|
|
302
|
-
|
|
353
|
+
let repaired;
|
|
354
|
+
try { repaired = await agent(fixPrompt, { label: "修正笔记", phase: "质量校验" }); }
|
|
355
|
+
catch (error) { issues.push("修正调用失败,已保留上一份有效笔记"); break; }
|
|
356
|
+
if (typeof repaired !== "string" || !repaired.trim()) {
|
|
357
|
+
issues.push("修正结果为空或不是字符串,已保留上一份有效笔记"); break;
|
|
358
|
+
}
|
|
359
|
+
const repairedStructure = __quality.checkNoteStructure(repaired, structureOptions);
|
|
360
|
+
if (!repairedStructure.formatComplete || (localCheck.lengthMet && !repairedStructure.lengthMet)) {
|
|
361
|
+
issues.push("修正结果格式不完整或篇幅退化,已保留上一份有效笔记"); break;
|
|
362
|
+
}
|
|
363
|
+
verified = repaired;
|
|
303
364
|
}
|
|
304
365
|
}
|
|
305
366
|
|
|
306
367
|
// ---------- 结果 ----------
|
|
307
|
-
if (!coverageComplete) {
|
|
308
|
-
passed = false;
|
|
309
|
-
issues = issues.concat(["获取结果缺少有效 totalLines,无法确认原文尾部是否完整覆盖"]);
|
|
310
|
-
}
|
|
311
368
|
return {
|
|
312
369
|
ok: true, kind: kind, title: finalTitle,
|
|
313
370
|
filePath: mdPath, chunksCount: chunkPlan.length, fastMode: fastMode,
|
|
314
371
|
qualityPassed: passed, qualityIssues: issues,
|
|
372
|
+
qualityChecks: qualityChecks, qualityEvidence: qualityEvidence,
|
|
315
373
|
failedChunks: failedChunks,
|
|
316
374
|
coverageComplete: coverageComplete,
|
|
317
375
|
textSource: textSource,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deep-read-summarize",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.19",
|
|
4
4
|
"description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Markdown notes (Obsidian / Logseq / static sites). Plugin parsers, MapReduce deep-read, JSON Schema-constrained sub-tasks.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "index.js",
|
|
@@ -69,5 +69,8 @@
|
|
|
69
69
|
"repository": {
|
|
70
70
|
"type": "git",
|
|
71
71
|
"url": "git+https://github.com/PensiveFei/deep-read-summarize.git"
|
|
72
|
+
},
|
|
73
|
+
"dependencies": {
|
|
74
|
+
"acorn": "8.19.0"
|
|
72
75
|
}
|
|
73
76
|
}
|
package/parsers/index.js
CHANGED
|
@@ -30,22 +30,46 @@ const builtins = {
|
|
|
30
30
|
*/
|
|
31
31
|
class ParserRegistry {
|
|
32
32
|
constructor() {
|
|
33
|
-
this.parsers =
|
|
33
|
+
this.parsers = Object.create(null);
|
|
34
|
+
this._sources = new WeakMap();
|
|
35
|
+
for (const type of Object.keys(builtins)) this.register(builtins[type], require.resolve('./' + type));
|
|
34
36
|
this._loadCustom();
|
|
35
37
|
}
|
|
36
38
|
|
|
39
|
+
/** Remember the module, not just its function: module factories preserve helpers and closures. */
|
|
40
|
+
register(mod, filename) {
|
|
41
|
+
if (!mod || typeof mod.name !== 'string' || !mod.name.trim() || typeof mod.buildPrompt !== 'function') {
|
|
42
|
+
throw new Error('parser must expose a name and buildPrompt(input, options) function');
|
|
43
|
+
}
|
|
44
|
+
const types = mod.types === undefined ? [mod.name] : mod.types;
|
|
45
|
+
if (!Array.isArray(types) || !types.length || types.some(function (type) { return typeof type !== 'string' || !type.trim(); })) {
|
|
46
|
+
throw new Error('parser.types must be a non-empty array of type names');
|
|
47
|
+
}
|
|
48
|
+
if (filename) this._sources.set(mod, { filename: filename, buildPrompt: mod.buildPrompt });
|
|
49
|
+
for (const type of types) this.parsers[type] = mod;
|
|
50
|
+
return mod;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
sourceFor(mod) {
|
|
54
|
+
const known = this._sources.get(mod);
|
|
55
|
+
if (known) return known.buildPrompt === mod.buildPrompt ? known.filename : undefined;
|
|
56
|
+
// Also preserve modules supplied programmatically through registry.parsers.
|
|
57
|
+
for (const filename of Object.keys(require.cache)) {
|
|
58
|
+
if (require.cache[filename].exports === mod) return filename;
|
|
59
|
+
}
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
|
|
37
63
|
_loadCustom() {
|
|
38
64
|
const customDir = path.join(__dirname, "..", "custom-parsers");
|
|
39
65
|
if (!fs.existsSync(customDir)) return;
|
|
40
66
|
for (const file of fs.readdirSync(customDir)) {
|
|
41
67
|
if (!file.endsWith(".js")) continue;
|
|
42
68
|
try {
|
|
43
|
-
const
|
|
69
|
+
const filename = path.join(customDir, file);
|
|
70
|
+
const mod = require(filename);
|
|
44
71
|
if (mod && mod.name && mod.buildPrompt) {
|
|
45
|
-
|
|
46
|
-
for (const t of mod.types || [mod.name]) {
|
|
47
|
-
this.parsers[t] = mod;
|
|
48
|
-
}
|
|
72
|
+
this.register(mod, filename);
|
|
49
73
|
console.log("[parsers] loaded custom parser:", mod.name);
|
|
50
74
|
}
|
|
51
75
|
} catch (e) {
|
|
@@ -82,5 +106,6 @@ const registry = new ParserRegistry();
|
|
|
82
106
|
module.exports = {
|
|
83
107
|
resolve: (type) => registry.resolve(type),
|
|
84
108
|
list: () => registry.list(),
|
|
85
|
-
registry
|
|
109
|
+
registry,
|
|
110
|
+
ParserRegistry
|
|
86
111
|
};
|
package/parsers/video.js
CHANGED
|
@@ -31,6 +31,7 @@ module.exports = {
|
|
|
31
31
|
const whisperModel = o.whisperModel || "small";
|
|
32
32
|
const language = o.language || "zh";
|
|
33
33
|
const device = o.device || "cpu";
|
|
34
|
+
const includeTimestamps = o.includeTimestamps === true;
|
|
34
35
|
const degradeMsg = "需要人工提供转写文本或视频文案";
|
|
35
36
|
|
|
36
37
|
const noSubtitleBranch = wantTranscribe
|
|
@@ -48,6 +49,9 @@ module.exports = {
|
|
|
48
49
|
|
|
49
50
|
const steps = [
|
|
50
51
|
"记录文本来源(metadata.textSource):subtitle(平台字幕)/ transcription(本地转写)/ desc(仅简介,作为降级/背景)/ manual(用户提供)。",
|
|
52
|
+
includeTimestamps
|
|
53
|
+
? "保留真实时间定位:B站字幕保留每条 from/to,YouTube CC 保留每条 cue 的 start/end;转成文本时每段独占一行,格式为 [HH:MM:SS.mmm --> HH:MM:SS.mmm] 原文。写入目标文件时保留这些时间,不可把全文压成一行。简介/人工文案若没有真实时间信息,明确注明无法提供时间戳,禁止猜测或编造。"
|
|
54
|
+
: "字幕或转写按原有分段逐段换行写入目标文件,保留内容结构,不要把全文压成一行;本次不要求在正文标时间戳。",
|
|
51
55
|
"统一流程:① 先取视频字幕(B站 AI 字幕 / YouTube yt-dlp CC);" + noSubtitleBranch,
|
|
52
56
|
"若输入是 B 站链接(bilibili.com / b23.tv,短链先用 curl -L 跟随重定向提取 BV 号):用 curl(Windows 10+ 与 macOS/Linux 均自带,零安装)调官方公开 API:\n" +
|
|
53
57
|
" 1) 元数据+简介:curl 'https://api.bilibili.com/x/web-interface/view?bvid=<BV>' → 取 title / desc(简介仅作背景/补充,不满一篇精读)/ owner.name / duration / aid / cid\n" +
|
|
@@ -64,7 +68,7 @@ module.exports = {
|
|
|
64
68
|
"转写执行(faster-whisper " + whisperModel + " / int8 / " + language + ",镜像+缓存,保证本机与用户一致):\n" +
|
|
65
69
|
" ① 先取音频:B站用 playurl(curl 'https://api.bilibili.com/x/player/playurl?avid=<aid>&cid=<cid>&fnval=16&fourk=1' 的 dash.audio[0].baseUrl)或 yt-dlp,下载存到 '" + tempDir + "/audio.m4a';\n" +
|
|
66
70
|
" ② 定位插件转写脚本(文件名固定 `transcribe.js`):Windows 用 `Get-ChildItem -Path $env:USERPROFILE/.dsh -Recurse -Filter transcribe.js`;macOS/Linux 用 `find \"$HOME/.dsh\" -name transcribe.js`(也可直接用 glob 搜 *transcribe.js);\n" +
|
|
67
|
-
" ③ 运行(**三平台命令行完全一致**,脚本是 Node 写的,与 pwsh / bash 无关):`node <脚本路径> --audio '" + tempDir + "/audio.m4a' --out '" + tempDir + "/transcript.txt' --device " + device + " --model " + whisperModel + " --language " + language + "`;\n" +
|
|
71
|
+
" ③ 运行(**三平台命令行完全一致**,脚本是 Node 写的,与 pwsh / bash 无关):`node <脚本路径> --audio '" + tempDir + "/audio.m4a' --out '" + tempDir + "/transcript.txt' --device " + device + " --model " + whisperModel + " --language " + language + (includeTimestamps ? " --timestamps" : "") + "`;\n" +
|
|
68
72
|
" ④ 转写可能较慢:可用 run_in_background: true 启动,但必须跟踪**本次启动的进程**直到结束;只有确认**本次进程退出码为 0**,且本次目标文件 '" + tempDir + "/transcript.txt' 存在并非空,才算成功。脚本先写本次独有的暂存文件,校验后才原子发布目标文件;**不要仅凭文件非空/稳定或日志出现 DONE 判断完成,不要复用其他任务或上次运行的逐字稿**。设合理超时,超时则终止本次进程;超时/非 0 退出/空产物均按降级处理;\n" +
|
|
69
73
|
" ⑤ 成功后读取 transcript.txt 作为全文(textSource=transcription)。脚本自举:uv 建 Python 3.12 环境 + 镜像装 faster-whisper + hf-mirror 下模型并缓存;Windows/macOS/Linux 是**同一份实现**。**⚠️ 首次转写会先下载模型(small 约 484MB,hf-mirror,非 GitHub)并较慢——先向用户说明这是正常的一次性下载,勿当成卡死;之后缓存复用、秒开。**\n" +
|
|
70
74
|
" ⑥ 若 `node` 不在 PATH(macOS 从图形界面启动时常见),先用 `command -v node` 确认;确认不可用就直接降级,不要为了跑转写脚本去安装 Node。"
|
package/schemas/index.js
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
// (kind 的 enum 按解析器注册表补齐),脚本里不再有第二份副本。
|
|
7
7
|
// - chunkReadSchema:**workflow 没用**。波次2 的子代理直接返回自由文本,
|
|
8
8
|
// 成稿时按 Markdown 拼接;这份留给程序化调用方(自建管线要结构化块输出时)。
|
|
9
|
-
// - qualityChecklistSchema
|
|
10
|
-
//
|
|
9
|
+
// - qualityChecklistSchema:留给程序化调用方的通用六项清单。
|
|
10
|
+
// workflow 使用脚本内的原文核验 schema(逐块摘录与引用行号证据);
|
|
11
|
+
// 格式和篇幅由 lib/quality.js 的程序检查决定,不能被模型的 pass 覆盖。
|
|
11
12
|
// 改这个文件会直接影响波次1 —— 改完跑 npm test(内含 schema 子集回归)。
|
|
12
13
|
//
|
|
13
14
|
// 注意:本 schema 子集遵循 DSH workflow agent() 的约束
|