deep-read-summarize 0.3.19 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.en.md +23 -10
- package/README.md +23 -10
- package/cordis.patch.yml +2 -1
- package/docs/RELEASE.md +2 -1
- package/index.js +139 -9
- package/lib/cache.js +5 -12
- package/lib/quality.js +25 -5
- package/lib/workflow-script.txt +116 -32
- package/package.json +2 -1
- package/parsers/index.js +23 -7
- package/parsers/video.js +24 -6
- package/scripts/transcribe.js +104 -15
- package/skills/deep-read-summarize/SKILL.md +8 -6
- package/workflow.js +5 -6
package/lib/workflow-script.txt
CHANGED
|
@@ -6,6 +6,17 @@
|
|
|
6
6
|
// 标题/作者里出现引号时不会再截断 frontmatter(旧版是裸拼 "...")。
|
|
7
7
|
function yamlStr(v) { return JSON.stringify(v == null ? "" : String(v)); }
|
|
8
8
|
|
|
9
|
+
function fatalWorkflowError(error) {
|
|
10
|
+
return !!error && (error.fatal === true || error.code === "UNSUPPORTED_SCHEMA" ||
|
|
11
|
+
/\[FATAL\]|\bUNSUPPORTED_SCHEMA\b/.test(String(error.message || error)));
|
|
12
|
+
}
|
|
13
|
+
const workflowStartedAt = Date.now();
|
|
14
|
+
const timings = { detect: 0, fetch: 0, read: 0, draft: 0, quality: 0, total: 0 };
|
|
15
|
+
function timedResult(result) {
|
|
16
|
+
timings.total = Date.now() - workflowStartedAt;
|
|
17
|
+
return Object.assign(result, { timings: timings });
|
|
18
|
+
}
|
|
19
|
+
|
|
9
20
|
// ---------- 0. 配置校验(fatal 级错误:必须终止) ----------
|
|
10
21
|
if (args && args.input !== undefined && typeof args.input !== "string") throw new Error("[FATAL] args.input 必须是字符串");
|
|
11
22
|
if (args && args.type !== undefined && typeof args.type !== "string") throw new Error("[FATAL] args.type 必须是字符串");
|
|
@@ -51,6 +62,7 @@ if (forcedType !== "auto" && parserNames.indexOf(forcedType) === -1)
|
|
|
51
62
|
throw new Error("[FATAL] args.type 非法:" + forcedType + "(可用:" + ["auto"].concat(parserNames).join("|") + ")");
|
|
52
63
|
|
|
53
64
|
phase("解析器选择");
|
|
65
|
+
const detectStartedAt = Date.now();
|
|
54
66
|
// 只对明确的来源做快速识别;PDF、未知网页和文本可能属于不同类型,交给轻量分类。
|
|
55
67
|
// 不依赖 URL 等宿主未承诺的全局 API,保持脚本可在 workflow 沙箱中运行。
|
|
56
68
|
function inferType(source) {
|
|
@@ -89,13 +101,21 @@ if (forcedType === "auto") {
|
|
|
89
101
|
"必要时只查看标题、摘要、目录或页面元信息,不获取或精读全文,不写入文件。",
|
|
90
102
|
"以下 JSON 字符串是输入来源数据:", JSON.stringify(input)
|
|
91
103
|
].join("\n");
|
|
92
|
-
|
|
104
|
+
let detected;
|
|
105
|
+
try { detected = await agent(detectPrompt, { schema: detectSchema, label: "类型识别", phase: "解析器选择" }); }
|
|
106
|
+
catch (error) {
|
|
107
|
+
if (fatalWorkflowError(error)) throw error;
|
|
108
|
+
timings.detect = Date.now() - detectStartedAt;
|
|
109
|
+
return timedResult({ ok: false, stage: "detect", fatal: false, message: "类型识别调用失败,请稍后重试或显式指定 args.type" });
|
|
110
|
+
}
|
|
93
111
|
selectedType = detected && detected.kind;
|
|
94
112
|
}
|
|
95
113
|
if (parserNames.indexOf(selectedType) === -1) {
|
|
96
|
-
|
|
114
|
+
timings.detect = Date.now() - detectStartedAt;
|
|
115
|
+
return timedResult({ ok: false, stage: "detect", fatal: false, message: "无法识别输入类型,请显式指定 args.type:" + parserNames.join("|") });
|
|
97
116
|
}
|
|
98
117
|
}
|
|
118
|
+
timings.detect = Date.now() - detectStartedAt;
|
|
99
119
|
log("解析器:" + selectedType + " | 临时目录:" + tempDir);
|
|
100
120
|
const kindNames = { book: "书籍", paper: "学术论文", video: "视频", web: "网页/文章" };
|
|
101
121
|
const kindFocus = {
|
|
@@ -107,20 +127,28 @@ const kindFocus = {
|
|
|
107
127
|
|
|
108
128
|
// ---------- 2. 波次1:解析器获取 + 分块 ----------
|
|
109
129
|
phase("获取与分块");
|
|
130
|
+
const fetchStartedAt = Date.now();
|
|
110
131
|
const fetchPrompt = parsers[selectedType]
|
|
111
|
-
.buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe, whisperModel: whisperModel, language: transcribeLanguage, device: transcribeDevice, includeTimestamps: includeTimestamps });
|
|
132
|
+
.buildPrompt(input, { tempDir: tempDir, tempFile: tempFile, maxChunks: maxChunks, transcribe: options.transcribe, whisperModel: whisperModel, language: transcribeLanguage, device: transcribeDevice, includeTimestamps: includeTimestamps, transcribeScriptPath: __runtimePaths.transcribeScript });
|
|
112
133
|
// fetchSchema 由插件在构建时从 schemas/index.js 注入(单一真相,避免两份 schema 漂移)。
|
|
113
134
|
|
|
114
|
-
|
|
135
|
+
let fetched;
|
|
136
|
+
try { fetched = await agent(fetchPrompt, { schema: fetchSchema, label: "获取+分块", phase: "获取与分块" }); }
|
|
137
|
+
catch (error) {
|
|
138
|
+
if (fatalWorkflowError(error)) throw error;
|
|
139
|
+
timings.fetch = Date.now() - fetchStartedAt;
|
|
140
|
+
return timedResult({ ok: false, stage: "fetch", fatal: false, message: "内容获取调用失败,请稍后重试或手动提供文本" });
|
|
141
|
+
}
|
|
115
142
|
|
|
116
143
|
// 失败纪律:内容获取失败 = 可降级错误(返回失败结果,不抛 fatal)
|
|
117
144
|
if (!fetched || !fetched.saved) {
|
|
118
|
-
|
|
145
|
+
timings.fetch = Date.now() - fetchStartedAt;
|
|
146
|
+
return timedResult({ ok: false, stage: "fetch", fatal: false, message: (fetched && fetched.message) || "获取失败(可降级:检查输入或手动提供文本)" });
|
|
119
147
|
}
|
|
120
148
|
// 先校验完整计划,按行排序、裁掉重复覆盖并补齐缺口,再合并相邻块。
|
|
121
149
|
// 块数上限约束的是子代理数量,不能通过丢掉尾部原文来实现。
|
|
122
150
|
const rawPlan = (fetched && fetched.chunkPlan) || [];
|
|
123
|
-
if (rawPlan.length === 0) return { ok: false, stage: "fetch", fatal: false, message: "分块计划为空" };
|
|
151
|
+
if (rawPlan.length === 0) { timings.fetch = Date.now() - fetchStartedAt; return timedResult({ ok: false, stage: "fetch", fatal: false, message: "分块计划为空" }); }
|
|
124
152
|
const lineCount = Number(fetched.totalLines);
|
|
125
153
|
const declaredLines = Number.isFinite(lineCount) && lineCount > 0 ? Math.floor(lineCount) : 0;
|
|
126
154
|
const validPlan = [];
|
|
@@ -136,7 +164,7 @@ for (let i = 0; i < rawPlan.length; i++) {
|
|
|
136
164
|
endLine: declaredLines > 0 ? Math.min(e, declaredLines) : e
|
|
137
165
|
});
|
|
138
166
|
}
|
|
139
|
-
if (validPlan.length === 0) return { ok: false, stage: "fetch", fatal: false, message: "分块计划无效:行号区间缺失或非法" };
|
|
167
|
+
if (validPlan.length === 0) { timings.fetch = Date.now() - fetchStartedAt; return timedResult({ ok: false, stage: "fetch", fatal: false, message: "分块计划无效:行号区间缺失或非法" }); }
|
|
140
168
|
validPlan.sort(function (a, b) { return a.startLine - b.startLine || a.endLine - b.endLine; });
|
|
141
169
|
const coveredPlan = [];
|
|
142
170
|
let nextLine = 1;
|
|
@@ -153,14 +181,17 @@ if (declaredLines > 0 && nextLine <= declaredLines) {
|
|
|
153
181
|
coveredPlan.push({ topic: "补充尾部内容", startLine: nextLine, endLine: declaredLines });
|
|
154
182
|
}
|
|
155
183
|
const chunkPlan = [];
|
|
156
|
-
const
|
|
184
|
+
const knownLines = coveredPlan[coveredPlan.length - 1].endLine;
|
|
185
|
+
const groups = Math.min(maxChunks, coveredPlan.length, knownLines);
|
|
157
186
|
for (let i = 0; i < groups; i++) {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
const
|
|
187
|
+
// Balance source lines, not the number of logical topics: a very long topic
|
|
188
|
+
// may span workers, and every line still belongs to exactly one worker.
|
|
189
|
+
const startLine = Math.floor(i * knownLines / groups) + 1;
|
|
190
|
+
const endLine = Math.floor((i + 1) * knownLines / groups);
|
|
191
|
+
const parts = coveredPlan.filter(function (c) { return c.startLine <= endLine && c.endLine >= startLine; });
|
|
161
192
|
chunkPlan.push({
|
|
162
193
|
id: i + 1, topic: parts.map(function (c) { return c.topic; }).join(" / "),
|
|
163
|
-
startLine:
|
|
194
|
+
startLine: startLine, endLine: endLine
|
|
164
195
|
});
|
|
165
196
|
}
|
|
166
197
|
const coverageComplete = declaredLines > 0;
|
|
@@ -171,13 +202,22 @@ log("分块计划:" + rawPlan.length + " -> " + chunkPlan.length + " 块(上
|
|
|
171
202
|
const kind = selectedType;
|
|
172
203
|
const metadata = (fetched && fetched.metadata) || {};
|
|
173
204
|
const textSource = (metadata && metadata.textSource) || "unknown";
|
|
205
|
+
// Reading every line of a description does not establish coverage of a video.
|
|
206
|
+
// Full transcripts/subtitles remain eligible; descriptions and unknown video
|
|
207
|
+
// sources must retain their source-level gap even if the note is well written.
|
|
208
|
+
const sourceComplete = kind !== "video" || ["subtitle", "transcription", "manual"].indexOf(textSource) !== -1;
|
|
209
|
+
const sourceNotice = sourceComplete ? "" : textSource === "desc"
|
|
210
|
+
? "> 来源限制:本笔记仅依据视频简介生成,未取得完整字幕或逐字稿,不能代表完整视频内容的精读。"
|
|
211
|
+
: "> 来源限制:本次视频来源尚未确认为完整字幕或逐字稿,本笔记不能代表完整视频内容的精读。";
|
|
174
212
|
const srcLabels = { desc: "视频简介/desc", subtitle: "平台字幕", transcription: "完整语音转写", manual: "用户提供转写文本" };
|
|
175
213
|
const srcLabel = srcLabels[textSource] || textSource;
|
|
176
214
|
log("[source] " + textSource + " -> " + srcLabel);
|
|
177
215
|
log("类型:" + kind + " 行数:" + fetched.totalLines + " 块数:" + chunkPlan.length);
|
|
216
|
+
timings.fetch = Date.now() - fetchStartedAt;
|
|
178
217
|
|
|
179
218
|
// ---------- 3. 波次2:并行精读(Map) ----------
|
|
180
219
|
phase("并行精读");
|
|
220
|
+
const readStartedAt = Date.now();
|
|
181
221
|
const chunkPrompts = chunkPlan.map(function (c) {
|
|
182
222
|
return [
|
|
183
223
|
"请深度精读以下内容块,输出结构化精读笔记。",
|
|
@@ -193,18 +233,19 @@ const chunkPrompts = chunkPlan.map(function (c) {
|
|
|
193
233
|
].join("\n");
|
|
194
234
|
});
|
|
195
235
|
|
|
196
|
-
const reads =
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
236
|
+
const reads = new Array(chunkPrompts.length);
|
|
237
|
+
let nextRead = 0;
|
|
238
|
+
const readWorkers = [];
|
|
239
|
+
for (let worker = 0; worker < Math.min(3, chunkPrompts.length); worker++) {
|
|
240
|
+
readWorkers.push(async function () {
|
|
241
|
+
while (nextRead < chunkPrompts.length) {
|
|
242
|
+
const index = nextRead++;
|
|
243
|
+
try { reads[index] = await agent(chunkPrompts[index], { label: "精读块", phase: "并行精读" }); }
|
|
244
|
+
catch (error) { log("精读调用失败,保留对应分块缺口"); reads[index] = ""; }
|
|
245
|
+
}
|
|
246
|
+
});
|
|
207
247
|
}
|
|
248
|
+
await parallel(readWorkers);
|
|
208
249
|
const validReads = reads.filter(function (r) { return typeof r === "string" && r.trim().length > 0; });
|
|
209
250
|
const readEvidence = chunkPlan.map(function (chunk, i) {
|
|
210
251
|
return { chunkId: chunk.id, topic: chunk.topic, startLine: chunk.startLine, endLine: chunk.endLine,
|
|
@@ -215,18 +256,40 @@ const failedChunkIds = readEvidence.filter(function (r) { return !r.readSucceede
|
|
|
215
256
|
|
|
216
257
|
// 失败纪律:某块精读失败 = 可降级(跳过该块,标记缺口继续)
|
|
217
258
|
const failedChunks = chunkPlan.length - validReads.length;
|
|
218
|
-
|
|
259
|
+
timings.read = Date.now() - readStartedAt;
|
|
260
|
+
if (validReads.length === 0) return timedResult({ ok: false, stage: "read", fatal: false, message: "所有分块精读均失败" });
|
|
219
261
|
if (failedChunks > 0) log("警告:" + failedChunks + " 块精读失败,标记缺口继续成稿");
|
|
220
262
|
log("精读完成:" + validReads.length + "/" + chunkPlan.length);
|
|
221
263
|
|
|
222
264
|
// ---------- 4. 波次3:成稿 + 内嵌质量自检(Reduce) ----------
|
|
223
265
|
phase("汇总成稿");
|
|
266
|
+
const draftStartedAt = Date.now();
|
|
224
267
|
const todayDate = new Date();
|
|
225
268
|
const today = todayDate.getFullYear() + "-" + ("0" + (todayDate.getMonth() + 1)).slice(-2) + "-" + ("0" + todayDate.getDate()).slice(-2);
|
|
226
269
|
// 标题先折叠换行/制表与连续空白:否则会同时污染 .md 文件名和 frontmatter。
|
|
227
270
|
const finalTitle = String(metadata.title || "").replace(/[\r\n\t]+/g, " ").replace(/\s+/g, " ").trim() || "未命名内容";
|
|
228
|
-
//
|
|
229
|
-
|
|
271
|
+
// Keep normal titles stable. Changed/long titles receive a deterministic suffix
|
|
272
|
+
// so distinct unsafe titles do not collapse onto the same sanitized filename.
|
|
273
|
+
function titleHash(value) {
|
|
274
|
+
let hash = 2166136261;
|
|
275
|
+
for (let i = 0; i < value.length; i++) { hash ^= value.charCodeAt(i); hash = Math.imul(hash, 16777619); }
|
|
276
|
+
return ("00000000" + (hash >>> 0).toString(16)).slice(-8);
|
|
277
|
+
}
|
|
278
|
+
function utf8Prefix(value, budget) {
|
|
279
|
+
let result = "", bytes = 0;
|
|
280
|
+
for (const character of value) {
|
|
281
|
+
const point = character.codePointAt(0);
|
|
282
|
+
const size = point <= 127 ? 1 : point <= 2047 ? 2 : point <= 65535 ? 3 : 4;
|
|
283
|
+
if (bytes + size > budget) break;
|
|
284
|
+
result += character; bytes += size;
|
|
285
|
+
}
|
|
286
|
+
return result;
|
|
287
|
+
}
|
|
288
|
+
let safeTitle = finalTitle.replace(/[\u0000-\u001f\u007f\\/:*?"<>|]/g, " ").replace(/\s+/g, " ").trim().replace(/[. ]+$/, "") || "未命名内容";
|
|
289
|
+
if (/^(?:CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(?:\.|$)/i.test(safeTitle)) safeTitle = "_" + safeTitle;
|
|
290
|
+
if (safeTitle !== finalTitle || utf8Prefix(safeTitle, 197) !== safeTitle) {
|
|
291
|
+
safeTitle = (utf8Prefix(safeTitle, 188).replace(/[. ]+$/, "") || "未命名内容") + "-" + titleHash(finalTitle);
|
|
292
|
+
}
|
|
230
293
|
const mdPath = outputDir.replace(/[\\/]+$/, "") + "/" + safeTitle + ".md";
|
|
231
294
|
|
|
232
295
|
const extraSections = fastMode ? [] : [
|
|
@@ -266,6 +329,7 @@ fm.push("---");
|
|
|
266
329
|
|
|
267
330
|
const draftPrompt = [
|
|
268
331
|
"将以下多块精读结果整合为完整的结构化精读笔记,只输出标准 Markdown(含 YAML frontmatter)。",
|
|
332
|
+
sourceComplete ? "来源完整性仍需由独立原文核验确认。" : "本次来源不完整。必须在正文开头保留以下可见来源限制,不得称为完整视频精读或补写未取得的视频内容:\n" + sourceNotice,
|
|
269
333
|
"模板:"
|
|
270
334
|
].concat(fm, [
|
|
271
335
|
"# 《" + finalTitle + "》深度精读笔记", "",
|
|
@@ -287,11 +351,13 @@ const draftPrompt = [
|
|
|
287
351
|
|
|
288
352
|
let draft;
|
|
289
353
|
try { draft = await agent(draftPrompt, { label: "汇总成稿", phase: "汇总成稿" }); }
|
|
290
|
-
catch (error) { return { ok: false, stage: "draft", fatal: false, message: "成稿调用失败,未产生可保存的笔记" }; }
|
|
291
|
-
|
|
354
|
+
catch (error) { timings.draft = Date.now() - draftStartedAt; return timedResult({ ok: false, stage: "draft", fatal: false, message: "成稿调用失败,未产生可保存的笔记" }); }
|
|
355
|
+
timings.draft = Date.now() - draftStartedAt;
|
|
356
|
+
if (typeof draft !== "string" || !draft.trim()) return timedResult({ ok: false, stage: "draft", fatal: false, message: "成稿为空或不是字符串" });
|
|
292
357
|
|
|
293
358
|
// ---------- 5. 程序检查 + 独立原文核验 ----------
|
|
294
359
|
phase("质量校验");
|
|
360
|
+
const qualityStartedAt = Date.now();
|
|
295
361
|
let verified = draft;
|
|
296
362
|
let passed = false;
|
|
297
363
|
let issues = [];
|
|
@@ -299,7 +365,11 @@ let qualityChecks = {};
|
|
|
299
365
|
let qualityEvidence = { sourceFile: tempFile, totalLines: declaredLines, failedChunkIds: failedChunkIds, chunkChecks: [], citationChecks: [] };
|
|
300
366
|
const structureOptions = { minWords: minWords, fastMode: fastMode, expectedFrontmatter: fm };
|
|
301
367
|
const sourceContext = { sourceFile: tempFile, input: input, totalLines: declaredLines,
|
|
302
|
-
|
|
368
|
+
kind: kind, textSource: textSource, sourceComplete: sourceComplete,
|
|
369
|
+
chunkPlan: chunkPlan, readEvidence: readEvidence.map(function (r) {
|
|
370
|
+
return { chunkId: r.chunkId, topic: r.topic, startLine: r.startLine, endLine: r.endLine, readSucceeded: r.readSucceeded };
|
|
371
|
+
}), failedChunkIds: failedChunkIds, coverageComplete: coverageComplete };
|
|
372
|
+
const repairableSource = sourceComplete && coverageComplete && failedChunkIds.length === 0;
|
|
303
373
|
for (let round = 0; round <= maxRetries; round++) {
|
|
304
374
|
const localCheck = __quality.checkNoteStructure(verified, structureOptions);
|
|
305
375
|
const checkPrompt = [
|
|
@@ -332,7 +402,7 @@ for (let round = 0; round <= maxRetries; round++) {
|
|
|
332
402
|
try { check = await agent(checkPrompt, { schema: checkSchema, label: "质量校验", phase: "质量校验" }); }
|
|
333
403
|
catch (error) { log("原文核验调用失败,保留当前有效笔记并报告质量未通过"); }
|
|
334
404
|
const sourceCheck = __quality.validateSourceCheck(check, {
|
|
335
|
-
chunkPlan: chunkPlan, failedChunkIds: failedChunkIds, coverageComplete: coverageComplete,
|
|
405
|
+
chunkPlan: chunkPlan, failedChunkIds: failedChunkIds, coverageComplete: coverageComplete, sourceComplete: sourceComplete, textSource: textSource,
|
|
336
406
|
requireCitations: requireCitations, note: verified, visibleNote: localCheck.visibleText, quotes: localCheck.quotes
|
|
337
407
|
});
|
|
338
408
|
issues = Array.from(new Set(localCheck.issues.concat(sourceCheck.issues)));
|
|
@@ -342,6 +412,9 @@ for (let round = 0; round <= maxRetries; round++) {
|
|
|
342
412
|
chunkChecks: sourceCheck.chunkChecks, citationChecks: sourceCheck.citationChecks };
|
|
343
413
|
log("校验:" + (passed ? "通过" : "发现问题 " + issues.length + " 条"));
|
|
344
414
|
if (passed) break;
|
|
415
|
+
// A rewrite cannot recover missing source lines, failed reads, or a full
|
|
416
|
+
// transcript. Preserve the candidate and report the gap after one audit.
|
|
417
|
+
if (!repairableSource) break;
|
|
345
418
|
if (round < maxRetries) {
|
|
346
419
|
const fixPrompt = [
|
|
347
420
|
"修正以下笔记以解决校验问题,输出完整修正版:",
|
|
@@ -365,7 +438,17 @@ for (let round = 0; round <= maxRetries; round++) {
|
|
|
365
438
|
}
|
|
366
439
|
|
|
367
440
|
// ---------- 结果 ----------
|
|
368
|
-
|
|
441
|
+
if (!sourceComplete) {
|
|
442
|
+
// Enforce source disclosure even when the drafting model omits it. Keep the
|
|
443
|
+
// YAML at the beginning and insert the notice outside any model-made fence.
|
|
444
|
+
const front = verified.match(/^\s*---[ \t]*\r?\n[\s\S]*?\r?\n---[ \t]*(?:\r?\n|$)/);
|
|
445
|
+
const prefix = front ? front[0].trim() + "\n\n" : "";
|
|
446
|
+
const body = (front ? verified.slice(front[0].length) : verified).split(/\r?\n/)
|
|
447
|
+
.filter(function (line) { return line.trim() !== sourceNotice; }).join("\n").replace(/^\s*\n/, "");
|
|
448
|
+
verified = prefix + sourceNotice + "\n\n" + body;
|
|
449
|
+
}
|
|
450
|
+
timings.quality = Date.now() - qualityStartedAt;
|
|
451
|
+
return timedResult({
|
|
369
452
|
ok: true, kind: kind, title: finalTitle,
|
|
370
453
|
filePath: mdPath, chunksCount: chunkPlan.length, fastMode: fastMode,
|
|
371
454
|
qualityPassed: passed, qualityIssues: issues,
|
|
@@ -373,5 +456,6 @@ return {
|
|
|
373
456
|
failedChunks: failedChunks,
|
|
374
457
|
coverageComplete: coverageComplete,
|
|
375
458
|
textSource: textSource,
|
|
459
|
+
sourceComplete: sourceComplete,
|
|
376
460
|
note: verified
|
|
377
|
-
};
|
|
461
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "deep-read-summarize",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Deep reading & summarization for DSH: books/papers/videos/web → structured Markdown notes (Obsidian / Logseq / static sites). Plugin parsers, MapReduce deep-read, JSON Schema-constrained sub-tasks.",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"main": "index.js",
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
"index.js",
|
|
17
17
|
"workflow.js",
|
|
18
18
|
"parsers",
|
|
19
|
+
"custom-parsers",
|
|
19
20
|
"schemas",
|
|
20
21
|
"lib",
|
|
21
22
|
"skills",
|
package/parsers/index.js
CHANGED
|
@@ -29,11 +29,22 @@ const builtins = {
|
|
|
29
29
|
* 同名类型会覆盖内置解析器(用户按需替换)。
|
|
30
30
|
*/
|
|
31
31
|
class ParserRegistry {
|
|
32
|
-
constructor() {
|
|
32
|
+
constructor(settings = {}) {
|
|
33
|
+
if (!settings || typeof settings !== 'object' || Array.isArray(settings)) throw new Error('parser registry settings must be an object');
|
|
34
|
+
if (settings.customParsersDir !== undefined && (typeof settings.customParsersDir !== 'string' || !settings.customParsersDir.trim() || /[\u0000\r\n]/.test(settings.customParsersDir))) {
|
|
35
|
+
throw new Error('customParsersDir must be a non-empty directory path');
|
|
36
|
+
}
|
|
33
37
|
this.parsers = Object.create(null);
|
|
34
38
|
this._sources = new WeakMap();
|
|
35
39
|
for (const type of Object.keys(builtins)) this.register(builtins[type], require.resolve('./' + type));
|
|
36
|
-
|
|
40
|
+
const bundledDir = path.join(__dirname, '..', 'custom-parsers');
|
|
41
|
+
this._loadCustom(bundledDir);
|
|
42
|
+
// Explicit profile/project paths overlay package-owned defaults. Never scan
|
|
43
|
+
// arbitrary working directories for executable code without configuration.
|
|
44
|
+
if (settings.customParsersDir !== undefined) {
|
|
45
|
+
const customDir = path.resolve(settings.baseDir || process.cwd(), settings.customParsersDir);
|
|
46
|
+
if (customDir !== path.resolve(bundledDir)) this._loadCustom(customDir, true);
|
|
47
|
+
}
|
|
37
48
|
}
|
|
38
49
|
|
|
39
50
|
/** Remember the module, not just its function: module factories preserve helpers and closures. */
|
|
@@ -60,11 +71,16 @@ class ParserRegistry {
|
|
|
60
71
|
return undefined;
|
|
61
72
|
}
|
|
62
73
|
|
|
63
|
-
_loadCustom() {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
74
|
+
_loadCustom(customDir, explicit) {
|
|
75
|
+
if (!fs.existsSync(customDir)) {
|
|
76
|
+
if (explicit) console.warn('[parsers] custom parser directory does not exist:', customDir);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
let files;
|
|
80
|
+
try { files = fs.readdirSync(customDir); }
|
|
81
|
+
catch (error) { console.warn('[parsers] cannot read custom parser directory:', customDir, error.message); return; }
|
|
82
|
+
for (const file of files.sort()) {
|
|
83
|
+
if (!/\.(?:js|cjs)$/.test(file)) continue;
|
|
68
84
|
try {
|
|
69
85
|
const filename = path.join(customDir, file);
|
|
70
86
|
const mod = require(filename);
|
package/parsers/video.js
CHANGED
|
@@ -33,6 +33,21 @@ module.exports = {
|
|
|
33
33
|
const device = o.device || "cpu";
|
|
34
34
|
const includeTimestamps = o.includeTimestamps === true;
|
|
35
35
|
const degradeMsg = "需要人工提供转写文本或视频文案";
|
|
36
|
+
const sourceUrl = String(input).match(/https?:\/\/[^\s<>"']+/i);
|
|
37
|
+
const source = sourceUrl ? sourceUrl[0] : String(input);
|
|
38
|
+
const isBilibili = /^https?:\/\/(?:[^/?#]*@)?(?:[^/?#:]+\.)?(?:bilibili\.com|b23\.tv)(?=[:/?#]|$)/i.test(source);
|
|
39
|
+
const pageMatch = isBilibili ? source.split('#')[0].match(/[?&]p=([^&#]*)/) : null;
|
|
40
|
+
let pageValue = pageMatch ? pageMatch[1] : '';
|
|
41
|
+
try { pageValue = decodeURIComponent(pageValue); } catch (error) { pageValue = 'invalid'; }
|
|
42
|
+
if (pageMatch && (!/^[1-9]\d*$/.test(pageValue) || !Number.isSafeInteger(Number(pageValue)))) {
|
|
43
|
+
return 'B站输入的 p 分P参数非法,必须是从 1 开始的整数。不要获取字幕或音频,也不要转写。只输出 ' +
|
|
44
|
+
JSON.stringify({ saved: false, message: 'B站 p 分P参数非法,请指定有效的正整数分P编号' });
|
|
45
|
+
}
|
|
46
|
+
const requestedPage = pageMatch ? Number(pageValue) : null;
|
|
47
|
+
const scriptPath = typeof o.transcribeScriptPath === 'string' && o.transcribeScriptPath ? o.transcribeScriptPath : '';
|
|
48
|
+
// JSON encoding preserves arbitrary package paths as data. The executing
|
|
49
|
+
// agent must quote the path for its shell, rather than copying raw text.
|
|
50
|
+
const scriptCommand = scriptPath ? JSON.stringify(scriptPath) : '<脚本路径>';
|
|
36
51
|
|
|
37
52
|
const noSubtitleBranch = wantTranscribe
|
|
38
53
|
? "② 若**无公开字幕**,用插件自带 faster-whisper 转写得到全文;③ 仍不可行才降级(提示人工提供转写文本 或 退回 desc 作背景)。"
|
|
@@ -53,9 +68,10 @@ module.exports = {
|
|
|
53
68
|
? "保留真实时间定位:B站字幕保留每条 from/to,YouTube CC 保留每条 cue 的 start/end;转成文本时每段独占一行,格式为 [HH:MM:SS.mmm --> HH:MM:SS.mmm] 原文。写入目标文件时保留这些时间,不可把全文压成一行。简介/人工文案若没有真实时间信息,明确注明无法提供时间戳,禁止猜测或编造。"
|
|
54
69
|
: "字幕或转写按原有分段逐段换行写入目标文件,保留内容结构,不要把全文压成一行;本次不要求在正文标时间戳。",
|
|
55
70
|
"统一流程:① 先取视频字幕(B站 AI 字幕 / YouTube yt-dlp CC);" + noSubtitleBranch,
|
|
56
|
-
"若输入是 B 站链接(bilibili.com / b23.tv
|
|
57
|
-
"
|
|
58
|
-
"
|
|
71
|
+
"若输入是 B 站链接(bilibili.com / b23.tv):用 curl(Windows 10+ 与 macOS/Linux 均自带,零安装)调官方公开 API:\n" +
|
|
72
|
+
" 0) 短链用 curl -L 取得完整最终 URL,保留 query。输入明确带 p 时保留该分P选择;否则读取重定向最终 URL 的 p;都没有才默认 p=1。不能只提取 BV 号而丢掉 p。p 必须为正整数,非法则 saved=false,禁止偷偷改读第1P。" + (requestedPage === null ? '' : '本次输入明确指定 p=' + requestedPage + '。') + "\n" +
|
|
73
|
+
" 1) 元数据+简介:curl 'https://api.bilibili.com/x/web-interface/view?bvid=<BV>' → 取 title / desc(简介仅作背景/补充,不满一篇精读)/ owner.name / aid / pages。检查 p <= pages.length,取 selectedPage = pages[p-1](并核对 selectedPage.page === p);越界或该项缺失则 saved=false。选用 selectedPage.cid / selectedPage.part / selectedPage.duration;不能取顶层 cid/duration 代替指定分P。metadata 记录 bvid / page / cid / part / duration,标题包含分P名称,来源 URL 保留 p。\n" +
|
|
74
|
+
" 2) 字幕(**优先,最快**):curl 'https://api.bilibili.com/x/player/v2?aid=<aid>&cid=<selectedPage.cid>' 取 subtitle 列表;若有则下载转文本(textSource=subtitle)——只使用选中分P的字幕\n" +
|
|
59
75
|
biliNoSubtitle,
|
|
60
76
|
|
|
61
77
|
"若输入是抖音链接/分享口令(douyin.com / v.douyin.com):用户分享时复制的文字本身就是该视频的文案(desc)。" + douyinTail,
|
|
@@ -66,9 +82,11 @@ module.exports = {
|
|
|
66
82
|
if (wantTranscribe) {
|
|
67
83
|
steps.push(
|
|
68
84
|
"转写执行(faster-whisper " + whisperModel + " / int8 / " + language + ",镜像+缓存,保证本机与用户一致):\n" +
|
|
69
|
-
" ① 先取音频:B站用 playurl(curl 'https://api.bilibili.com/x/player/playurl?avid=<aid>&cid=<cid>&fnval=16&fourk=1' 的 dash.audio[0].baseUrl)或 yt-dlp,下载存到 '" + tempDir + "/audio.m4a';\n" +
|
|
70
|
-
|
|
71
|
-
|
|
85
|
+
" ① 先取音频:B站用 playurl(curl 'https://api.bilibili.com/x/player/playurl?avid=<aid>&cid=<selectedPage.cid>&fnval=16&fourk=1' 的 dash.audio[0].baseUrl)或 yt-dlp,下载存到 '" + tempDir + "/audio.m4a';B站必须使用上面选中的 selectedPage.cid,yt-dlp 也必须保留指定分P,禁止改用第1P;\n" +
|
|
86
|
+
(scriptPath
|
|
87
|
+
? " ② 插件已定位本版本转写脚本:" + JSON.stringify(scriptPath) + "。直接使用这个文件;不要递归搜索其他安装目录或选取其他版本。路径是 JSON 字符串数据,执行前按当前 shell 规则正确引用(空格/引号/美元符号等不能作为命令解释);\n"
|
|
88
|
+
: " ② 定位插件转写脚本(文件名固定 `transcribe.js`):Windows 用 `Get-ChildItem -Path $env:USERPROFILE/.dsh -Recurse -Filter transcribe.js`;macOS/Linux 用 `find \"$HOME/.dsh\" -name transcribe.js`(也可直接用 glob 搜 *transcribe.js);\n") +
|
|
89
|
+
" ③ 运行(**三平台命令行完全一致**,脚本是 Node 写的,与 pwsh / bash 无关):`node " + scriptCommand + " --audio '" + tempDir + "/audio.m4a' --out '" + tempDir + "/transcript.txt' --device " + device + " --model " + whisperModel + " --language " + language + (includeTimestamps ? " --timestamps" : "") + "`;\n" +
|
|
72
90
|
" ④ 转写可能较慢:可用 run_in_background: true 启动,但必须跟踪**本次启动的进程**直到结束;只有确认**本次进程退出码为 0**,且本次目标文件 '" + tempDir + "/transcript.txt' 存在并非空,才算成功。脚本先写本次独有的暂存文件,校验后才原子发布目标文件;**不要仅凭文件非空/稳定或日志出现 DONE 判断完成,不要复用其他任务或上次运行的逐字稿**。设合理超时,超时则终止本次进程;超时/非 0 退出/空产物均按降级处理;\n" +
|
|
73
91
|
" ⑤ 成功后读取 transcript.txt 作为全文(textSource=transcription)。脚本自举:uv 建 Python 3.12 环境 + 镜像装 faster-whisper + hf-mirror 下模型并缓存;Windows/macOS/Linux 是**同一份实现**。**⚠️ 首次转写会先下载模型(small 约 484MB,hf-mirror,非 GitHub)并较慢——先向用户说明这是正常的一次性下载,勿当成卡死;之后缓存复用、秒开。**\n" +
|
|
74
92
|
" ⑥ 若 `node` 不在 PATH(macOS 从图形界面启动时常见),先用 `command -v node` 确认;确认不可用就直接降级,不要为了跑转写脚本去安装 Node。"
|
package/scripts/transcribe.js
CHANGED
|
@@ -21,8 +21,12 @@ const fs = require('node:fs');
|
|
|
21
21
|
const os = require('node:os');
|
|
22
22
|
const path = require('node:path');
|
|
23
23
|
|
|
24
|
-
//
|
|
24
|
+
// 新环境使用 Python 3.12;ASR 主包固定为已验证版本。
|
|
25
|
+
// 复用环境不强制更换解释器;实际 Python / 传递依赖版本记录在 runtime.json。
|
|
25
26
|
const PY_VERSION = '3.12';
|
|
27
|
+
const FASTER_WHISPER_VERSION = '1.2.1';
|
|
28
|
+
const DEPENDENCY_MISSING = 20;
|
|
29
|
+
const DEPENDENCY_VERSION_MISMATCH = 21;
|
|
26
30
|
const PYPI_MIRROR = 'https://pypi.tuna.tsinghua.edu.cn/simple';
|
|
27
31
|
// uv 拉 CPython 走的是 GitHub(python-build-standalone),在部分网络下极慢 → 默认走镜像加速。
|
|
28
32
|
// ⚠️ 镜像会失效,而且代价很大:旧的 ghproxy.com 现在 301 到 ghfast.top 并丢掉路径,返回的是
|
|
@@ -204,7 +208,61 @@ function parseArgs(argv) {
|
|
|
204
208
|
const PY_RUNNER = [
|
|
205
209
|
"import os",
|
|
206
210
|
"import sys",
|
|
207
|
-
"
|
|
211
|
+
"import json",
|
|
212
|
+
"import tempfile",
|
|
213
|
+
"import importlib.metadata as metadata",
|
|
214
|
+
"",
|
|
215
|
+
"def bootstrap_required(code, reason, message):",
|
|
216
|
+
" status_file = os.environ.get('DRS_BOOTSTRAP_STATUS')",
|
|
217
|
+
" if status_file:",
|
|
218
|
+
" with open(status_file, 'w', encoding='utf-8') as f:",
|
|
219
|
+
" json.dump({'code': code, 'reason': reason}, f)",
|
|
220
|
+
" print('[transcribe] ' + message, file=sys.stderr)",
|
|
221
|
+
" sys.exit(code)",
|
|
222
|
+
"",
|
|
223
|
+
"expected_version = os.environ.get('DRS_FASTER_WHISPER_VERSION')",
|
|
224
|
+
"if expected_version:",
|
|
225
|
+
" try:",
|
|
226
|
+
" installed_version = metadata.version('faster-whisper')",
|
|
227
|
+
" except metadata.PackageNotFoundError:",
|
|
228
|
+
" bootstrap_required(" + DEPENDENCY_MISSING + ", 'missing-dependency', 'faster-whisper is not installed')",
|
|
229
|
+
" if installed_version != expected_version:",
|
|
230
|
+
" bootstrap_required(" + DEPENDENCY_VERSION_MISMATCH + ", 'version-mismatch', 'faster-whisper %s requires %s' % (installed_version, expected_version))",
|
|
231
|
+
"",
|
|
232
|
+
"try:",
|
|
233
|
+
" from faster_whisper import WhisperModel",
|
|
234
|
+
"except ModuleNotFoundError as exc:",
|
|
235
|
+
" bootstrap_required(" + DEPENDENCY_MISSING + ", 'missing-dependency', 'missing import dependency: %s' % exc)",
|
|
236
|
+
"",
|
|
237
|
+
"runtime_versions = None",
|
|
238
|
+
"def record_runtime(device, phase):",
|
|
239
|
+
" global runtime_versions",
|
|
240
|
+
" target = os.environ.get('DRS_RUNTIME')",
|
|
241
|
+
" if not target:",
|
|
242
|
+
" return",
|
|
243
|
+
" if runtime_versions is None:",
|
|
244
|
+
" runtime_versions = {}",
|
|
245
|
+
" for name in ('faster-whisper', 'ctranslate2', 'av', 'huggingface-hub', 'tokenizers', 'onnxruntime'):",
|
|
246
|
+
" try:",
|
|
247
|
+
" runtime_versions[name] = metadata.version(name)",
|
|
248
|
+
" except metadata.PackageNotFoundError:",
|
|
249
|
+
" runtime_versions[name] = None",
|
|
250
|
+
" runtime = {'python': sys.version.split()[0], 'packages': runtime_versions, 'model': os.environ['DRS_MODEL'], 'device': device, 'computeType': 'int8', 'phase': phase}",
|
|
251
|
+
" temporary = None",
|
|
252
|
+
" try:",
|
|
253
|
+
" with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=os.path.dirname(target), prefix='.runtime-', suffix='.json', delete=False) as f:",
|
|
254
|
+
" temporary = f.name",
|
|
255
|
+
" json.dump(runtime, f, ensure_ascii=False, indent=2)",
|
|
256
|
+
" os.replace(temporary, target)",
|
|
257
|
+
" except OSError as exc:",
|
|
258
|
+
" print('[transcribe] runtime record unavailable: %s' % exc, file=sys.stderr)",
|
|
259
|
+
" finally:",
|
|
260
|
+
" if temporary and os.path.exists(temporary):",
|
|
261
|
+
" try:",
|
|
262
|
+
" os.unlink(temporary)",
|
|
263
|
+
" except OSError:",
|
|
264
|
+
" pass",
|
|
265
|
+
" print('[transcribe] runtime python=%s faster-whisper=%s device=%s phase=%s' % (runtime['python'], runtime_versions['faster-whisper'], device, phase), file=sys.stderr)",
|
|
208
266
|
"",
|
|
209
267
|
"def load(device):",
|
|
210
268
|
" return WhisperModel(os.environ['DRS_MODEL'], device=device, compute_type='int8')",
|
|
@@ -224,13 +282,17 @@ const PY_RUNNER = [
|
|
|
224
282
|
"# device 默认 cpu:ctranslate2 只要探测到 NVIDIA 显卡就会在 auto 下选 CUDA,",
|
|
225
283
|
"# 而多数机器没装 cuBLAS/cuDNN(cublas64_12.dll),会在推理时硬失败(实测 RTX 3050 复现)。",
|
|
226
284
|
"device = os.environ.get('DRS_DEVICE') or 'cpu'",
|
|
285
|
+
"record_runtime(device, 'starting')",
|
|
227
286
|
"try:",
|
|
228
287
|
" segments, info = run(device)",
|
|
229
288
|
"except Exception as exc:",
|
|
230
289
|
" if device == 'cpu':",
|
|
231
290
|
" raise",
|
|
232
291
|
" print('[transcribe] device=%s failed (%s); retrying on cpu' % (device, exc), file=sys.stderr)",
|
|
292
|
+
" device = 'cpu'",
|
|
293
|
+
" record_runtime(device, 'starting')",
|
|
233
294
|
" segments, info = run('cpu')",
|
|
295
|
+
"record_runtime(device, 'completed')",
|
|
234
296
|
"",
|
|
235
297
|
"# DRS_OUT 是本次调用独有的暂存路径;Node 校验后才原子发布到最终路径。",
|
|
236
298
|
"# 先收集全部分段再落盘,中途推理失败不会留下半截文本。",
|
|
@@ -268,10 +330,25 @@ function fail(msg, code) {
|
|
|
268
330
|
|
|
269
331
|
function runStep(cmd, cmdArgs, env, errLabel) {
|
|
270
332
|
const r = spawnSync(cmd, cmdArgs, { stdio: 'inherit', env: env, shell: false });
|
|
333
|
+
checkStep(r, cmd, errLabel);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function checkStep(r, cmd, errLabel) {
|
|
271
337
|
if (r.error) throw new Error(errLabel + ' 无法执行(' + cmd + '):' + r.error.message);
|
|
272
338
|
if (r.status !== 0) throw new Error(errLabel + ' 失败(退出码 ' + r.status + ')');
|
|
273
339
|
}
|
|
274
340
|
|
|
341
|
+
/** 仅 runner 导入/版本预检写出的特定状态允许安装,推理失败不能触发安装。 */
|
|
342
|
+
function needsBootstrap(result, statusFile) {
|
|
343
|
+
if (result.error || (result.status !== DEPENDENCY_MISSING && result.status !== DEPENDENCY_VERSION_MISMATCH)) return false;
|
|
344
|
+
try {
|
|
345
|
+
const status = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
|
|
346
|
+
return status.code === result.status &&
|
|
347
|
+
((status.code === DEPENDENCY_MISSING && status.reason === 'missing-dependency') ||
|
|
348
|
+
(status.code === DEPENDENCY_VERSION_MISMATCH && status.reason === 'version-mismatch'));
|
|
349
|
+
} catch (e) { return false; }
|
|
350
|
+
}
|
|
351
|
+
|
|
275
352
|
function main(argv) {
|
|
276
353
|
const args = parseArgs(argv);
|
|
277
354
|
const c = context({});
|
|
@@ -292,6 +369,8 @@ function main(argv) {
|
|
|
292
369
|
uv: uv,
|
|
293
370
|
uvInstallHint: uvInstallHint(c),
|
|
294
371
|
pythonVersion: PY_VERSION,
|
|
372
|
+
fasterWhisperVersion: FASTER_WHISPER_VERSION,
|
|
373
|
+
runtimeRecord: path.join(root, 'runtime.json'),
|
|
295
374
|
node: process.version
|
|
296
375
|
}, null, 2) + '\n');
|
|
297
376
|
return 0;
|
|
@@ -315,9 +394,6 @@ function main(argv) {
|
|
|
315
394
|
fs.rmSync(outputPath, { force: true });
|
|
316
395
|
if (!fs.existsSync(args.audio)) return fail('音频文件不存在:' + args.audio);
|
|
317
396
|
if (!isFileAt(args.audio)) return fail('--audio 不是文件(是不是给了目录?):' + args.audio);
|
|
318
|
-
if (uv === null) {
|
|
319
|
-
return fail('没有找到 uv。请先安装:' + uvInstallHint(c) + '(装好后重跑本脚本;或改用 options.transcribe=false / 手动提供转写文本降级)');
|
|
320
|
-
}
|
|
321
397
|
|
|
322
398
|
fs.mkdirSync(root, { recursive: true });
|
|
323
399
|
// 输出目录可能在别处(如 ./.tmp),提前建好,免得 python 侧报一句难懂的错
|
|
@@ -331,6 +407,7 @@ function main(argv) {
|
|
|
331
407
|
// 1) venv:优先复用已装好的 3.12,没有才让 uv 装。
|
|
332
408
|
// 镜像只用于加速;失败就换直连重试 —— 镜像挂掉不该让用户彻底装不上。
|
|
333
409
|
if (!fs.existsSync(py)) {
|
|
410
|
+
if (uv === null) throw new Error('需要创建 Python 环境,但没有找到 uv。请先安装:' + uvInstallHint(c));
|
|
334
411
|
process.stdout.write('[transcribe] 创建 venv(Python ' + PY_VERSION + '):' + venvDir + '\n');
|
|
335
412
|
const attempts = venvEnvAttempts(childEnv, resolveUvMirror(process.env));
|
|
336
413
|
let lastErr = null;
|
|
@@ -348,14 +425,8 @@ function main(argv) {
|
|
|
348
425
|
if (!fs.existsSync(py)) throw (lastErr || new Error('uv venv 失败:' + venvDir));
|
|
349
426
|
}
|
|
350
427
|
|
|
351
|
-
// 2)
|
|
352
|
-
|
|
353
|
-
if (probe.status !== 0) {
|
|
354
|
-
process.stdout.write('[transcribe] 安装 faster-whisper(清华镜像)…\n');
|
|
355
|
-
runStep(uv, ['pip', 'install', '--python', py, 'faster-whisper', '-i', PYPI_MIRROR], childEnv, 'faster-whisper 安装');
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
// 3) 运行时环境变量:hf-mirror + 关掉会绕开镜像并 401 的 hf-xet 后端
|
|
428
|
+
// 2) 健康环境直接运行;不另开 Python 重复 import,也不要求 uv 仍在 PATH。
|
|
429
|
+
// 运行时环境变量:hf-mirror + 关掉会绕开镜像并 401 的 hf-xet 后端
|
|
359
430
|
childEnv.HF_ENDPOINT = HF_ENDPOINT;
|
|
360
431
|
childEnv.HF_HOME = path.join(root, 'hf');
|
|
361
432
|
childEnv.HF_HUB_DISABLE_XET = '1';
|
|
@@ -366,12 +437,26 @@ function main(argv) {
|
|
|
366
437
|
childEnv.DRS_LANG = args.language;
|
|
367
438
|
childEnv.DRS_DEVICE = args.device;
|
|
368
439
|
childEnv.DRS_TIMESTAMPS = args.timestamps ? '1' : '0';
|
|
440
|
+
childEnv.DRS_FASTER_WHISPER_VERSION = FASTER_WHISPER_VERSION;
|
|
441
|
+
childEnv.DRS_RUNTIME = path.join(root, 'runtime.json');
|
|
442
|
+
const bootstrapStatus = path.join(stagingDir, 'bootstrap-status.json');
|
|
443
|
+
childEnv.DRS_BOOTSTRAP_STATUS = bootstrapStatus;
|
|
369
444
|
|
|
370
445
|
const pyScript = path.join(stagingDir, 'transcribe_run.py');
|
|
371
446
|
fs.writeFileSync(pyScript, PY_RUNNER, 'utf8');
|
|
372
447
|
|
|
373
|
-
//
|
|
374
|
-
|
|
448
|
+
// 3) 转写(首次会下载模型到 HF_HOME,之后缓存复用)。
|
|
449
|
+
// 只有导入缺失/明确版本不符才允许自举并重试一次,解码/CUDA/推理失败不重装。
|
|
450
|
+
const result = spawnSync(py, [pyScript], { stdio: 'inherit', env: childEnv, shell: false });
|
|
451
|
+
if (needsBootstrap(result, bootstrapStatus)) {
|
|
452
|
+
if (uv === null) throw new Error('需要安装 faster-whisper==' + FASTER_WHISPER_VERSION + ',但没有找到 uv。请先安装:' + uvInstallHint(c));
|
|
453
|
+
process.stdout.write('[transcribe] 安装 faster-whisper==' + FASTER_WHISPER_VERSION + '(清华镜像)…\n');
|
|
454
|
+
runStep(uv, ['pip', 'install', '--python', py, 'faster-whisper==' + FASTER_WHISPER_VERSION, '-i', PYPI_MIRROR], childEnv, 'faster-whisper 安装');
|
|
455
|
+
fs.rmSync(bootstrapStatus, { force: true });
|
|
456
|
+
runStep(py, [pyScript], childEnv, '转写');
|
|
457
|
+
} else {
|
|
458
|
+
checkStep(result, py, '转写');
|
|
459
|
+
}
|
|
375
460
|
|
|
376
461
|
const bytes = fs.statSync(stagingOutput).size;
|
|
377
462
|
const verdict = judgeTranscript(bytes);
|
|
@@ -409,5 +494,9 @@ module.exports = {
|
|
|
409
494
|
findUv: findUv,
|
|
410
495
|
parseArgs: parseArgs,
|
|
411
496
|
PY_VERSION: PY_VERSION,
|
|
497
|
+
FASTER_WHISPER_VERSION: FASTER_WHISPER_VERSION,
|
|
498
|
+
DEPENDENCY_MISSING: DEPENDENCY_MISSING,
|
|
499
|
+
DEPENDENCY_VERSION_MISMATCH: DEPENDENCY_VERSION_MISMATCH,
|
|
500
|
+
needsBootstrap: needsBootstrap,
|
|
412
501
|
PY_RUNNER: PY_RUNNER
|
|
413
502
|
};
|