deep-read-summarize 0.3.18 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.en.md +33 -14
- package/README.md +33 -14
- package/cordis.patch.yml +2 -1
- package/docs/RELEASE.md +3 -1
- package/index.js +140 -59
- package/lib/cache.js +28 -15
- package/lib/options.js +66 -0
- package/lib/parser-bundle.js +197 -0
- package/lib/quality.js +218 -0
- package/lib/workflow-script.txt +200 -58
- package/package.json +5 -1
- package/parsers/index.js +55 -14
- package/parsers/video.js +28 -6
- package/schemas/index.js +3 -2
- package/scripts/transcribe.js +132 -23
- package/skills/deep-read-summarize/SKILL.md +9 -6
- package/workflow.js +13 -20
package/schemas/index.js
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
// (kind 的 enum 按解析器注册表补齐),脚本里不再有第二份副本。
|
|
7
7
|
// - chunkReadSchema:**workflow 没用**。波次2 的子代理直接返回自由文本,
|
|
8
8
|
// 成稿时按 Markdown 拼接;这份留给程序化调用方(自建管线要结构化块输出时)。
|
|
9
|
-
// - qualityChecklistSchema
|
|
10
|
-
//
|
|
9
|
+
// - qualityChecklistSchema:留给程序化调用方的通用六项清单。
|
|
10
|
+
// workflow 使用脚本内的原文核验 schema(逐块摘录与引用行号证据);
|
|
11
|
+
// 格式和篇幅由 lib/quality.js 的程序检查决定,不能被模型的 pass 覆盖。
|
|
11
12
|
// 改这个文件会直接影响波次1 —— 改完跑 npm test(内含 schema 子集回归)。
|
|
12
13
|
//
|
|
13
14
|
// 注意:本 schema 子集遵循 DSH workflow agent() 的约束
|
package/scripts/transcribe.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// transcribe.js — faster-whisper 转写自举脚本(插件内部,跨平台:Windows/macOS/Linux)
|
|
4
4
|
// ============================================================================
|
|
5
5
|
// 用法(三平台完全一致,不需要 pwsh / bash 之别):
|
|
6
|
-
// node transcribe.js --audio <音频文件> --out <输出txt> [--model small] [--language zh] [--device cpu|auto|cuda]
|
|
6
|
+
// node transcribe.js --audio <音频文件> --out <输出txt> [--model small] [--language zh] [--device cpu|auto|cuda] [--timestamps]
|
|
7
7
|
// node transcribe.js --self-check # 只打印解析出的路径,不安装、不下载、不联网
|
|
8
8
|
//
|
|
9
9
|
// 为什么是 Node 而不是 PowerShell / shell:
|
|
@@ -21,8 +21,12 @@ const fs = require('node:fs');
|
|
|
21
21
|
const os = require('node:os');
|
|
22
22
|
const path = require('node:path');
|
|
23
23
|
|
|
24
|
-
//
|
|
24
|
+
// 新环境使用 Python 3.12;ASR 主包固定为已验证版本。
|
|
25
|
+
// 复用环境不强制更换解释器;实际 Python / 传递依赖版本记录在 runtime.json。
|
|
25
26
|
const PY_VERSION = '3.12';
|
|
27
|
+
const FASTER_WHISPER_VERSION = '1.2.1';
|
|
28
|
+
const DEPENDENCY_MISSING = 20;
|
|
29
|
+
const DEPENDENCY_VERSION_MISMATCH = 21;
|
|
26
30
|
const PYPI_MIRROR = 'https://pypi.tuna.tsinghua.edu.cn/simple';
|
|
27
31
|
// uv 拉 CPython 走的是 GitHub(python-build-standalone),在部分网络下极慢 → 默认走镜像加速。
|
|
28
32
|
// ⚠️ 镜像会失效,而且代价很大:旧的 ghproxy.com 现在 301 到 ghfast.top 并丢掉路径,返回的是
|
|
@@ -171,7 +175,7 @@ function isFileAt(p) {
|
|
|
171
175
|
|
|
172
176
|
/** 解析命令行(支持 --k v、--k=v、-k v 三种写法)。 */
|
|
173
177
|
function parseArgs(argv) {
|
|
174
|
-
const out = { audio: '', out: '', model: 'small', language: 'zh', device: 'cpu', selfCheck: false, help: false, unknown: [] };
|
|
178
|
+
const out = { audio: '', out: '', model: 'small', language: 'zh', device: 'cpu', timestamps: false, selfCheck: false, help: false, unknown: [] };
|
|
175
179
|
const takes = {
|
|
176
180
|
'--audio': 'audio', '-a': 'audio',
|
|
177
181
|
'--out': 'out', '-o': 'out',
|
|
@@ -184,6 +188,8 @@ function parseArgs(argv) {
|
|
|
184
188
|
const a = String(list[i]);
|
|
185
189
|
if (a === '--help' || a === '-h') { out.help = true; continue; }
|
|
186
190
|
if (a === '--self-check') { out.selfCheck = true; continue; }
|
|
191
|
+
if (a === '--timestamps' || a === '--timestamps=true') { out.timestamps = true; continue; }
|
|
192
|
+
if (a === '--no-timestamps' || a === '--timestamps=false') { out.timestamps = false; continue; }
|
|
187
193
|
if (takes[a]) {
|
|
188
194
|
const v = list[i + 1];
|
|
189
195
|
if (v === undefined || String(v).indexOf('-') === 0) { out.unknown.push(a); continue; }
|
|
@@ -198,11 +204,65 @@ function parseArgs(argv) {
|
|
|
198
204
|
return out;
|
|
199
205
|
}
|
|
200
206
|
|
|
201
|
-
/** 运行时 Python
|
|
207
|
+
/** 运行时 Python 片段:逐段换行,可保留真实起止时间;device 默认 cpu,非 cpu 失败自动回退。 */
|
|
202
208
|
const PY_RUNNER = [
|
|
203
209
|
"import os",
|
|
204
210
|
"import sys",
|
|
205
|
-
"
|
|
211
|
+
"import json",
|
|
212
|
+
"import tempfile",
|
|
213
|
+
"import importlib.metadata as metadata",
|
|
214
|
+
"",
|
|
215
|
+
"def bootstrap_required(code, reason, message):",
|
|
216
|
+
" status_file = os.environ.get('DRS_BOOTSTRAP_STATUS')",
|
|
217
|
+
" if status_file:",
|
|
218
|
+
" with open(status_file, 'w', encoding='utf-8') as f:",
|
|
219
|
+
" json.dump({'code': code, 'reason': reason}, f)",
|
|
220
|
+
" print('[transcribe] ' + message, file=sys.stderr)",
|
|
221
|
+
" sys.exit(code)",
|
|
222
|
+
"",
|
|
223
|
+
"expected_version = os.environ.get('DRS_FASTER_WHISPER_VERSION')",
|
|
224
|
+
"if expected_version:",
|
|
225
|
+
" try:",
|
|
226
|
+
" installed_version = metadata.version('faster-whisper')",
|
|
227
|
+
" except metadata.PackageNotFoundError:",
|
|
228
|
+
" bootstrap_required(" + DEPENDENCY_MISSING + ", 'missing-dependency', 'faster-whisper is not installed')",
|
|
229
|
+
" if installed_version != expected_version:",
|
|
230
|
+
" bootstrap_required(" + DEPENDENCY_VERSION_MISMATCH + ", 'version-mismatch', 'faster-whisper %s requires %s' % (installed_version, expected_version))",
|
|
231
|
+
"",
|
|
232
|
+
"try:",
|
|
233
|
+
" from faster_whisper import WhisperModel",
|
|
234
|
+
"except ModuleNotFoundError as exc:",
|
|
235
|
+
" bootstrap_required(" + DEPENDENCY_MISSING + ", 'missing-dependency', 'missing import dependency: %s' % exc)",
|
|
236
|
+
"",
|
|
237
|
+
"runtime_versions = None",
|
|
238
|
+
"def record_runtime(device, phase):",
|
|
239
|
+
" global runtime_versions",
|
|
240
|
+
" target = os.environ.get('DRS_RUNTIME')",
|
|
241
|
+
" if not target:",
|
|
242
|
+
" return",
|
|
243
|
+
" if runtime_versions is None:",
|
|
244
|
+
" runtime_versions = {}",
|
|
245
|
+
" for name in ('faster-whisper', 'ctranslate2', 'av', 'huggingface-hub', 'tokenizers', 'onnxruntime'):",
|
|
246
|
+
" try:",
|
|
247
|
+
" runtime_versions[name] = metadata.version(name)",
|
|
248
|
+
" except metadata.PackageNotFoundError:",
|
|
249
|
+
" runtime_versions[name] = None",
|
|
250
|
+
" runtime = {'python': sys.version.split()[0], 'packages': runtime_versions, 'model': os.environ['DRS_MODEL'], 'device': device, 'computeType': 'int8', 'phase': phase}",
|
|
251
|
+
" temporary = None",
|
|
252
|
+
" try:",
|
|
253
|
+
" with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=os.path.dirname(target), prefix='.runtime-', suffix='.json', delete=False) as f:",
|
|
254
|
+
" temporary = f.name",
|
|
255
|
+
" json.dump(runtime, f, ensure_ascii=False, indent=2)",
|
|
256
|
+
" os.replace(temporary, target)",
|
|
257
|
+
" except OSError as exc:",
|
|
258
|
+
" print('[transcribe] runtime record unavailable: %s' % exc, file=sys.stderr)",
|
|
259
|
+
" finally:",
|
|
260
|
+
" if temporary and os.path.exists(temporary):",
|
|
261
|
+
" try:",
|
|
262
|
+
" os.unlink(temporary)",
|
|
263
|
+
" except OSError:",
|
|
264
|
+
" pass",
|
|
265
|
+
" print('[transcribe] runtime python=%s faster-whisper=%s device=%s phase=%s' % (runtime['python'], runtime_versions['faster-whisper'], device, phase), file=sys.stderr)",
|
|
206
266
|
"",
|
|
207
267
|
"def load(device):",
|
|
208
268
|
" return WhisperModel(os.environ['DRS_MODEL'], device=device, compute_type='int8')",
|
|
@@ -210,23 +270,43 @@ const PY_RUNNER = [
|
|
|
210
270
|
"def run(device):",
|
|
211
271
|
" model = load(device)",
|
|
212
272
|
" segs, info = model.transcribe(os.environ['DRS_AUDIO'], vad_filter=True, language=(os.environ.get('DRS_LANG') or None))",
|
|
213
|
-
" return [s.text.strip() for s in segs], info",
|
|
273
|
+
" return [{'start': s.start, 'end': s.end, 'text': s.text.strip()} for s in segs], info",
|
|
274
|
+
"",
|
|
275
|
+
"def timestamp(seconds):",
|
|
276
|
+
" millis = max(0, int(round(float(seconds) * 1000)))",
|
|
277
|
+
" hours, millis = divmod(millis, 3600000)",
|
|
278
|
+
" minutes, millis = divmod(millis, 60000)",
|
|
279
|
+
" seconds, millis = divmod(millis, 1000)",
|
|
280
|
+
" return '%02d:%02d:%02d.%03d' % (hours, minutes, seconds, millis)",
|
|
214
281
|
"",
|
|
215
282
|
"# device 默认 cpu:ctranslate2 只要探测到 NVIDIA 显卡就会在 auto 下选 CUDA,",
|
|
216
283
|
"# 而多数机器没装 cuBLAS/cuDNN(cublas64_12.dll),会在推理时硬失败(实测 RTX 3050 复现)。",
|
|
217
284
|
"device = os.environ.get('DRS_DEVICE') or 'cpu'",
|
|
285
|
+
"record_runtime(device, 'starting')",
|
|
218
286
|
"try:",
|
|
219
287
|
" segments, info = run(device)",
|
|
220
288
|
"except Exception as exc:",
|
|
221
289
|
" if device == 'cpu':",
|
|
222
290
|
" raise",
|
|
223
291
|
" print('[transcribe] device=%s failed (%s); retrying on cpu' % (device, exc), file=sys.stderr)",
|
|
292
|
+
" device = 'cpu'",
|
|
293
|
+
" record_runtime(device, 'starting')",
|
|
224
294
|
" segments, info = run('cpu')",
|
|
295
|
+
"record_runtime(device, 'completed')",
|
|
225
296
|
"",
|
|
226
297
|
"# DRS_OUT 是本次调用独有的暂存路径;Node 校验后才原子发布到最终路径。",
|
|
227
298
|
"# 先收集全部分段再落盘,中途推理失败不会留下半截文本。",
|
|
228
|
-
"
|
|
229
|
-
"
|
|
299
|
+
"include_timestamps = os.environ.get('DRS_TIMESTAMPS') == '1'",
|
|
300
|
+
"lines = []",
|
|
301
|
+
"for segment in segments:",
|
|
302
|
+
" text = segment['text'].replace('\\r', ' ').replace('\\n', ' ')",
|
|
303
|
+
" if not text:",
|
|
304
|
+
" continue",
|
|
305
|
+
" if include_timestamps:",
|
|
306
|
+
" text = '[%s --> %s] %s' % (timestamp(segment['start']), timestamp(segment['end']), text)",
|
|
307
|
+
" lines.append(text)",
|
|
308
|
+
"with open(os.environ['DRS_OUT'], 'w', encoding='utf-8', newline='\\n') as f:",
|
|
309
|
+
" f.write('\\n'.join(lines))",
|
|
230
310
|
""
|
|
231
311
|
].join("\n");
|
|
232
312
|
|
|
@@ -235,11 +315,12 @@ const PY_RUNNER = [
|
|
|
235
315
|
// ---------------------------------------------------------------------------
|
|
236
316
|
|
|
237
317
|
const USAGE = [
|
|
238
|
-
'用法:node transcribe.js --audio <音频文件> --out <输出txt> [--model small] [--language zh] [--device cpu|auto|cuda]',
|
|
318
|
+
'用法:node transcribe.js --audio <音频文件> --out <输出txt> [--model small] [--language zh] [--device cpu|auto|cuda] [--timestamps]',
|
|
239
319
|
' node transcribe.js --self-check # 只打印解析出的路径,不安装不下载',
|
|
240
320
|
'',
|
|
241
321
|
'选项:--audio/-a --out/-o --model/-m(默认 small) --language/-l(默认 zh)',
|
|
242
|
-
' --device(默认 cpu;auto/cuda 需自备 CUDA 运行库,失败会自动回退 cpu)'
|
|
322
|
+
' --device(默认 cpu;auto/cuda 需自备 CUDA 运行库,失败会自动回退 cpu)',
|
|
323
|
+
' --timestamps(逐段保留真实起止时间;默认关闭,关闭时仍按段换行)'
|
|
243
324
|
].join('\n');
|
|
244
325
|
|
|
245
326
|
function fail(msg, code) {
|
|
@@ -249,10 +330,25 @@ function fail(msg, code) {
|
|
|
249
330
|
|
|
250
331
|
function runStep(cmd, cmdArgs, env, errLabel) {
|
|
251
332
|
const r = spawnSync(cmd, cmdArgs, { stdio: 'inherit', env: env, shell: false });
|
|
333
|
+
checkStep(r, cmd, errLabel);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function checkStep(r, cmd, errLabel) {
|
|
252
337
|
if (r.error) throw new Error(errLabel + ' 无法执行(' + cmd + '):' + r.error.message);
|
|
253
338
|
if (r.status !== 0) throw new Error(errLabel + ' 失败(退出码 ' + r.status + ')');
|
|
254
339
|
}
|
|
255
340
|
|
|
341
|
+
/** 仅 runner 导入/版本预检写出的特定状态允许安装,推理失败不能触发安装。 */
|
|
342
|
+
function needsBootstrap(result, statusFile) {
|
|
343
|
+
if (result.error || (result.status !== DEPENDENCY_MISSING && result.status !== DEPENDENCY_VERSION_MISMATCH)) return false;
|
|
344
|
+
try {
|
|
345
|
+
const status = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
|
|
346
|
+
return status.code === result.status &&
|
|
347
|
+
((status.code === DEPENDENCY_MISSING && status.reason === 'missing-dependency') ||
|
|
348
|
+
(status.code === DEPENDENCY_VERSION_MISMATCH && status.reason === 'version-mismatch'));
|
|
349
|
+
} catch (e) { return false; }
|
|
350
|
+
}
|
|
351
|
+
|
|
256
352
|
function main(argv) {
|
|
257
353
|
const args = parseArgs(argv);
|
|
258
354
|
const c = context({});
|
|
@@ -273,6 +369,8 @@ function main(argv) {
|
|
|
273
369
|
uv: uv,
|
|
274
370
|
uvInstallHint: uvInstallHint(c),
|
|
275
371
|
pythonVersion: PY_VERSION,
|
|
372
|
+
fasterWhisperVersion: FASTER_WHISPER_VERSION,
|
|
373
|
+
runtimeRecord: path.join(root, 'runtime.json'),
|
|
276
374
|
node: process.version
|
|
277
375
|
}, null, 2) + '\n');
|
|
278
376
|
return 0;
|
|
@@ -296,9 +394,6 @@ function main(argv) {
|
|
|
296
394
|
fs.rmSync(outputPath, { force: true });
|
|
297
395
|
if (!fs.existsSync(args.audio)) return fail('音频文件不存在:' + args.audio);
|
|
298
396
|
if (!isFileAt(args.audio)) return fail('--audio 不是文件(是不是给了目录?):' + args.audio);
|
|
299
|
-
if (uv === null) {
|
|
300
|
-
return fail('没有找到 uv。请先安装:' + uvInstallHint(c) + '(装好后重跑本脚本;或改用 options.transcribe=false / 手动提供转写文本降级)');
|
|
301
|
-
}
|
|
302
397
|
|
|
303
398
|
fs.mkdirSync(root, { recursive: true });
|
|
304
399
|
// 输出目录可能在别处(如 ./.tmp),提前建好,免得 python 侧报一句难懂的错
|
|
@@ -312,6 +407,7 @@ function main(argv) {
|
|
|
312
407
|
// 1) venv:优先复用已装好的 3.12,没有才让 uv 装。
|
|
313
408
|
// 镜像只用于加速;失败就换直连重试 —— 镜像挂掉不该让用户彻底装不上。
|
|
314
409
|
if (!fs.existsSync(py)) {
|
|
410
|
+
if (uv === null) throw new Error('需要创建 Python 环境,但没有找到 uv。请先安装:' + uvInstallHint(c));
|
|
315
411
|
process.stdout.write('[transcribe] 创建 venv(Python ' + PY_VERSION + '):' + venvDir + '\n');
|
|
316
412
|
const attempts = venvEnvAttempts(childEnv, resolveUvMirror(process.env));
|
|
317
413
|
let lastErr = null;
|
|
@@ -329,14 +425,8 @@ function main(argv) {
|
|
|
329
425
|
if (!fs.existsSync(py)) throw (lastErr || new Error('uv venv 失败:' + venvDir));
|
|
330
426
|
}
|
|
331
427
|
|
|
332
|
-
// 2)
|
|
333
|
-
|
|
334
|
-
if (probe.status !== 0) {
|
|
335
|
-
process.stdout.write('[transcribe] 安装 faster-whisper(清华镜像)…\n');
|
|
336
|
-
runStep(uv, ['pip', 'install', '--python', py, 'faster-whisper', '-i', PYPI_MIRROR], childEnv, 'faster-whisper 安装');
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
// 3) 运行时环境变量:hf-mirror + 关掉会绕开镜像并 401 的 hf-xet 后端
|
|
428
|
+
// 2) 健康环境直接运行;不另开 Python 重复 import,也不要求 uv 仍在 PATH。
|
|
429
|
+
// 运行时环境变量:hf-mirror + 关掉会绕开镜像并 401 的 hf-xet 后端
|
|
340
430
|
childEnv.HF_ENDPOINT = HF_ENDPOINT;
|
|
341
431
|
childEnv.HF_HOME = path.join(root, 'hf');
|
|
342
432
|
childEnv.HF_HUB_DISABLE_XET = '1';
|
|
@@ -346,12 +436,27 @@ function main(argv) {
|
|
|
346
436
|
childEnv.DRS_MODEL = args.model;
|
|
347
437
|
childEnv.DRS_LANG = args.language;
|
|
348
438
|
childEnv.DRS_DEVICE = args.device;
|
|
439
|
+
childEnv.DRS_TIMESTAMPS = args.timestamps ? '1' : '0';
|
|
440
|
+
childEnv.DRS_FASTER_WHISPER_VERSION = FASTER_WHISPER_VERSION;
|
|
441
|
+
childEnv.DRS_RUNTIME = path.join(root, 'runtime.json');
|
|
442
|
+
const bootstrapStatus = path.join(stagingDir, 'bootstrap-status.json');
|
|
443
|
+
childEnv.DRS_BOOTSTRAP_STATUS = bootstrapStatus;
|
|
349
444
|
|
|
350
445
|
const pyScript = path.join(stagingDir, 'transcribe_run.py');
|
|
351
446
|
fs.writeFileSync(pyScript, PY_RUNNER, 'utf8');
|
|
352
447
|
|
|
353
|
-
//
|
|
354
|
-
|
|
448
|
+
// 3) 转写(首次会下载模型到 HF_HOME,之后缓存复用)。
|
|
449
|
+
// 只有导入缺失/明确版本不符才允许自举并重试一次,解码/CUDA/推理失败不重装。
|
|
450
|
+
const result = spawnSync(py, [pyScript], { stdio: 'inherit', env: childEnv, shell: false });
|
|
451
|
+
if (needsBootstrap(result, bootstrapStatus)) {
|
|
452
|
+
if (uv === null) throw new Error('需要安装 faster-whisper==' + FASTER_WHISPER_VERSION + ',但没有找到 uv。请先安装:' + uvInstallHint(c));
|
|
453
|
+
process.stdout.write('[transcribe] 安装 faster-whisper==' + FASTER_WHISPER_VERSION + '(清华镜像)…\n');
|
|
454
|
+
runStep(uv, ['pip', 'install', '--python', py, 'faster-whisper==' + FASTER_WHISPER_VERSION, '-i', PYPI_MIRROR], childEnv, 'faster-whisper 安装');
|
|
455
|
+
fs.rmSync(bootstrapStatus, { force: true });
|
|
456
|
+
runStep(py, [pyScript], childEnv, '转写');
|
|
457
|
+
} else {
|
|
458
|
+
checkStep(result, py, '转写');
|
|
459
|
+
}
|
|
355
460
|
|
|
356
461
|
const bytes = fs.statSync(stagingOutput).size;
|
|
357
462
|
const verdict = judgeTranscript(bytes);
|
|
@@ -389,5 +494,9 @@ module.exports = {
|
|
|
389
494
|
findUv: findUv,
|
|
390
495
|
parseArgs: parseArgs,
|
|
391
496
|
PY_VERSION: PY_VERSION,
|
|
497
|
+
FASTER_WHISPER_VERSION: FASTER_WHISPER_VERSION,
|
|
498
|
+
DEPENDENCY_MISSING: DEPENDENCY_MISSING,
|
|
499
|
+
DEPENDENCY_VERSION_MISMATCH: DEPENDENCY_VERSION_MISMATCH,
|
|
500
|
+
needsBootstrap: needsBootstrap,
|
|
392
501
|
PY_RUNNER: PY_RUNNER
|
|
393
502
|
};
|
|
@@ -13,7 +13,7 @@ whenToUse: 用户提供一本书、论文(arXiv/PDF)、视频链接(YouTub
|
|
|
13
13
|
## 能力
|
|
14
14
|
|
|
15
15
|
- 书籍(PDF/EPUB/MOBI)、论文(arXiv/PDF/HTML)、视频(字幕优先,无字幕自动转写)、网页
|
|
16
|
-
- 长内容分块后由并行子代理精读(MapReduce
|
|
16
|
+
- 长内容分块后由并行子代理精读(MapReduce),降低单个代理的上下文压力;超长资料仍受宿主上下文和工具读取上限约束
|
|
17
17
|
- 获取/分块与质量校验的输出受 JSON Schema 约束,结构稳定、下游可解析
|
|
18
18
|
- 分块计划与子代理数量受 `maxChunks` 强制约束(计划失控会顶穿单次调用时限)
|
|
19
19
|
- 关键引用标注页码/章节/段落,降低编造风险
|
|
@@ -25,6 +25,8 @@ whenToUse: 用户提供一本书、论文(arXiv/PDF)、视频链接(YouTub
|
|
|
25
25
|
用户提供:内容链接或本地文件路径。
|
|
26
26
|
可选参数:`type`(auto/book/paper/video/web)、`options`(minWords / fastMode / maxChunks / maxRetries / requireCitations / includeTimestamps / transcribe / whisperModel / language / device / outputDir / tempDir / frontmatter)。
|
|
27
27
|
|
|
28
|
+
优先调用插件提供的 `deep_read_summarize` 工具,参数为 `{ input, type, options }`。如果当前宿主没有这个工具,按技能末尾提供的 meta / script / args 调用 `workflow` 工具。两种入口执行相同的获取、精读与质量核验。
|
|
29
|
+
|
|
28
30
|
`options.frontmatter` 决定 frontmatter 用哪套键(默认 `obsidian`,与历史行为一致):
|
|
29
31
|
- `obsidian`:title / aliases / author / year / type / url / tags / created / status —— 配 Dataview 用这套
|
|
30
32
|
- `generic`:把 Hugo 的保留键换掉(`type` → `docType`、`url` → `sourceUrl`、`created` → `date`),可直接放进 Hugo / Jekyll
|
|
@@ -37,16 +39,16 @@ whenToUse: 用户提供一本书、论文(arXiv/PDF)、视频链接(YouTub
|
|
|
37
39
|
1. **配置校验**:缺 input 或 type 非法 → 报 FATAL 终止
|
|
38
40
|
2. **幂等检查(由你执行,脚本不做)**:脚本在沙箱内没有文件系统,写不了指纹记录。落盘前先看目标 `filePath` 是否已有同名笔记——已存在就问用户是覆盖还是跳过,别默默重跑一遍烧 token。(要在程序里做去重,可用插件导出的 `lib/cache.js`:fingerprint / hasProcessed / markProcessed。)
|
|
39
41
|
3. **解析器选择**:按类型从 parsers 注册表选解析器(book/paper/video/web)。`auto` 对明确视频链接、arXiv 论文链接、电子书扩展名直接分流;同站的图文/频道/帮助页、PDF/未知网页/文本等模糊输入先轻量识别,不能固定当作网页。识别失败返回 `stage: "detect"`,再请用户显式指定类型
|
|
40
|
-
4. **波次1 获取+分块**:解析器获取全文 → 写入本次独立的临时子目录 → 生成分块计划(JSON Schema 校验)。`tempDir` 是根目录,不要复用其他 `run-*` 任务的 input/audio/transcript
|
|
41
|
-
5. **波次2 并行精读**:每个分块一个子代理深度精读(Map
|
|
42
|
+
4. **波次1 获取+分块**:解析器获取全文 → 写入本次独立的临时子目录 → 生成分块计划(JSON Schema 校验)。`tempDir` 是根目录,不要复用其他 `run-*` 任务的 input/audio/transcript;规范化全部区间并按行数均衡工作量,保持从第 1 行到 totalLines 的完整覆盖,不得截断原文
|
|
43
|
+
5. **波次2 并行精读**:每个分块一个子代理深度精读(Map),最多三个槽位同时工作,空闲即领取下一块,输出保持分块顺序
|
|
42
44
|
6. **波次3 合并成稿**:整合为完整笔记(Reduce),内嵌质量自检
|
|
43
|
-
7.
|
|
44
|
-
8.
|
|
45
|
+
7. **质量校验**:程序检查 frontmatter 模板、必要章节和篇幅;核验代理必须读取本次原文,逐块提供摘录、对第 4 节逐字引用提供原句与行号。不得只看笔记或仅返回 pass。`qualityEvidence` 可供回查;核验失败必须如实报告。`maxRetries` 为 0–3,修订失败、空结果或结构退化时保留上一份有效成稿。缺总行数、精读失败、仅简介或未知视频来源等问题不能靠改写笔记消除,不进行徒劳全文修订
|
|
46
|
+
8. **落盘(必须,别跳过)**:精读工具只返回 `note`(笔记正文)与 `filePath`(建议路径)——**脚本本身没有文件系统权限**。拿到结果后要用 `write` 工具把 `note` 原样写入 `filePath`(目录不存在就先创建);不写就等于笔记丢了,用户只会看到一段返回文本
|
|
45
47
|
9. **如实交代结果**:`ok: false` 要说明卡在哪一步(`stage`)和原因,别拿半成品冒充成品;`qualityPassed: false`、`qualityIssues` 非空、或 `failedChunks > 0`(有块精读失败、笔记存在缺口)时,必须在回复里点出来
|
|
46
48
|
|
|
47
49
|
## 输出
|
|
48
50
|
|
|
49
|
-
|
|
51
|
+
精读工具返回 `{ ok, filePath, note, qualityPassed, qualityIssues, qualityChecks, qualityEvidence, failedChunks, coverageComplete, sourceComplete, textSource, timings }`。`coverageComplete: false` 表示缺少有效总行数,无法确认尾部覆盖;`sourceComplete: false` 表示视频仅有简介或来源未确认,笔记会保留来源限制且质量降级。来源完整性仍依据获取代理声明,不能当作视频真实内容已全部证明。`timings` 提供 detect/fetch/read/draft/quality/total 毫秒耗时,不能由此推测未提供的 token 用量。
|
|
50
52
|
|
|
51
53
|
> ⚠️ **`note` 就是完整笔记正文,`filePath` 只是建议路径**:脚本运行在无文件系统的沙箱里(DSH 的 workflow 契约:脚本只负责协调子代理),**它不会替你写文件**。必须由你(主代理)在拿到结果后用 `write` 工具把它写到 `filePath`——默认 `./output/`,可用 `options.outputDir` 指向 Obsidian 仓库或静态站点的内容目录。
|
|
52
54
|
|
|
@@ -85,6 +87,7 @@ status: 已完成
|
|
|
85
87
|
- 运行 yt-dlp 加 `--socket-timeout 15 --retries 3` 防超时。
|
|
86
88
|
- **跨平台**:Windows / macOS / Linux **同等支持**(0.3.9 起)。无字幕自动转写走 `scripts/transcribe.js`(Node + `uv` + Python 3.12),三平台命令一致:`node <脚本路径> --audio <音频> --out <txt> [--device cpu]`;脚本定位命令按系统选(Windows `Get-ChildItem -Path $env:USERPROFILE/.dsh -Recurse -Filter transcribe.js`,macOS/Linux `find "$HOME/.dsh" -name transcribe.js`)。缓存目录按平台取:Windows `%LOCALAPPDATA%\deep-read-summarize`、macOS `~/Library/Caches/deep-read-summarize`、Linux `$XDG_CACHE_HOME/deep-read-summarize`。若 `node` 不在 PATH(macOS 从图形界面启动时常见)则直接降级,不要为此安装 Node。
|
|
87
89
|
- 默认不标时间戳(`options.includeTimestamps` 可开)
|
|
90
|
+
- 开启时间戳时保留字幕 cue 或转写分段的真实起止时间,输出 `[HH:MM:SS.mmm --> HH:MM:SS.mmm]`;关闭时仍逐段换行。原文没有时间信息时注明无法定位,不得推算或编造。
|
|
88
91
|
|
|
89
92
|
## 参考
|
|
90
93
|
|
package/workflow.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// Version: 3.0.0
|
|
4
4
|
// ============================================================================
|
|
5
5
|
// 【用法】meta → workflow meta;script → workflow script;
|
|
6
|
-
// args: { "input": "<url|path>", "type": "auto", "options": { "minWords": 2500, "fastMode": false, "maxChunks": 4, "transcribe": true, "requireCitations": true, "includeTimestamps": false, "
|
|
6
|
+
// args: { "input": "<url|path>", "type": "auto", "options": { "minWords": 2500, "fastMode": false, "maxChunks": 4, "transcribe": true, "requireCitations": true, "includeTimestamps": false, "outputDir": "./output", "tempDir": "./.tmp" } }
|
|
7
7
|
// ============================================================================
|
|
8
8
|
|
|
9
9
|
const meta = {
|
|
@@ -44,9 +44,10 @@ const RAW_SCRIPT = fs.readFileSync(path.join(__dirname, "lib", "workflow-script.
|
|
|
44
44
|
|
|
45
45
|
// ===== parsers 注册表(运行时注入,实现插件化)=====
|
|
46
46
|
const parsers = require("./parsers");
|
|
47
|
-
const promptImpl = require("./parsers/_prompt");
|
|
48
47
|
const schemas = require("./schemas");
|
|
49
|
-
const
|
|
48
|
+
const { buildParserBundle } = require("./lib/parser-bundle");
|
|
49
|
+
const { normalizeWorkflowOptions } = require("./lib/options");
|
|
50
|
+
const quality = require("./lib/quality");
|
|
50
51
|
|
|
51
52
|
/**
|
|
52
53
|
* 把注册表里的解析器(buildPrompt)连同其依赖的 buildFetchPrompt 内联进
|
|
@@ -59,13 +60,7 @@ const parserRegistry = parsers.registry;
|
|
|
59
60
|
* 和这里两道关都被丢掉)。fetchSchema 同样从 schemas/index.js 注入,
|
|
60
61
|
* 保持单一真相,不再和脚本里的副本各自漂移。
|
|
61
62
|
*/
|
|
62
|
-
function buildSelfContainedScript() {
|
|
63
|
-
const parserSource = (name) =>
|
|
64
|
-
parserRegistry.parsers[name].buildPrompt
|
|
65
|
-
.toString()
|
|
66
|
-
// buildPrompt 闭包引用模块级 prompt(parsers/_prompt),内联后改指 __prompt
|
|
67
|
-
.replace(/\bprompt\.buildFetchPrompt\b/g, "__prompt.buildFetchPrompt");
|
|
68
|
-
|
|
63
|
+
function buildSelfContainedScript(parserRegistry = parsers.registry) {
|
|
69
64
|
const parserNames = Object.keys(parserRegistry.parsers);
|
|
70
65
|
// kind 的合法取值跟着解析器表走(含自定义类型),不再写死四个内置类型。
|
|
71
66
|
const fetchSchema = JSON.parse(JSON.stringify(schemas.fetchResultSchema));
|
|
@@ -74,12 +69,7 @@ function buildSelfContainedScript() {
|
|
|
74
69
|
}
|
|
75
70
|
|
|
76
71
|
const impl = [
|
|
77
|
-
|
|
78
|
-
"const __parsers = {",
|
|
79
|
-
parserNames
|
|
80
|
-
.map((name) => " " + JSON.stringify(name) + ": { buildPrompt: " + parserSource(name) + " }")
|
|
81
|
-
.join(",\n"),
|
|
82
|
-
"};",
|
|
72
|
+
buildParserBundle(parserRegistry),
|
|
83
73
|
"const fetchSchema = " + JSON.stringify(fetchSchema) + ";"
|
|
84
74
|
].join("\n");
|
|
85
75
|
|
|
@@ -87,9 +77,12 @@ function buildSelfContainedScript() {
|
|
|
87
77
|
if (!RAW_SCRIPT.includes(marker)) {
|
|
88
78
|
throw new Error("workflow script marker not found: " + marker);
|
|
89
79
|
}
|
|
90
|
-
|
|
80
|
+
const runtimePaths = { transcribeScript: fs.realpathSync(path.join(__dirname, 'scripts', 'transcribe.js')) };
|
|
81
|
+
return 'const __runtimePaths = ' + JSON.stringify(runtimePaths) + ';\n' +
|
|
82
|
+
'const normalizeWorkflowOptions = ' + normalizeWorkflowOptions.toString() + ';\n' +
|
|
83
|
+
'const __quality = { checkNoteStructure: ' + quality.checkNoteStructure.toString() +
|
|
84
|
+
', validateSourceCheck: ' + quality.validateSourceCheck.toString() + ' };\n' +
|
|
85
|
+
RAW_SCRIPT.replace(marker, impl + "\nconst parsers = args._parsers || __parsers;");
|
|
91
86
|
}
|
|
92
87
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
module.exports = { meta, script, parsers, buildSelfContainedScript };
|
|
88
|
+
module.exports = { meta, get script() { return buildSelfContainedScript(); }, parsers, buildSelfContainedScript };
|