dsh-video-insight 0.8.1 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/client/client.js +10 -0
- package/lib/asr.js +161 -44
- package/lib/assets.js +59 -39
- package/lib/frames.js +4 -1
- package/lib/index.js +11 -7
- package/lib/vision.js +50 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](LICENSE)
|
|
4
4
|
[](https://github.com/deepseek-ai)
|
|
5
5
|
[](#已知限制)
|
|
6
|
-
[](package.json)
|
|
7
7
|
|
|
8
8
|

|
|
9
9
|
|
|
@@ -153,7 +153,7 @@ host 半零外部依赖(只用 Node 内置 `fetch`/`crypto`),不写锁文
|
|
|
153
153
|
| GET | `/dsh-video/api/status` | `{ok,value:{version,model,bilibili:{cookie},platforms,transcriptLimitChars,asr:{enabled,ready,engine,activeEngine,engineLabel,downloadReady}}}` |
|
|
154
154
|
| GET | `/dsh-video/api/resolve?url=&force=1` | 解析链接 → `VideoDoc`(失败也是 HTTP 200 + `{ok:false,code,error}`) |
|
|
155
155
|
| GET | `/dsh-video/api/asr/status?fresh=1` | 本地转写组件/引擎/模型状态 + `ocr`(画面文字识别)+ `vision`(画面理解/视觉模型能力)+ `privacy` 文案 + `hints` |
|
|
156
|
-
| POST | `/dsh-video/api/asr/config` | `{enabled?,engine?,threads?,maxSeconds?,keepAudio?,autoTranscribe?,ocrFallback?,ocrIntervalSeconds?,ocrMaxFrames?,frameSampling?:cuts\|uniform,cutThreshold?,frameMode?:auto\|ocr\|vision,visionModel?,visionMaxFrames?,visionPack?:sheet\|frames,sheetCols?}` → 新状态 |
|
|
156
|
+
| POST | `/dsh-video/api/asr/config` | `{enabled?,engine?,threads?,maxSeconds?,keepAudio?,autoTranscribe?,crossCheck?,ocrFallback?,ocrIntervalSeconds?,ocrMaxFrames?,frameSampling?:cuts\|uniform,cutThreshold?,frameMode?:auto\|ocr\|vision,visionModel?,visionMaxFrames?,visionPack?:sheet\|frames,sheetCols?}` → 新状态 |
|
|
157
157
|
| POST | `/dsh-video/api/transcribe` | SSE:`{url?/docId?, force?}`,帧 `phase` / `progress{percent,stage}` / `done{docId,transcript,cached?}` / `error{code,message}`;无语音时自动兜底 OCR / 画面理解(stage 含 `vision`) |
|
|
158
158
|
| POST | `/dsh-video/api/analyze` | SSE:`{mode:'summary'\|'ask', url?/docId?, question?, history?}`,帧 `phase` / `delta` / `done` / `error` |
|
|
159
159
|
| POST | `/dsh-video/api/chapters` | `{url?/docId?}` → `{ok,value:{chapters:[{start,title,gist}],duration}}` |
|
|
@@ -214,6 +214,7 @@ host 半零外部依赖(只用 Node 内置 `fetch`/`crypto`),不写锁文
|
|
|
214
214
|
enabled: true
|
|
215
215
|
engine: auto # auto | whisper | sensevoice
|
|
216
216
|
autoTranscribe: true # 解析到无字幕视频时自动本地转写
|
|
217
|
+
crossCheck: true # 语音与画面两路并行分析、结果互相印证(默认 true)
|
|
217
218
|
ocrFallback: true # 连语音都没有时,抽帧做画面文字识别(OCR)
|
|
218
219
|
frameSampling: cuts # cuts=按镜头切点抽帧(默认)| uniform=固定间隔
|
|
219
220
|
cutThreshold: 27 # 镜头切点阈值(6-40),越高越只抓明显转场
|
|
@@ -236,6 +237,7 @@ host 半零外部依赖(只用 Node 内置 `fetch`/`crypto`),不写锁文
|
|
|
236
237
|
- YouTube 字幕是定时文本,标点/语气词可能有噪声;滚动字幕的重复窗口已做去重。
|
|
237
238
|
- B 站字幕只有「有 Cookie + 该视频确有 AI 字幕」才拿得到,不是每个视频都有。
|
|
238
239
|
- 按镜头抽帧时,画面文字段的「结束时间」按相邻帧间隔的中位数估算(各帧自身的秒数是准的),时间轴粗一点但不会串行;
|
|
240
|
+
- 语音转写与画面分析**并行**:`crossCheck` 默认开启,`POST /transcribe` 一开始就同时下载音频和低清视频、跑 whisper 与抽帧识别,两路结果合并成一份稿子(画面描述带「画面:」前缀)互相印证;显式设 `crossCheck: false` 才回到「先语音、没有语音再看画面」的顺序。
|
|
239
241
|
- 画面文字识别:只在 macOS 可用(依赖系统 Vision 框架);识别质量取决于画面文字是否清晰,滚动弹幕/花哨字体容易出错;抽帧默认按镜头切点(找不到切点时均匀铺满),因此长视频的覆盖密度取决于镜头数量而不是固定间隔。
|
|
240
242
|
- 画面理解:需要当前(或设置里指定的)模型声明 `image` 输入能力;会把拼好的分镜图发给模型(与 OCR 的本机离线不同);默认一张图装 9–24 格,所以按「张」而不是按「帧」计费(比逐帧省很多),单次生成仍要 15–30s;画面里既没字也没可描述内容时仍会得到 `asr-empty`。
|
|
241
243
|
- 本地转写:Whisper 首次加载 574MB 模型约 6–12 秒(每次调用固定成本,长视频占比可忽略);SenseVoice 段级时间戳是按标点聚合的近似值;`threads: 0` 交给引擎自己决定(whisper.cpp 默认 4)。
|
package/client/client.js
CHANGED
|
@@ -64,6 +64,8 @@ window.__ModuleLoader__.load({
|
|
|
64
64
|
'local-whisper': '本地 Whisper', 'local-sensevoice': '本地 SenseVoice',
|
|
65
65
|
'frame-ocr': '画面文字(OCR)',
|
|
66
66
|
'frame-vision': '画面理解',
|
|
67
|
+
'audio+vision': '语音 + 画面',
|
|
68
|
+
'audio+ocr': '语音 + 画面文字',
|
|
67
69
|
};
|
|
68
70
|
const LANG_LABEL = {
|
|
69
71
|
zh: '中文', 'zh-cn': '中文', 'zh-hans': '中文', 'zh-tw': '繁體中文', 'zh-hant': '繁體中文',
|
|
@@ -1944,6 +1946,14 @@ window.__ModuleLoader__.load({
|
|
|
1944
1946
|
}),
|
|
1945
1947
|
h('span', { key: 't' }, '无字幕视频自动转写(解析到没有字幕的视频时自动在本地跑一次转写,长视频可能要等一会儿)'),
|
|
1946
1948
|
]));
|
|
1949
|
+
rows.push(h('label', { key: 'cross', className: 'dvv-check' }, [
|
|
1950
|
+
h('input', {
|
|
1951
|
+
key: 'c', type: 'checkbox', checked: config.crossCheck !== false,
|
|
1952
|
+
onChange: () => onConfig({ crossCheck: config.crossCheck === false }),
|
|
1953
|
+
}),
|
|
1954
|
+
h('span', { key: 't' }, '语音与画面并行分析'),
|
|
1955
|
+
]));
|
|
1956
|
+
rows.push(h('div', { key: 'cross-note', className: 'dvv-note' }, '和语音转写同时跑画面分析,两路结果互相印证(多花一次视觉模型调用)。'));
|
|
1947
1957
|
const ocr = asr.ocr || {};
|
|
1948
1958
|
rows.push(h('label', { key: 'ocr', className: 'dvv-check' }, [
|
|
1949
1959
|
h('input', {
|
package/lib/asr.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* 这样 analyze.js / 面板都不用区分来源。
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
13
|
+
import { mkdir, readFile, rm, unlink, writeFile } from 'node:fs/promises';
|
|
14
14
|
import { homedir } from 'node:os';
|
|
15
15
|
import { join } from 'node:path';
|
|
16
16
|
|
|
@@ -37,6 +37,8 @@ export const ASR_DEFAULTS = {
|
|
|
37
37
|
maxSeconds: 0,
|
|
38
38
|
keepAudio: false,
|
|
39
39
|
autoTranscribe: true,
|
|
40
|
+
// 并行管线:语音转写与画面理解同时跑,结果合并互相印证(默认开)。
|
|
41
|
+
crossCheck: true,
|
|
40
42
|
// 纯无声视频(没有配音也没有字幕)的兜底:抽帧做画面文字识别(OCR)。
|
|
41
43
|
ocrFallback: true,
|
|
42
44
|
ocrIntervalSeconds: 8,
|
|
@@ -80,6 +82,12 @@ export async function readLocalTranscript(docId) {
|
|
|
80
82
|
try {
|
|
81
83
|
const data = JSON.parse(await readFile(transcriptFile(docId), 'utf8'));
|
|
82
84
|
if (data && data.available === true && Array.isArray(data.segments) && data.segments.length > 0) {
|
|
85
|
+
// 旧版本可能把 whisper 的幻听结果写进了缓存:整段重复同一句话时当作没有,
|
|
86
|
+
// 顺手把坏缓存删掉,下次会重新走转写/画面分析。
|
|
87
|
+
if (isDegenerateTranscript(data.segments)) {
|
|
88
|
+
await unlink(transcriptFile(docId)).catch(() => {});
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
83
91
|
return { ...data, cached: true };
|
|
84
92
|
}
|
|
85
93
|
} catch { /* 没有缓存或坏了:当作没转写过 */ }
|
|
@@ -111,6 +119,7 @@ export function asrSettings(settings) {
|
|
|
111
119
|
maxSeconds: Math.min(Math.max(Number(raw.maxSeconds) || 0, 0), 4 * 60 * 60),
|
|
112
120
|
keepAudio: Boolean(raw.keepAudio),
|
|
113
121
|
autoTranscribe: raw.autoTranscribe !== false,
|
|
122
|
+
crossCheck: raw.crossCheck !== false,
|
|
114
123
|
ocrFallback: raw.ocrFallback !== false,
|
|
115
124
|
ocrIntervalSeconds: Math.min(Math.max(Math.round(Number(raw.ocrIntervalSeconds) || 8), 3), 60),
|
|
116
125
|
ocrMaxFrames: Math.min(Math.max(Math.round(Number(raw.ocrMaxFrames) || 120), 20), 240),
|
|
@@ -129,6 +138,31 @@ function formatBytes(bytes) {
|
|
|
129
138
|
}
|
|
130
139
|
|
|
131
140
|
/** 去掉 *music* / [Applause] / (音乐) 这类非语音标注后,真正说出来的字数。 */
|
|
141
|
+
/** 去掉标点/空白,用于「同一句话」比较。 */
|
|
142
|
+
function normalizeSegmentText(text) {
|
|
143
|
+
return String(text ?? '')
|
|
144
|
+
.toLowerCase()
|
|
145
|
+
.replace(/[\s♪♫.,,。!?!?、…"'“”‘’()()\-—–::;;]/g, '');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* whisper 对纯音乐/无对白视频常见的退化输出:同一句话重复几十上百遍。
|
|
150
|
+
* 这种稿子当成「没有语音」处理,才能让画面理解兜底。
|
|
151
|
+
*/
|
|
152
|
+
export function isDegenerateTranscript(segments) {
|
|
153
|
+
const list = Array.isArray(segments) ? segments : [];
|
|
154
|
+
const usable = list.filter((segment) => normalizeSegmentText(segment?.text).length >= 4);
|
|
155
|
+
if (usable.length < 12) return false;
|
|
156
|
+
const counts = new Map();
|
|
157
|
+
for (const segment of usable) {
|
|
158
|
+
const key = normalizeSegmentText(segment.text);
|
|
159
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
160
|
+
}
|
|
161
|
+
let top = 0;
|
|
162
|
+
for (const count of counts.values()) top = Math.max(top, count);
|
|
163
|
+
return top / usable.length >= 0.45;
|
|
164
|
+
}
|
|
165
|
+
|
|
132
166
|
function spokenChars(segments) {
|
|
133
167
|
let chars = 0;
|
|
134
168
|
for (const segment of Array.isArray(segments) ? segments : []) {
|
|
@@ -261,6 +295,7 @@ export async function asrStatus(settings, { fresh = false, ai = null, attachment
|
|
|
261
295
|
maxSeconds: asr.maxSeconds,
|
|
262
296
|
keepAudio: asr.keepAudio,
|
|
263
297
|
autoTranscribe: asr.autoTranscribe,
|
|
298
|
+
crossCheck: asr.crossCheck,
|
|
264
299
|
ocrFallback: asr.ocrFallback,
|
|
265
300
|
ocrIntervalSeconds: asr.ocrIntervalSeconds,
|
|
266
301
|
ocrMaxFrames: asr.ocrMaxFrames,
|
|
@@ -396,6 +431,46 @@ async function transcribeSenseVoice({ engine, wav, workdir, asr, signal }) {
|
|
|
396
431
|
|
|
397
432
|
/* --------------------------------- public ---------------------------------- */
|
|
398
433
|
|
|
434
|
+
const MERGED_VISUAL_LIMIT = 24;
|
|
435
|
+
|
|
436
|
+
/** 把语音稿和画面描述合并成一份带时间戳的稿子:语音段原样、画面段加「画面:」前缀。 */
|
|
437
|
+
export function mergeTranscripts(audioTranscript, visualTranscript) {
|
|
438
|
+
const audioSegments = Array.isArray(audioTranscript?.segments) ? audioTranscript.segments : [];
|
|
439
|
+
const visualSegments = Array.isArray(visualTranscript?.segments) ? visualTranscript.segments : [];
|
|
440
|
+
const segments = audioSegments.map((segment) => ({
|
|
441
|
+
start: Number(segment.start) || 0,
|
|
442
|
+
end: Math.max(Number(segment.end) || 0, Number(segment.start) || 0 + 0.5),
|
|
443
|
+
text: String(segment.text ?? '').trim(),
|
|
444
|
+
}));
|
|
445
|
+
for (const segment of visualSegments.slice(0, MERGED_VISUAL_LIMIT)) {
|
|
446
|
+
const text = String(segment.text ?? '').trim();
|
|
447
|
+
if (!text) continue;
|
|
448
|
+
const start = Number(segment.start) || 0;
|
|
449
|
+
segments.push({ start, end: Math.max(Number(segment.end) || start, start + 0.5), text: `画面:${text}` });
|
|
450
|
+
}
|
|
451
|
+
segments.sort((left, right) => left.start - right.start);
|
|
452
|
+
const chars = segments.reduce((sum, segment) => sum + segment.text.length, 0);
|
|
453
|
+
const visualSource = String(visualTranscript?.source ?? '');
|
|
454
|
+
const ocrOnly = visualSource === 'frame-ocr';
|
|
455
|
+
return {
|
|
456
|
+
available: true,
|
|
457
|
+
source: ocrOnly ? 'audio+ocr' : 'audio+vision',
|
|
458
|
+
engine: `${audioTranscript?.engine ?? 'asr'}+${ocrOnly ? 'vision-ocr' : 'vision'}`,
|
|
459
|
+
engineLabel: `${audioTranscript?.engineLabel ?? '语音识别'} + ${ocrOnly ? '画面文字识别' : '画面理解'}`,
|
|
460
|
+
lang: audioTranscript?.lang ?? 'auto',
|
|
461
|
+
chars,
|
|
462
|
+
segments: segments.slice(0, MAX_TRANSCRIPT_SEGMENTS * 2),
|
|
463
|
+
tookMs: Math.max(Number(audioTranscript?.tookMs) || 0, Number(visualTranscript?.tookMs) || 0),
|
|
464
|
+
crossChecked: true,
|
|
465
|
+
visual: {
|
|
466
|
+
source: visualTranscript?.source ?? '',
|
|
467
|
+
engine: visualTranscript?.engine ?? '',
|
|
468
|
+
engineLabel: visualTranscript?.engineLabel ?? '',
|
|
469
|
+
describedFrames: visualTranscript?.describedFrames ?? null,
|
|
470
|
+
},
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
399
474
|
/**
|
|
400
475
|
* 把视频音轨转成带时间戳的文字稿。调用方负责 SSE 进度帧与缓存更新。
|
|
401
476
|
* 返回的 transcript 与平台字幕同构,额外带 `engine` / `tookMs` 供面板展示。
|
|
@@ -424,12 +499,12 @@ export async function transcribeVideo({ doc, settings, ai = null, attachments =
|
|
|
424
499
|
}
|
|
425
500
|
|
|
426
501
|
const workdir = await createWorkdir();
|
|
502
|
+
const frameWorkdir = await createWorkdir();
|
|
427
503
|
let keep = false;
|
|
428
504
|
try {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
if (engine) {
|
|
505
|
+
/* 音频一路:下载音轨 → 16kHz wav → whisper / SenseVoice。 */
|
|
506
|
+
const runAudio = async () => {
|
|
507
|
+
if (!engine) return { skip: true };
|
|
433
508
|
try {
|
|
434
509
|
onPhase?.('正在下载音频…');
|
|
435
510
|
const audio = await downloadAudio({
|
|
@@ -458,11 +533,13 @@ export async function transcribeVideo({ doc, settings, ai = null, attachments =
|
|
|
458
533
|
const segments = result.segments.slice(0, MAX_TRANSCRIPT_SEGMENTS);
|
|
459
534
|
const chars = segments.reduce((sum, item) => sum + item.text.length, 0);
|
|
460
535
|
// whisper 对纯音乐/无对白视频会输出 "*music*" 之类的占位标签,这不算有语音内容,
|
|
461
|
-
//
|
|
536
|
+
// 或者整段重复同一句话(幻听),都当作没有语音,让画面那一路顶上。
|
|
537
|
+
if (isDegenerateTranscript(segments)) {
|
|
538
|
+
throw new VideoError('asr-empty', '识别结果几乎都是同一句话(多半是纯音乐 / 无对白),改走画面分析');
|
|
539
|
+
}
|
|
462
540
|
if (!segments.length || spokenChars(segments) < 8) {
|
|
463
541
|
throw new VideoError('asr-empty', '没有识别出可用的语音(可能是纯音乐 / 无对白视频)');
|
|
464
542
|
}
|
|
465
|
-
// 长视频只截断返回给模型的量;面板仍能拿到完整段落。
|
|
466
543
|
onProgress?.({ percent: 100, stage: 'transcribe' });
|
|
467
544
|
onPhase?.(`识别完成:${segments.length} 段 / ${chars} 字,用时 ${(tookMs / 1000).toFixed(1)}s`);
|
|
468
545
|
|
|
@@ -471,59 +548,99 @@ export async function transcribeVideo({ doc, settings, ai = null, attachments =
|
|
|
471
548
|
await keepAudioCopy(audio.file).catch(() => '');
|
|
472
549
|
}
|
|
473
550
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
551
|
+
return {
|
|
552
|
+
transcript: {
|
|
553
|
+
available: true,
|
|
554
|
+
source: engine.source,
|
|
555
|
+
engine: engine.id,
|
|
556
|
+
engineLabel: engine.label,
|
|
557
|
+
lang: result.lang,
|
|
558
|
+
chars,
|
|
559
|
+
segments,
|
|
560
|
+
tookMs,
|
|
561
|
+
},
|
|
483
562
|
};
|
|
484
563
|
} catch (error) {
|
|
485
|
-
// 下载不到音轨、或者整段没有语音(纯音乐/无声视频)时,不直接失败:
|
|
486
|
-
// 退到「画面文字识别」,让这类视频也有可用的文字稿。
|
|
487
564
|
if (signal?.aborted || error?.code === 'aborted') throw error;
|
|
488
|
-
|
|
565
|
+
return { error };
|
|
489
566
|
}
|
|
490
|
-
}
|
|
567
|
+
};
|
|
568
|
+
|
|
569
|
+
/* 画面一路:低清视频 → 镜头切点 → 抽帧 → 本机 OCR →(视觉模型描述画面)。 */
|
|
570
|
+
const runFrames = async () => {
|
|
571
|
+
if (!frameUsable) return { skip: true };
|
|
572
|
+
try {
|
|
573
|
+
const transcript = await frameTranscript({
|
|
574
|
+
// 和语音那一路并行时,画面也交给视觉模型看一遍:两路结果互相印证。
|
|
575
|
+
preferVision: asr.frameMode !== 'ocr',
|
|
576
|
+
doc,
|
|
577
|
+
settings,
|
|
578
|
+
workdir: frameWorkdir,
|
|
579
|
+
tools,
|
|
580
|
+
asr,
|
|
581
|
+
ai,
|
|
582
|
+
attachments,
|
|
583
|
+
onPhase: (text) => onPhase?.(`画面:${text}`),
|
|
584
|
+
onProgress: (event) => onProgress?.({ ...event, stage: 'frames' }),
|
|
585
|
+
signal,
|
|
586
|
+
});
|
|
587
|
+
return { transcript };
|
|
588
|
+
} catch (error) {
|
|
589
|
+
if (signal?.aborted || error?.code === 'aborted') throw error;
|
|
590
|
+
return { error };
|
|
591
|
+
}
|
|
592
|
+
};
|
|
491
593
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
594
|
+
const parallel = asr.crossCheck !== false && Boolean(engine) && frameUsable;
|
|
595
|
+
let audioResult = { skip: true };
|
|
596
|
+
let frameResult = { skip: true };
|
|
597
|
+
|
|
598
|
+
if (parallel) {
|
|
599
|
+
onPhase?.('语音转写与画面理解同时进行(两路并行,结果互相印证)…');
|
|
600
|
+
[audioResult, frameResult] = await Promise.all([runAudio(), runFrames()]);
|
|
601
|
+
} else {
|
|
602
|
+
audioResult = await runAudio();
|
|
603
|
+
if (audioResult?.error) {
|
|
604
|
+
const reason = audioResult.error.code === 'asr-empty' ? '没有识别出语音' : '拿不到音频';
|
|
605
|
+
onPhase?.(
|
|
606
|
+
asr.frameMode === 'ocr'
|
|
607
|
+
? `${reason},改为抽取画面文字(视频不会离开这台电脑)…`
|
|
608
|
+
: `${reason},改为识别画面内容…`,
|
|
609
|
+
);
|
|
499
610
|
}
|
|
500
|
-
transcript = await
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
611
|
+
if (!audioResult?.transcript && frameUsable) frameResult = await runFrames();
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
let transcript = audioResult?.transcript ?? null;
|
|
615
|
+
const audioError = audioResult?.error ?? null;
|
|
616
|
+
const visual = frameResult?.transcript ?? null;
|
|
617
|
+
const visualError = frameResult?.error ?? null;
|
|
618
|
+
const visualSegments = Array.isArray(visual?.segments)
|
|
619
|
+
? visual.segments.filter((segment) => String(segment?.text ?? '').trim())
|
|
620
|
+
: [];
|
|
621
|
+
|
|
622
|
+
if (transcript && visualSegments.length) {
|
|
623
|
+
// 两路都有内容:合并成一份稿子,摘要/思维导图/追问都能同时参考「说了什么」和「画面是什么」。
|
|
624
|
+
const audioSegments = Array.isArray(transcript.segments) ? transcript.segments.length : 0;
|
|
625
|
+
const merged = mergeTranscripts(transcript, visual);
|
|
626
|
+
onPhase?.(`语音稿 ${audioSegments} 段 + 画面描述 ${visualSegments.length} 段,已合并互相印证。`);
|
|
627
|
+
transcript = merged;
|
|
628
|
+
} else if (!transcript && visual?.available) {
|
|
629
|
+
transcript = visual;
|
|
512
630
|
}
|
|
513
631
|
|
|
514
632
|
if (!transcript) {
|
|
633
|
+
if (audioError && audioError.code !== 'asr-empty') throw audioError;
|
|
634
|
+
if (visualError) throw visualError;
|
|
515
635
|
if (audioError) throw audioError;
|
|
516
|
-
|
|
517
|
-
throw new VideoError(
|
|
518
|
-
'asr-unavailable',
|
|
519
|
-
`本机还没有可用的语音识别引擎。${hints.length ? `\n${hints.join('\n')}` : ''}`,
|
|
520
|
-
);
|
|
636
|
+
throw new VideoError('asr-failed', '转写失败:语音识别和画面分析都没有拿到内容。');
|
|
521
637
|
}
|
|
522
638
|
|
|
523
639
|
await writeLocalTranscript(doc.docId, transcript);
|
|
524
640
|
return transcript;
|
|
525
641
|
} finally {
|
|
526
642
|
if (!keep) await cleanupWorkdir(workdir);
|
|
643
|
+
await cleanupWorkdir(frameWorkdir).catch(() => {});
|
|
527
644
|
}
|
|
528
645
|
}
|
|
529
646
|
|
package/lib/assets.js
CHANGED
|
@@ -166,6 +166,22 @@ export async function pruneAssets() {
|
|
|
166
166
|
*
|
|
167
167
|
* @returns {Promise<{docId, file, bytes, duration, downloadedAt, cached:boolean}>}
|
|
168
168
|
*/
|
|
169
|
+
const inflight = new Map();
|
|
170
|
+
|
|
171
|
+
function dedupe(key, factory) {
|
|
172
|
+
const pending = inflight.get(key);
|
|
173
|
+
if (pending) return pending;
|
|
174
|
+
const task = (async () => {
|
|
175
|
+
try {
|
|
176
|
+
return await factory();
|
|
177
|
+
} finally {
|
|
178
|
+
inflight.delete(key);
|
|
179
|
+
}
|
|
180
|
+
})();
|
|
181
|
+
inflight.set(key, task);
|
|
182
|
+
return task;
|
|
183
|
+
}
|
|
184
|
+
|
|
169
185
|
export async function ensureVideoAsset({ doc, settings = {}, tools, onPhase, onProgress, signal, force = false } = {}) {
|
|
170
186
|
const docId = String(doc?.docId ?? '');
|
|
171
187
|
if (!docId) throw new VideoError('bad-param', '缺少 docId,无法缓存视频素材。');
|
|
@@ -179,30 +195,32 @@ export async function ensureVideoAsset({ doc, settings = {}, tools, onPhase, onP
|
|
|
179
195
|
return hit;
|
|
180
196
|
}
|
|
181
197
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
198
|
+
return dedupe(`video:${docId}`, async () => {
|
|
199
|
+
const paths = assetPaths(docId);
|
|
200
|
+
await mkdir(paths.dir, { recursive: true });
|
|
201
|
+
onPhase?.('正在下载一份低清视频用来抽帧…');
|
|
202
|
+
const downloaded = await downloadVideo({
|
|
203
|
+
url: doc.url,
|
|
204
|
+
platform: doc.platform,
|
|
205
|
+
tools,
|
|
206
|
+
workdir: paths.dir,
|
|
207
|
+
cookie: settings.bilibiliCookie,
|
|
208
|
+
onProgress,
|
|
209
|
+
signal,
|
|
210
|
+
});
|
|
211
|
+
const meta = {
|
|
212
|
+
docId,
|
|
213
|
+
file: downloaded.file,
|
|
214
|
+
bytes: downloaded.bytes,
|
|
215
|
+
duration: normalizeDuration(doc?.duration),
|
|
216
|
+
platform: doc?.platform ?? '',
|
|
217
|
+
url: doc?.url ?? '',
|
|
218
|
+
downloadedAt: Date.now(),
|
|
219
|
+
};
|
|
220
|
+
await writeJson(paths.meta, meta);
|
|
221
|
+
await pruneAssets().catch(() => {});
|
|
222
|
+
return { ...meta, cached: false };
|
|
193
223
|
});
|
|
194
|
-
const meta = {
|
|
195
|
-
docId,
|
|
196
|
-
file: downloaded.file,
|
|
197
|
-
bytes: downloaded.bytes,
|
|
198
|
-
duration: normalizeDuration(doc?.duration),
|
|
199
|
-
platform: doc?.platform ?? '',
|
|
200
|
-
url: doc?.url ?? '',
|
|
201
|
-
downloadedAt: Date.now(),
|
|
202
|
-
};
|
|
203
|
-
await writeJson(paths.meta, meta);
|
|
204
|
-
await pruneAssets().catch(() => {});
|
|
205
|
-
return { ...meta, cached: false };
|
|
206
224
|
}
|
|
207
225
|
|
|
208
226
|
/**
|
|
@@ -233,23 +251,25 @@ export async function ensureCutsAsset({
|
|
|
233
251
|
return hit;
|
|
234
252
|
}
|
|
235
253
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
254
|
+
return dedupe(`cuts:${docId}:${cutThreshold}`, async () => {
|
|
255
|
+
const scanned = await scanCuts({
|
|
256
|
+
ffmpeg: ffmpegPath,
|
|
257
|
+
video: file,
|
|
258
|
+
duration,
|
|
259
|
+
threshold: cutThreshold,
|
|
260
|
+
signal,
|
|
261
|
+
onProgress,
|
|
262
|
+
});
|
|
263
|
+
const data = {
|
|
264
|
+
cuts: scanned.cuts,
|
|
265
|
+
scanned: scanned.scanned,
|
|
266
|
+
fps: scanned.fps,
|
|
267
|
+
duration: normalizeDuration(duration),
|
|
268
|
+
threshold: cutThreshold,
|
|
269
|
+
};
|
|
270
|
+
await writeCutsAsset(docId, data).catch(() => {});
|
|
271
|
+
return { ...data, docId, cached: false };
|
|
243
272
|
});
|
|
244
|
-
const data = {
|
|
245
|
-
cuts: scanned.cuts,
|
|
246
|
-
scanned: scanned.scanned,
|
|
247
|
-
fps: scanned.fps,
|
|
248
|
-
duration: normalizeDuration(duration),
|
|
249
|
-
threshold: cutThreshold,
|
|
250
|
-
};
|
|
251
|
-
await writeCutsAsset(docId, data).catch(() => {});
|
|
252
|
-
return { ...data, cached: false };
|
|
253
273
|
}
|
|
254
274
|
|
|
255
275
|
/** 清掉某个视频的素材缓存(视频 + 切点)。 */
|
package/lib/frames.js
CHANGED
|
@@ -77,6 +77,7 @@ export async function frameTranscript({
|
|
|
77
77
|
onPhase,
|
|
78
78
|
onProgress,
|
|
79
79
|
signal,
|
|
80
|
+
preferVision = false,
|
|
80
81
|
} = {}) {
|
|
81
82
|
const visionCfg = visionSettings(settings);
|
|
82
83
|
const ocrAvailable = asr.ocrFallback !== false && (await ocrStatus({ fresh: true, prepare: true })).available;
|
|
@@ -136,7 +137,9 @@ export async function frameTranscript({
|
|
|
136
137
|
// 2) 画面理解:给抽样的几帧写描述
|
|
137
138
|
let described = 0;
|
|
138
139
|
const sparse = frames.length === 0 || ocrChars / Math.max(1, frames.length) < SPARSE_TEXT_PER_FRAME;
|
|
139
|
-
|
|
140
|
+
// preferVision:音频里没有语音(纯音乐 / 无对白)时,就算 OCR 从画面纹理里读出一些
|
|
141
|
+
// 噪声文字,也照样让视觉模型描述画面——这类视频的信息几乎全在画面上。
|
|
142
|
+
const shouldDescribe = visionUsable && (visionCfg.frameMode === 'vision' || sparse || preferVision);
|
|
140
143
|
if (shouldDescribe) {
|
|
141
144
|
// 拼图一张最多装 SHEET_MAX_CELLS 格;逐帧发时不受这个限制。
|
|
142
145
|
const frameLimit = visionCfg.visionPack === 'sheet'
|
package/lib/index.js
CHANGED
|
@@ -63,7 +63,7 @@ export const name = 'dsh-video-insight';
|
|
|
63
63
|
/** Route prefix the panel calls; same origin as the GUI, so no CORS preflight. */
|
|
64
64
|
export const API_PREFIX = '/dsh-video/api';
|
|
65
65
|
|
|
66
|
-
const VERSION = '0.8.
|
|
66
|
+
const VERSION = '0.8.4';
|
|
67
67
|
const CREDENTIALS_FILE = join(homedir(), '.dsh', 'video-credentials.json');
|
|
68
68
|
|
|
69
69
|
const DEFAULTS = {
|
|
@@ -100,7 +100,7 @@ function sanitizeAsr(raw, base) {
|
|
|
100
100
|
for (const key of Object.keys(ASR_DEFAULTS)) {
|
|
101
101
|
if (raw[key] === undefined) continue;
|
|
102
102
|
const value = raw[key];
|
|
103
|
-
if (key === 'enabled' || key === 'keepAudio' || key === 'autoTranscribe' || key === 'ocrFallback') out[key] = Boolean(value);
|
|
103
|
+
if (key === 'enabled' || key === 'keepAudio' || key === 'autoTranscribe' || key === 'ocrFallback' || key === 'crossCheck') out[key] = Boolean(value);
|
|
104
104
|
else if (key === 'engine') {
|
|
105
105
|
out.engine = ['auto', 'whisper', 'sensevoice'].includes(String(value)) ? String(value) : 'auto';
|
|
106
106
|
} else if (key === 'threads' || key === 'maxSeconds') {
|
|
@@ -292,7 +292,7 @@ async function attachBestTranscript(doc) {
|
|
|
292
292
|
} catch {
|
|
293
293
|
local = null;
|
|
294
294
|
}
|
|
295
|
-
const isLocalSource = (source) => String(source ?? '').startsWith('local-') || source === 'frame-ocr';
|
|
295
|
+
const isLocalSource = (source) => String(source ?? '').startsWith('local-') || source === 'frame-ocr' || source === 'frame-vision' || source === 'audio+vision';
|
|
296
296
|
const dropStaleWarnings = () => {
|
|
297
297
|
if (Array.isArray(doc.warnings)) {
|
|
298
298
|
doc.warnings = doc.warnings.filter((item) => !/没有可用字幕|只基于标题/.test(String(item)));
|
|
@@ -313,10 +313,12 @@ async function attachBestTranscript(doc) {
|
|
|
313
313
|
} else {
|
|
314
314
|
// 上一步「没有可用字幕」的提示已经过时,换成「用的是本地转写」
|
|
315
315
|
dropStaleWarnings();
|
|
316
|
-
const
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
316
|
+
const localLabel = local.source === 'frame-ocr'
|
|
317
|
+
? '本地画面文字识别稿(OCR)'
|
|
318
|
+
: local.source === 'frame-vision'
|
|
319
|
+
? '本地画面分析稿(语音识别 + 视觉模型看画面)'
|
|
320
|
+
: '本地语音转写稿';
|
|
321
|
+
doc.warnings.push(`这个视频没有平台字幕,已使用${localLabel}。`);
|
|
320
322
|
}
|
|
321
323
|
doc.transcript = local;
|
|
322
324
|
return doc;
|
|
@@ -511,6 +513,8 @@ const TRANSCRIPT_SOURCE_LABELS = {
|
|
|
511
513
|
'local-sensevoice': '本地 SenseVoice 转写',
|
|
512
514
|
'frame-ocr': '画面文字识别(OCR)',
|
|
513
515
|
'frame-vision': '画面理解(视觉模型)',
|
|
516
|
+
'audio+vision': '语音 + 画面(互相印证)',
|
|
517
|
+
'audio+ocr': '语音 + 画面文字(互相印证)',
|
|
514
518
|
};
|
|
515
519
|
|
|
516
520
|
function describeTranscriptForDigest(transcript) {
|
package/lib/vision.js
CHANGED
|
@@ -201,6 +201,52 @@ function parseJsonArray(text) {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
|
|
204
|
+
/**
|
|
205
|
+
* 宽松解析:输出被 maxTokens 截断时,JSON 数组可能不完整。
|
|
206
|
+
* 这里逐个抠出完整的 `{...}` 对象(正确跳过字符串里的括号),能抢救多少算多少。
|
|
207
|
+
*/
|
|
208
|
+
function parseJsonArrayLoose(text) {
|
|
209
|
+
const raw = String(text ?? '');
|
|
210
|
+
const strict = parseJsonArray(raw);
|
|
211
|
+
if (Array.isArray(strict) && strict.length) return strict;
|
|
212
|
+
const items = [];
|
|
213
|
+
let depth = 0;
|
|
214
|
+
let start = -1;
|
|
215
|
+
let inString = false;
|
|
216
|
+
let escaped = false;
|
|
217
|
+
for (let i = 0; i < raw.length; i += 1) {
|
|
218
|
+
const ch = raw[i];
|
|
219
|
+
if (inString) {
|
|
220
|
+
if (escaped) escaped = false;
|
|
221
|
+
else if (ch === '\\') escaped = true;
|
|
222
|
+
else if (ch === '"') inString = false;
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (ch === '"') inString = true;
|
|
226
|
+
else if (ch === '{') {
|
|
227
|
+
if (depth === 0) start = i;
|
|
228
|
+
depth += 1;
|
|
229
|
+
} else if (ch === '}') {
|
|
230
|
+
if (depth > 0) depth -= 1;
|
|
231
|
+
if (depth === 0 && start >= 0) {
|
|
232
|
+
try {
|
|
233
|
+
const value = JSON.parse(raw.slice(start, i + 1));
|
|
234
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) items.push(value);
|
|
235
|
+
} catch { /* 忽略这一个不完整的对象 */ }
|
|
236
|
+
start = -1;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return items.length ? items : null;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** 画面描述的 token 预算:按张数给足,避免 JSON 被截断。 */
|
|
244
|
+
function descriptionBudget(count, requested) {
|
|
245
|
+
const need = 500 + Math.max(1, count) * 170;
|
|
246
|
+
const base = Number(requested) > 0 ? Number(requested) : 0;
|
|
247
|
+
return Math.max(base, Math.min(8000, need));
|
|
248
|
+
}
|
|
249
|
+
|
|
204
250
|
function cleanDescription(value) {
|
|
205
251
|
return String(value ?? '')
|
|
206
252
|
.replace(/^["'“”「」]+|["'“”「」]+$/g, '')
|
|
@@ -260,7 +306,7 @@ export async function describeFrames({
|
|
|
260
306
|
provider: route.provider,
|
|
261
307
|
model: route.model,
|
|
262
308
|
messages: [{ role: 'user', content }],
|
|
263
|
-
maxTokens,
|
|
309
|
+
maxTokens: descriptionBudget(slice.length, maxTokens),
|
|
264
310
|
signal,
|
|
265
311
|
sessionId: route.sessionId || 'dsh-video-panel',
|
|
266
312
|
})) {
|
|
@@ -275,7 +321,7 @@ export async function describeFrames({
|
|
|
275
321
|
throw new VideoError('vision-failed', `画面理解失败:${failure.message ?? '模型返回错误'}`);
|
|
276
322
|
}
|
|
277
323
|
|
|
278
|
-
const parsed =
|
|
324
|
+
const parsed = parseJsonArrayLoose(text);
|
|
279
325
|
if (!Array.isArray(parsed) || !parsed.length) {
|
|
280
326
|
throw new VideoError('vision-failed', `模型没有按格式返回画面描述:${String(text).trim().slice(0, 80)}`);
|
|
281
327
|
}
|
|
@@ -339,7 +385,7 @@ ${table}` },
|
|
|
339
385
|
provider: route.provider,
|
|
340
386
|
model: route.model,
|
|
341
387
|
messages: [{ role: 'user', content }],
|
|
342
|
-
maxTokens,
|
|
388
|
+
maxTokens: descriptionBudget(cells.length, maxTokens),
|
|
343
389
|
signal,
|
|
344
390
|
sessionId: route.sessionId || 'dsh-video-panel',
|
|
345
391
|
})) {
|
|
@@ -354,7 +400,7 @@ ${table}` },
|
|
|
354
400
|
throw new VideoError('vision-failed', `画面理解失败:${failure.message ?? '模型返回错误'}`);
|
|
355
401
|
}
|
|
356
402
|
|
|
357
|
-
const parsed =
|
|
403
|
+
const parsed = parseJsonArrayLoose(text);
|
|
358
404
|
if (!Array.isArray(parsed) || !parsed.length) {
|
|
359
405
|
throw new VideoError('vision-failed', `模型没有按格式返回分镜描述:${String(text).trim().slice(0, 80)}`);
|
|
360
406
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-video-insight",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4",
|
|
4
4
|
"description": "在 DeepSeek Harness 里快速分析视频:左侧栏「视频解析」面板,粘贴 YouTube / 哔哩哔哩链接,自动抓字幕并生成总结、章节、要点、原文与追问;无字幕时可用本机 Whisper / SenseVoice 转写兜底(音频不出本机),纯无声视频还能用画面文字识别(OCR)或视觉模型描述画面;右栏有 AI 思维导图;追问是基于转写稿的 RAG 问答(回答带「依据」时间戳);分析结果按视频缓存,历史记录可回看。",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Jerry (Azerz)",
|