dsh-pf-stream 0.2.4 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +171 -34
- package/README.md +168 -31
- package/client.js +1193 -0
- package/icon.svg +18 -0
- package/lib/asr/catalog.d.ts +74 -0
- package/lib/asr/catalog.js +168 -0
- package/lib/asr/catalog.js.map +1 -0
- package/lib/asr/detect.d.ts +33 -0
- package/lib/asr/detect.js +117 -0
- package/lib/asr/detect.js.map +1 -0
- package/lib/asr/engine.d.ts +43 -0
- package/lib/asr/engine.js +143 -0
- package/lib/asr/engine.js.map +1 -0
- package/lib/asr/install.d.ts +56 -0
- package/lib/asr/install.js +317 -0
- package/lib/asr/install.js.map +1 -0
- package/lib/asr/verify.d.ts +22 -0
- package/lib/asr/verify.js +174 -0
- package/lib/asr/verify.js.map +1 -0
- package/lib/audio/acousticVad.d.ts +45 -0
- package/lib/audio/acousticVad.js +179 -0
- package/lib/audio/acousticVad.js.map +1 -0
- package/lib/audio/asrProviders.d.ts +51 -0
- package/lib/audio/asrProviders.js +251 -0
- package/lib/audio/asrProviders.js.map +1 -0
- package/lib/audio/asrRouter.d.ts +30 -0
- package/lib/audio/asrRouter.js +187 -0
- package/lib/audio/asrRouter.js.map +1 -0
- package/lib/audio/hallucinationGuard.d.ts +18 -0
- package/lib/audio/hallucinationGuard.js +97 -0
- package/lib/audio/hallucinationGuard.js.map +1 -0
- package/lib/audio/sidecar.d.ts +76 -0
- package/lib/audio/sidecar.js +362 -0
- package/lib/audio/sidecar.js.map +1 -0
- package/lib/audio/subtitleExtractor.d.ts +36 -0
- package/lib/audio/subtitleExtractor.js +218 -0
- package/lib/audio/subtitleExtractor.js.map +1 -0
- package/lib/audio/transcriptCache.d.ts +33 -0
- package/lib/audio/transcriptCache.js +126 -0
- package/lib/audio/transcriptCache.js.map +1 -0
- package/lib/audio/types.d.ts +136 -0
- package/lib/audio/types.js +10 -0
- package/lib/audio/types.js.map +1 -0
- package/lib/host/api.d.ts +23 -0
- package/lib/host/api.js +286 -0
- package/lib/host/api.js.map +1 -0
- package/lib/host/runtime.d.ts +167 -0
- package/lib/host/runtime.js +335 -0
- package/lib/host/runtime.js.map +1 -0
- package/lib/index.d.ts +229 -1
- package/lib/index.js +103 -2
- package/lib/index.js.map +1 -1
- package/lib/session/state.js +4 -0
- package/lib/session/state.js.map +1 -1
- package/lib/stage0/decoder.d.ts +1 -9
- package/lib/stage0/decoder.js +15 -45
- package/lib/stage0/decoder.js.map +1 -1
- package/lib/stage2/stateMachine.d.ts +4 -1
- package/lib/stage2/stateMachine.js +7 -1
- package/lib/stage2/stateMachine.js.map +1 -1
- package/lib/stage3/deterministicRouter.js +3 -3
- package/lib/stage3/deterministicRouter.js.map +1 -1
- package/lib/stage3/streamSampler.d.ts +9 -2
- package/lib/stage3/streamSampler.js +37 -16
- package/lib/stage3/streamSampler.js.map +1 -1
- package/lib/stage6/history.d.ts +2 -1
- package/lib/stage6/history.js +4 -1
- package/lib/stage6/history.js.map +1 -1
- package/lib/tool/handler.d.ts +9 -0
- package/lib/tool/handler.js +33 -3
- package/lib/tool/handler.js.map +1 -1
- package/lib/tool/schema.js +15 -2
- package/lib/tool/schema.js.map +1 -1
- package/lib/tool/streamCall.d.ts +11 -0
- package/lib/tool/streamCall.js +122 -9
- package/lib/tool/streamCall.js.map +1 -1
- package/lib/types.d.ts +9 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +23 -2
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/stage6/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"history.js","sourceRoot":"","sources":["../../src/stage6/history.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAevD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,iDAAiD;AACjD,MAAM,UAAU,aAAa,CAC3B,IAAuB,EACvB,OAAqB,EACrB,SAA2B,EAC3B,qBAAwD;IAExD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,aAAa,GAAG,qBAAqB,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;YACxB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,kBAAkB,EAAE,CAAC,CAAC,kBAAkB;YACxC,aAAa,EAAE,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;SACnG,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE;gBACzC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS;gBAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ;gBAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,CAAC;gBACtF,gBAAgB,EAAE,CAAC,CAAC,WAAW;gBAC/B,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE;gBAC3C,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,KAAK,mBAAmB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU;gBAClF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,SAAS;gBAC9C,WAAW,EAAE,CAAC,CAAC,kBAAkB,IAAI,SAAS;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,kBAAkB,CAAC,IAAiB,EAAE,eAAuB;IAC3E,MAAM,MAAM,GAAiB;QAC3B,QAAQ,EAAE,IAAI,CAAC,WAAW;QAC1B,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QACtB,UAAU,EAAE,mBAAmB;QAC/B,WAAW,EAAE,QAAQ;QACrB,WAAW,EAAE,iBAAiB;QAC9B,qBAAqB,EAAE,IAAI,CAAC,cAAc;QAC1C,YAAY,EAAE,uBAAuB;QACrC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,WAAW,EAAE,IAAI;QACjB,eAAe;QACf,aAAa,EAAE,EAAE;QACjB,KAAK,EAAE;YACL;gBACE,QAAQ,EAAE,IAAI,CAAC,SAAS;gBACxB,WAAW,EAAE,IAAI,CAAC,KAAK;gBACvB,KAAK,EAAE,OAAO;gBACd,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC;gBAC7B,gBAAgB,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC;gBAClC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC;SACF;KACF,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,CAAC;AAC7G,CAAC;AAED,iCAAiC;AACjC,MAAM,UAAU,eAAe,CAAC,IAAiB;IAC/C,OAAO,YAAY,IAAI,CAAC,WAAW,YAAY,IAAI,CAAC,cAAc,eAAe,IAAI,CAAC,SAAS,OAAO,IAAI,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC;AAC9K,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAAoB,EACpB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;IAC7F,4CAA4C;IAC5C,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC;AACtC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAuB,EACvB,OAAqB,EACrB,OAAyE,EACzE,aAA2B,EAC3B,eAAuB;IAEvB,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,UAAU,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAElC,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACrF,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAE1F,oCAAoC;QACpC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,SAAS,CAAE,CAAC;YAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC9E,KAAK,CAAC,IAAI,CACR,IAAI,IAAI;gBACN,WAAW,EAAE,aAAa,EAAE;gBAC5B,cAAc,EAAE,CAAC,CAAC,QAAQ;gBAC1B,SAAS,EAAE,CAAC,CAAC,SAAU;gBACvB,aAAa,EAAE,CAAC,CAAC,aAAc;gBAC/B,KAAK,EAAE,KAAK,CAAC,WAAW;gBACxB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,KAAK,EAAE,KAAK,CAAC,WAAY;gBACzB,WAAW,EAAE,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,SAAU,CAAC,EAAE,WAAW;gBAC9D,aAAa,EAAE,KAAK,CAAC,aAAa;aACnC,CACF,CAAC;QACJ,CAAC;QAED,uCAAuC;QACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CACzE,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI;gBAAE,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAChF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CACvE,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QAChD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,cAAc,CAAC,KAAmB;IAChD,MAAM,MAAM,GAAiB,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChF,OAAO;QACL,MAAM;QACN,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;KAC7C,CAAC;AACJ,CAAC;AAED,mDAAmD;AACnD,MAAM,UAAU,eAAe,CAC7B,IAAuB,EACvB,eAAuB;IAEvB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IAC1F,iCAAiC;IACjC,MAAM,GAAG,GAAG,IAAI;SACb,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;SAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;AACvB,CAAC;AAED,4BAA4B;AAC5B,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,OAAO,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACtC,CAAC"}
|
package/lib/tool/handler.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ import type { IntentDirective, LegacyInspectionParams, ProbeResult } from '../ty
|
|
|
9
9
|
import type { InspectionResult } from '../stage3/deterministicRouter.js';
|
|
10
10
|
import type { PacketArtifact } from '../stage4/canvasPacker.js';
|
|
11
11
|
import type { StreamCallResult } from './streamCall.js';
|
|
12
|
+
import type { AudioSettings, SubtitleTimeline } from '../audio/types.js';
|
|
12
13
|
export { PfStreamError } from '../errors.js';
|
|
13
14
|
export interface ToolCallArgs {
|
|
14
15
|
video_path: string;
|
|
@@ -53,12 +54,20 @@ export type ProbeFn = (videoPath: string, signal: AbortSignal | undefined) => Pr
|
|
|
53
54
|
export type InspectFn = (videoPath: string, probe: ProbeResult, intent: IntentDirective, signal: AbortSignal | undefined) => Promise<InspectionResult>;
|
|
54
55
|
/** 流式通道实现，由插件入口注入。 */
|
|
55
56
|
export type StreamFn = (videoPath: string, probe: ProbeResult, intent: IntentDirective, sessionId: string | undefined, signal: AbortSignal | undefined) => Promise<StreamCallResult>;
|
|
57
|
+
export interface AudioToolDeps {
|
|
58
|
+
/** 按调用意图解析音频设置（env > Config > intent 覆盖）。 */
|
|
59
|
+
resolveSettings(intent: IntentDirective | undefined): AudioSettings;
|
|
60
|
+
/** 软失败的字幕时间线加载（无字幕/失败返回 null，不抛错）。 */
|
|
61
|
+
subtitlesFor(videoPath: string, probe: ProbeResult, signal: AbortSignal | undefined): Promise<SubtitleTimeline | null>;
|
|
62
|
+
}
|
|
56
63
|
export interface ToolDeps {
|
|
57
64
|
probe: ProbeFn;
|
|
58
65
|
inspect?: InspectFn | undefined;
|
|
59
66
|
stream?: StreamFn | undefined;
|
|
60
67
|
/** 宿主附件回传（可选：缺省时只返回画布元数据）。 */
|
|
61
68
|
attachImages?: AttachImagesFn | undefined;
|
|
69
|
+
/** 音频台词侧车（可选：缺省时 probe 不带字幕导航）。 */
|
|
70
|
+
audio?: AudioToolDeps | undefined;
|
|
62
71
|
/** 入口适配器钳制上限（缺省用插件默认配置值）。 */
|
|
63
72
|
limits?: {
|
|
64
73
|
maxPackets: number;
|
package/lib/tool/handler.js
CHANGED
|
@@ -45,7 +45,23 @@ export async function handleToolCall(args, exec, deps) {
|
|
|
45
45
|
});
|
|
46
46
|
const action = intent.action ?? 'pf_stream';
|
|
47
47
|
if (action === 'probe') {
|
|
48
|
-
|
|
48
|
+
const result = { action: 'probe', probe: probeResult };
|
|
49
|
+
const settings = deps.audio?.resolveSettings(intent);
|
|
50
|
+
if (settings?.extractSubtitles) {
|
|
51
|
+
const timeline = await deps.audio.subtitlesFor(args.video_path, probeResult, exec?.signal);
|
|
52
|
+
if (timeline) {
|
|
53
|
+
result.subtitles = {
|
|
54
|
+
source: timeline.status,
|
|
55
|
+
language: timeline.language,
|
|
56
|
+
segments: timeline.segments.map((s) => ({
|
|
57
|
+
startMs: s.startMs,
|
|
58
|
+
endMs: s.endMs,
|
|
59
|
+
text: s.text,
|
|
60
|
+
})),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return result;
|
|
49
65
|
}
|
|
50
66
|
if (action === 'timestamps' || action === 'uniform') {
|
|
51
67
|
if (!deps.inspect) {
|
|
@@ -66,13 +82,27 @@ export async function handleToolCall(args, exec, deps) {
|
|
|
66
82
|
}
|
|
67
83
|
const stream = await deps.stream(args.video_path, probeResult, intent, resolveSessionId(exec), exec?.signal);
|
|
68
84
|
const packets = await withPacketImages(stream.packets, deps.attachImages);
|
|
85
|
+
const audioByPacket = stream.audioByPacket;
|
|
86
|
+
const packetsWithAudio = audioByPacket
|
|
87
|
+
? packets.map((p) => {
|
|
88
|
+
const mount = audioByPacket[p.packetId];
|
|
89
|
+
return mount ? { ...p, audio: mount } : p;
|
|
90
|
+
})
|
|
91
|
+
: packets;
|
|
92
|
+
// 缺少转写引擎时给一句可行动的说明：结果里只有状态码，模型不知道该去哪配置。
|
|
93
|
+
const hintLines = [...stream.hintLines];
|
|
94
|
+
if (stream.audio?.status === 'no_subtitle_asr_unconfigured') {
|
|
95
|
+
hintLines.push('未生成转写文本：该视频没有字幕，且未配置转写引擎。' +
|
|
96
|
+
'可在左侧栏「插件」→「视频理解」中下载本地转写模型（约 190 MB），或配置云端服务商密钥。');
|
|
97
|
+
}
|
|
69
98
|
return {
|
|
70
99
|
action: 'pf_stream',
|
|
71
100
|
probe: probeResult,
|
|
72
|
-
packets,
|
|
73
|
-
hintLines
|
|
101
|
+
packets: packetsWithAudio,
|
|
102
|
+
hintLines,
|
|
74
103
|
heartbeats: stream.heartbeats,
|
|
75
104
|
telemetry: stream.telemetry,
|
|
105
|
+
audio: stream.audio,
|
|
76
106
|
session: { intentId: stream.sessionIntentId },
|
|
77
107
|
};
|
|
78
108
|
}
|
package/lib/tool/handler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handler.js","sourceRoot":"","sources":["../../src/tool/handler.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"handler.js","sourceRoot":"","sources":["../../src/tool/handler.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAMpD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAmB7C,mEAAmE;AACnE,MAAM,UAAU,gBAAgB,CAAC,IAA0B;IACzD,OAAO,IAAI,EAAE,SAAS,IAAI,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;AACrD,CAAC;AAgED,MAAM,UAAU,aAAa,CAAC,IAAkB;IAC9C,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IACnD,IAAI,IAAI,CAAC,MAAM,EAAE,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI;QAAE,OAAO,OAAO,CAAC;IACjF,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,wCAAwC;AACxC,KAAK,UAAU,gBAAgB,CAC7B,SAA2B,EAC3B,YAAwC;IAExC,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACvC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;IACxE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAkB,EAClB,IAA0B,EAC1B,IAAc;IAEd,8CAA8C;IAC9C,+CAA+C;IAC/C,sCAAsC;IACtC,IAAI,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,aAAa,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,IAA0C,EAAE;QAC3E,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,YAAY,EAAE,WAAW,CAAC,YAAY;QACtC,aAAa,EAAE,WAAW,CAAC,aAAa;QACxC,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,IAAI,EAAE;QACzC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,eAAe,IAAI,KAAK;QACtD,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,GAAG;KACzC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,WAAW,CAAC;IAE5C,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,MAAM,GAAe,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,QAAQ,EAAE,gBAAgB,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAM,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YAC5F,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,SAAS,GAAG;oBACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBACtC,OAAO,EAAE,CAAC,CAAC,OAAO;wBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,IAAI,EAAE,CAAC,CAAC,IAAI;qBACb,CAAC,CAAC;iBACJ,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,aAAa,CAAC,6BAA6B,EAAE,WAAW,MAAM,SAAS,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1F,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9E,OAAO;YACL,MAAM;YACN,KAAK,EAAE,WAAW;YAClB,OAAO;YACP,SAAS,EAAE,UAAU,CAAC,SAAS;SAChC,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,aAAa,CAAC,6BAA6B,EAAE,WAAW,MAAM,SAAS,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7G,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1E,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC3C,MAAM,gBAAgB,GAAG,aAAa;YACpC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAChB,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,QAAkB,CAAC,CAAC;gBAClD,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5C,CAAC,CAAC;YACJ,CAAC,CAAC,OAAO,CAAC;QACZ,wCAAwC;QACxC,MAAM,SAAS,GAAG,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,MAAM,CAAC,KAAK,EAAE,MAAM,KAAK,8BAA8B,EAAE,CAAC;YAC5D,SAAS,CAAC,IAAI,CACZ,2BAA2B;gBACzB,iDAAiD,CACpD,CAAC;QACJ,CAAC;QACD,OAAO;YACL,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,gBAAgB;YACzB,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE;SAC9C,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,aAAa,CAAC,6BAA6B,EAAE,WAAW,MAAM,SAAS,CAAC,CAAC;AACrF,CAAC"}
|
package/lib/tool/schema.js
CHANGED
|
@@ -17,7 +17,8 @@ export const INTENT_DESCRIPTION = '统一采样与检视指令：' +
|
|
|
17
17
|
'2) 定点时间戳高清检视传 action=\'timestamps\' 与 timestamps_ms；' +
|
|
18
18
|
'3) 均匀步长抽帧传 action=\'uniform\' 与 interval_ms/frame_count；' +
|
|
19
19
|
"4) 仅探测视频元数据传 action='probe'；" +
|
|
20
|
-
'5) 局部区域聚焦或放大传归一化 roi_hint
|
|
20
|
+
'5) 局部区域聚焦或放大传归一化 roi_hint；' +
|
|
21
|
+
"6) 台词与字幕传 audio（extract_subtitles 全片字幕导航，language 指定 ASR 语言）。";
|
|
21
22
|
export const intentValueSchema = {
|
|
22
23
|
type: 'object',
|
|
23
24
|
description: INTENT_DESCRIPTION,
|
|
@@ -44,6 +45,17 @@ export const intentValueSchema = {
|
|
|
44
45
|
max_packets: { type: 'number', description: '本次调用最多返回画布包数' },
|
|
45
46
|
max_tokens_budget: { type: 'number', description: '本次调用视觉 Token 预算上限' },
|
|
46
47
|
reset: { type: 'boolean', description: '重置会话意图为默认基准' },
|
|
48
|
+
audio: {
|
|
49
|
+
type: 'object',
|
|
50
|
+
description: '音频台词侧车：字幕/ASR 台词随画布包挂载（可用性取决于视频与配置）',
|
|
51
|
+
properties: {
|
|
52
|
+
mode: { type: 'string', enum: ['auto', 'off'], description: 'off 关闭音频侧车' },
|
|
53
|
+
language: { type: 'string', description: 'ASR 语言提示（如 zh、en）' },
|
|
54
|
+
extract_subtitles: { type: 'boolean', description: 'probe 返回全片字幕导航' },
|
|
55
|
+
assist_focus: { type: 'boolean', description: '语音停顿引导额外 F 定格' },
|
|
56
|
+
},
|
|
57
|
+
additionalProperties: false,
|
|
58
|
+
},
|
|
47
59
|
},
|
|
48
60
|
additionalProperties: false,
|
|
49
61
|
};
|
|
@@ -82,7 +94,8 @@ export const TOOL_DESCRIPTION = 'Look at a local video: adaptive dual-scale stre
|
|
|
82
94
|
'and roi_hint. Pass pts_window (ms) to inspect one segment; use action="uniform" with interval_ms ' +
|
|
83
95
|
'or frame_count for an even overview. Repeat streaming calls resume from the last position, so a ' +
|
|
84
96
|
'long video can be viewed in segments. Cite packet IDs, unit IDs, and timestamps from the returned ' +
|
|
85
|
-
'hint lines as evidence.'
|
|
97
|
+
'hint lines as evidence. When audio context is available, transcript segments are mounted on packets as ' +
|
|
98
|
+
'"Audio" hint lines; cite them with their timestamps for spoken content.';
|
|
86
99
|
/** 输出 schema：随各里程碑交付逐步收紧，当前为对象根。 */
|
|
87
100
|
export const toolOutputSchema = {
|
|
88
101
|
type: 'object',
|
package/lib/tool/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/tool/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAC;AAEzC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,YAAY;IACZ,wGAAwG;IACxG,sDAAsD;IACtD,0DAA0D;IAC1D,8BAA8B;IAC9B,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/tool/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,CAAC,MAAM,SAAS,GAAG,eAAe,CAAC;AAEzC,MAAM,CAAC,MAAM,kBAAkB,GAC7B,YAAY;IACZ,wGAAwG;IACxG,sDAAsD;IACtD,0DAA0D;IAC1D,8BAA8B;IAC9B,4BAA4B;IAC5B,+DAA+D,CAAC;AAsBlE,MAAM,CAAC,MAAM,iBAAiB,GAAgB;IAC5C,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,kBAAkB;IAC/B,UAAU,EAAE;QACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE;QACjF,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE;QAC/D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE;QACjE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,iCAAiC,EAAE;QACxG,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE;QAC1F,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,EAAE;QAC1D,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE;QACzD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACrB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACrB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACrB,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aACtB;YACD,QAAQ,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;YAC9B,WAAW,EAAE,mBAAmB;SACjC;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACpE,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;QAC5D,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;QACvE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE;QACtD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,qCAAqC;YAClD,UAAU,EAAE;gBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,WAAW,EAAE,YAAY,EAAE;gBAC1E,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;gBAC9D,iBAAiB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,gBAAgB,EAAE;gBACrE,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE;aAChE;YACD,oBAAoB,EAAE,KAAK;SAC5B;KACF;IACD,oBAAoB,EAAE,KAAK;CAC5B,CAAC;AAEF,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAgC;IAC7D,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6CAA6C,EAAE;IAC1F,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mCAAmC,EAAE;IAC9E,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,yBAAyB,EAAE;IAChG,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,gCAAgC,EAAE;IACvG,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,EAAE;IAC7D,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4BAA4B,EAAE;IACvE,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0BAA0B,EAAE;IACvE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;IAC5E,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,6BAA6B,EAAE;CAC5E,CAAC;AAEF;kDACkD;AAClD,MAAM,CAAC,MAAM,cAAc,GAAwB;IACjD,IAAI,EAAE,QAAQ;IACd,UAAU,EAAE;QACV,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;QAC3D,MAAM,EAAE,iBAAiB;QACzB,GAAG,kBAAkB;KACtB;IACD,QAAQ,EAAE,CAAC,YAAY,CAAC;IACxB,oBAAoB,EAAE,KAAK;CAC5B,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAC3B,oGAAoG;IACpG,wGAAwG;IACxG,wGAAwG;IACxG,qGAAqG;IACrG,6FAA6F;IAC7F,2DAA2D;IAC3D,kGAAkG;IAClG,mGAAmG;IACnG,mGAAmG;IACnG,kGAAkG;IAClG,oGAAoG;IACpG,yGAAyG;IACzG,yEAAyE,CAAC;AAE5E,oCAAoC;AACpC,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,IAAI,EAAE,QAAQ;CACN,CAAC"}
|
package/lib/tool/streamCall.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { type StreamRunResult } from '../stage3/streamSampler.js';
|
|
|
8
8
|
import { type BundleKind } from '../stage6/precheck.js';
|
|
9
9
|
import { baselinesFromIntent } from '../stage5/intent.js';
|
|
10
10
|
import type { IntentDirective, ProbeResult } from '../types.js';
|
|
11
|
+
import type { AudioContextSummary, AudioSettings, PacketAudioMount } from '../audio/types.js';
|
|
11
12
|
export interface StreamCallOptions {
|
|
12
13
|
ffmpegPath: string;
|
|
13
14
|
/** 末帧探测与定点索引用的 ffprobe；缺省由 ffmpegPath 同目录派生。 */
|
|
@@ -25,6 +26,12 @@ export interface StreamCallOptions {
|
|
|
25
26
|
imageCountTiersEnabled: boolean;
|
|
26
27
|
/** 单图视觉 Token 上限（meteringProfile 口径，缺省不校验）。 */
|
|
27
28
|
maxTokensPerImage?: number | undefined;
|
|
29
|
+
/** 音频台词侧车（缺省关闭；零侵入视觉主链路）。 */
|
|
30
|
+
audio?: {
|
|
31
|
+
settings: AudioSettings;
|
|
32
|
+
ffmpegPath: string;
|
|
33
|
+
tempDir?: string | undefined;
|
|
34
|
+
} | undefined;
|
|
28
35
|
signal?: AbortSignal | undefined;
|
|
29
36
|
}
|
|
30
37
|
/**
|
|
@@ -57,6 +64,10 @@ export interface StreamCallResult extends Omit<StreamRunResult, 'telemetry'> {
|
|
|
57
64
|
hotRefCount: number;
|
|
58
65
|
sessionIntentId: string;
|
|
59
66
|
appliedIntentId: string;
|
|
67
|
+
/** 音频侧车上下文摘要（无音频/关闭时 status='no_audio'|'disabled'）。 */
|
|
68
|
+
audio: AudioContextSummary;
|
|
69
|
+
/** 按包挂载的台词（仅含实际有台词的包）。 */
|
|
70
|
+
audioByPacket: Record<number, PacketAudioMount>;
|
|
60
71
|
}
|
|
61
72
|
export declare function runStreamCall(videoPath: string, probe: ProbeResult, intent: IntentDirective, sessionId: string, options: StreamCallOptions): Promise<StreamCallResult>;
|
|
62
73
|
/** 供 handler 输出遥测的基准参数（不触发解码）。 */
|
package/lib/tool/streamCall.js
CHANGED
|
@@ -10,6 +10,8 @@ import { coldRefHintLine, compactEpoch, HOT_WINDOW, planHistoryRefs, recordPacke
|
|
|
10
10
|
import { precheckBundle, } from '../stage6/precheck.js';
|
|
11
11
|
import { baselinesFromIntent, mergeSessionIntent, sessionIntentChanged } from '../stage5/intent.js';
|
|
12
12
|
import { fingerprintOf, getSessionState, getVideoSlot } from '../session/state.js';
|
|
13
|
+
import { audioHintLine, coldRefAudioHintLine, mountSegments, runAudioSidecar, } from '../audio/sidecar.js';
|
|
14
|
+
import { scanVad } from '../audio/acousticVad.js';
|
|
13
15
|
/**
|
|
14
16
|
* 会话级串行化：同一 sessionId 的流式调用按 promise 链排队执行。
|
|
15
17
|
*
|
|
@@ -47,13 +49,48 @@ async function runStreamCallLocked(videoPath, probe, intent, sessionId, options)
|
|
|
47
49
|
}
|
|
48
50
|
merged.intentId = `intent_${session.intentSeq}`;
|
|
49
51
|
session.intent = merged;
|
|
52
|
+
// 音频侧车（零侵入）：assistFocus 需要在解码前拿到语音岛 → 先做一次
|
|
53
|
+
// VAD 预扫（本地 ffmpeg，窗口量级毫秒级），全量侧车与解码并行。
|
|
54
|
+
const audio = options.audio;
|
|
55
|
+
const windowMs = intent.pts_window
|
|
56
|
+
? Math.max(1, intent.pts_window[1] - intent.pts_window[0])
|
|
57
|
+
: options.defaultStreamWindowMs;
|
|
58
|
+
const lastPts = probe.lastPtsMs;
|
|
59
|
+
const windowStart = Math.min(Math.max(0, intent.pts_window?.[0] ?? slot.cursor.lastConsumedPtsMs), lastPts);
|
|
60
|
+
const windowEnd = Math.min(windowStart + windowMs, lastPts);
|
|
61
|
+
const audioWindow = windowEnd > windowStart ? [windowStart, windowEnd] : null;
|
|
62
|
+
let preVad = null;
|
|
63
|
+
const audioEnabled = audio !== undefined &&
|
|
64
|
+
audio.settings.mode !== 'off' &&
|
|
65
|
+
audioWindow !== null &&
|
|
66
|
+
probe.audioStreams.length > 0;
|
|
67
|
+
if (audioEnabled && audio.settings.assistFocus) {
|
|
68
|
+
try {
|
|
69
|
+
preVad = await scanVad(videoPath, audioWindow, {
|
|
70
|
+
ffmpegPath: audio.ffmpegPath,
|
|
71
|
+
signal: options.signal,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
preVad = null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const sidecarPromise = audioEnabled && audio
|
|
79
|
+
? runAudioSidecar({
|
|
80
|
+
videoPath,
|
|
81
|
+
probe,
|
|
82
|
+
ffmpegPath: audio.ffmpegPath,
|
|
83
|
+
settings: audio.settings,
|
|
84
|
+
tempDir: audio.tempDir,
|
|
85
|
+
signal: options.signal,
|
|
86
|
+
precomputedVad: preVad,
|
|
87
|
+
}, audioWindow, Date.now() + audio.settings.graceMs).catch(() => null)
|
|
88
|
+
: null;
|
|
50
89
|
const result = await runStreamPass(videoPath, probe, merged, slot.cursor, {
|
|
51
90
|
ffmpegPath: options.ffmpegPath,
|
|
52
91
|
ffprobePath: options.ffprobePath,
|
|
53
92
|
renderVisualId: options.renderVisualId,
|
|
54
|
-
windowMs
|
|
55
|
-
? Math.max(1, intent.pts_window[1] - intent.pts_window[0])
|
|
56
|
-
: options.defaultStreamWindowMs,
|
|
93
|
+
windowMs,
|
|
57
94
|
windowStartMs: intent.pts_window?.[0],
|
|
58
95
|
streamWorkMaxEdge: options.streamWorkMaxEdge,
|
|
59
96
|
maxPackets: intent.max_packets ?? options.maxPacketsPerCall,
|
|
@@ -62,16 +99,44 @@ async function runStreamCallLocked(videoPath, probe, intent, sessionId, options)
|
|
|
62
99
|
packetIdStart: session.packetSeq + 1,
|
|
63
100
|
globalIdStart: session.globalUnitSeq + 1,
|
|
64
101
|
heartbeatSeqStart: session.heartbeatSeq + 1,
|
|
102
|
+
audioAssist: audioEnabled && preVad !== null
|
|
103
|
+
? { islands: preVad.islands, speechRate: null }
|
|
104
|
+
: undefined,
|
|
65
105
|
signal: options.signal,
|
|
66
106
|
});
|
|
67
107
|
slot.cursor = result.nextCursor;
|
|
68
108
|
session.packetSeq = result.counters.packetSeq;
|
|
69
109
|
session.globalUnitSeq = result.counters.globalUnitSeq;
|
|
70
110
|
session.heartbeatSeq = result.counters.heartbeatSeq;
|
|
111
|
+
// 音频侧车收尾（软截止内完成则挂载；超时/失败折叠为非 ok 状态）
|
|
112
|
+
const sidecar = sidecarPromise !== null ? await sidecarPromise : null;
|
|
113
|
+
const newPacketWindows = result.packets.map((artifact) => {
|
|
114
|
+
const units = artifact.packet.units;
|
|
115
|
+
const times = units.map((u) => u.timestampMs);
|
|
116
|
+
return {
|
|
117
|
+
packetId: artifact.packet.packetId,
|
|
118
|
+
startMs: times.length > 0 ? Math.min(...times) : 0,
|
|
119
|
+
endMs: times.length > 0 ? Math.max(...times) : 0,
|
|
120
|
+
unitIds: units.map((u) => ({ globalId: u.globalId, timestampMs: u.timestampMs })),
|
|
121
|
+
};
|
|
122
|
+
});
|
|
123
|
+
const newMounts = sidecar !== null && sidecar.segments.length > 0 && result.packets.length > 0
|
|
124
|
+
? mountSegments(newPacketWindows, sidecar.segments, sidecar.context.status, sidecar.context.source, sidecar.context.language)
|
|
125
|
+
: new Map();
|
|
126
|
+
const audioSegmentsByPacket = new Map();
|
|
127
|
+
for (const [pid, mount] of newMounts) {
|
|
128
|
+
audioSegmentsByPacket.set(pid, mount.segments.map((s) => ({
|
|
129
|
+
segId: s.segId,
|
|
130
|
+
startMs: s.startMs,
|
|
131
|
+
endMs: s.endMs,
|
|
132
|
+
text: s.text,
|
|
133
|
+
source: s.source,
|
|
134
|
+
})));
|
|
135
|
+
}
|
|
71
136
|
// Stage 6：热区引用取自本次记录之前的历史环尾部（热区=上一次调用的包），
|
|
72
137
|
// 再记录新包 → Epoch 收敛 → 冷区规划 → 预检门控
|
|
73
138
|
const hot = planHistoryRefs(slot, merged.intentId).hot;
|
|
74
|
-
recordPackets(slot, session, result.packets);
|
|
139
|
+
recordPackets(slot, session, result.packets, audioSegmentsByPacket);
|
|
75
140
|
compactEpoch(slot, session, {
|
|
76
141
|
hotWindow: HOT_WINDOW,
|
|
77
142
|
maxColdRefs: options.maxColdRefs,
|
|
@@ -107,19 +172,65 @@ async function runStreamCallLocked(videoPath, probe, intent, sessionId, options)
|
|
|
107
172
|
for (const it of precheck.kept) {
|
|
108
173
|
if (it.kind === 'cold') {
|
|
109
174
|
const plan = slot.epoch.coldRefs.find((c) => c.refPacketId === it.artifact.packet.packetId);
|
|
110
|
-
if (plan)
|
|
175
|
+
if (plan) {
|
|
111
176
|
keptHintLines.push(coldRefHintLine(plan));
|
|
177
|
+
if (plan.audioSegments !== undefined && plan.audioSegments.length > 0) {
|
|
178
|
+
keptHintLines.push(coldRefAudioHintLine(plan.sourcePacketId, plan.audioSegments.length));
|
|
179
|
+
}
|
|
180
|
+
}
|
|
112
181
|
}
|
|
113
182
|
else if (it.kind === 'hot') {
|
|
114
183
|
keptHintLines.push(timelineHintLine(it.artifact.packet, merged.roiHint ?? null));
|
|
115
184
|
}
|
|
116
185
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
186
|
+
// 增量提示行与 result.packets 同序 1:1（采样器每次 push 相邻执行）；
|
|
187
|
+
// 仅保留预检后仍在输出的增量包提示行
|
|
188
|
+
result.packets.forEach((artifact, i) => {
|
|
189
|
+
const line = result.hintLines[i];
|
|
190
|
+
if (line !== undefined && newIdSet.has(artifact.packet.packetId))
|
|
121
191
|
keptHintLines.push(line);
|
|
192
|
+
});
|
|
193
|
+
// 台词行紧跟对应增量包提示行（时序）
|
|
194
|
+
for (const it of precheck.kept) {
|
|
195
|
+
if (it.kind !== 'new')
|
|
196
|
+
continue;
|
|
197
|
+
const mount = newMounts.get(it.artifact.packet.packetId);
|
|
198
|
+
if (mount)
|
|
199
|
+
keptHintLines.push(audioHintLine(it.artifact.packet.packetId, mount));
|
|
200
|
+
}
|
|
201
|
+
// 冷区引用的台词挂载（Epoch 内 100% 保留）
|
|
202
|
+
const audioByPacket = {};
|
|
203
|
+
for (const [pid, mount] of newMounts) {
|
|
204
|
+
if (newIdSet.has(pid))
|
|
205
|
+
audioByPacket[pid] = mount;
|
|
206
|
+
}
|
|
207
|
+
for (const it of precheck.kept) {
|
|
208
|
+
if (it.kind !== 'cold')
|
|
209
|
+
continue;
|
|
210
|
+
const plan = slot.epoch.coldRefs.find((c) => c.refPacketId === it.artifact.packet.packetId);
|
|
211
|
+
if (plan !== undefined && plan.audioSegments !== undefined && plan.audioSegments.length > 0) {
|
|
212
|
+
audioByPacket[plan.refPacketId] = {
|
|
213
|
+
status: 'ok',
|
|
214
|
+
source: null,
|
|
215
|
+
language: null,
|
|
216
|
+
segments: plan.audioSegments,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
122
219
|
}
|
|
220
|
+
const audioContext = sidecar !== null
|
|
221
|
+
? sidecar.context
|
|
222
|
+
: {
|
|
223
|
+
status: audio !== undefined && audio.settings.mode === 'off' ? 'disabled' : 'no_audio',
|
|
224
|
+
source: null,
|
|
225
|
+
language: null,
|
|
226
|
+
speechRatio: null,
|
|
227
|
+
rmsDb: null,
|
|
228
|
+
speechRateCps: null,
|
|
229
|
+
segmentsInWindow: 0,
|
|
230
|
+
mountedPackets: Object.keys(audioByPacket).length,
|
|
231
|
+
telemetry: { cacheHit: false, chunks: 0, requests: 0, droppedByGuard: 0, costMs: 0 },
|
|
232
|
+
};
|
|
233
|
+
audioContext.mountedPackets = Object.keys(audioByPacket).length;
|
|
123
234
|
return {
|
|
124
235
|
...result,
|
|
125
236
|
packets: keptPackets,
|
|
@@ -129,6 +240,8 @@ async function runStreamCallLocked(videoPath, probe, intent, sessionId, options)
|
|
|
129
240
|
hotRefCount: precheck.kept.filter((it) => it.kind === 'hot').length,
|
|
130
241
|
sessionIntentId: merged.intentId,
|
|
131
242
|
appliedIntentId: merged.intentId,
|
|
243
|
+
audio: audioContext,
|
|
244
|
+
audioByPacket,
|
|
132
245
|
};
|
|
133
246
|
}
|
|
134
247
|
/** 预检结果 → 无损 JSON 投影（剔除 PNG Buffer 载体）。 */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streamCall.js","sourceRoot":"","sources":["../../src/tool/streamCall.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAwB,MAAM,4BAA4B,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAuB,MAAM,2BAA2B,CAAC;AAClF,OAAO,EACL,eAAe,EACf,YAAY,EACZ,UAAU,EACV,eAAe,EACf,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,GAIf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAgDnF;;;;;;GAMG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAA4B,CAAC;AAEzD,MAAM,UAAU,aAAa,CAC3B,SAAiB,EACjB,KAAkB,EAClB,MAAuB,EACvB,SAAiB,EACjB,OAA0B;IAE1B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI;SACd,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;SACtB,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CACxB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACtC,KAAK,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;YAAE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,SAAiB,EACjB,KAAkB,EAClB,MAAuB,EACvB,SAAiB,EACjB,OAA0B;IAE1B,MAAM,OAAO,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEhD,wCAAwC;IACxC,mDAAmD;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAClC,MAAM,MAAM,GAAG,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE;QACpD,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;KAClB,CAAC,CAAC;IACH,IAAI,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,UAAU,OAAO,CAAC,SAAS,EAAE,CAAC;IAChD,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;QACxE,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,QAAQ,EAAE,MAAM,CAAC,UAAU;YACzB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1D,CAAC,CAAC,OAAO,CAAC,qBAAqB;QACjC,aAAa,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACrC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,UAAU,EAAE,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,iBAAiB;QAC3D,eAAe,EAAE,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB;QAC3E,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,aAAa,EAAE,OAAO,CAAC,SAAS,GAAG,CAAC;QACpC,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,CAAC;QACxC,iBAAiB,EAAE,OAAO,CAAC,YAAY,GAAG,CAAC;QAC3C,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC;IAChC,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;IAC9C,OAAO,CAAC,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;IACtD,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;IAEpD,0CAA0C;IAC1C,iCAAiC;IACjC,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC;IACvD,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7C,YAAY,CACV,IAAI,EACJ,OAAO,EACP;QACE,SAAS,EAAE,UAAU;QACrB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,WAAW,EAAE,OAAO,CAAC,qBAAqB;KAC3C,EACD,GAAG,EAAE;QACH,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC,SAAS,CAAC;IAC3B,CAAC,EACD,MAAM,CAAC,QAAQ,CAChB,CAAC;IACF,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAExD,mCAAmC;IACnC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,IAAI,CAAC,KAAK,CAAC,QAAQ;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;SAC5B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACxC,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEzF,MAAM,KAAK,GAAiB;QAC1B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC9D,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;KACvE,CAAC;IACF,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE;QACrC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,UAAU,EAAE,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,iBAAiB;QAC3D,eAAe,EAAE,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB;QAC3E,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;KAC7C,CAAC,CAAC;IAEH,gCAAgC;IAChC,MAAM,WAAW,GAAqB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CACzF,CAAC;IACF,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CACrD,CAAC;YACF,IAAI,IAAI;gBAAE,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACpC,qCAAqC;QACrC,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,WAAW;QACpB,SAAS,EAAE,aAAa;QACxB,SAAS,EAAE,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE;QACzE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM;QACrE,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM;QACnE,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,eAAe,EAAE,MAAM,CAAC,QAAQ;KACjC,CAAC;AACJ,CAAC;AAED,2CAA2C;AAC3C,SAAS,iBAAiB,CAAC,QAAwB;IACjD,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3F,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjF,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;QAC3C,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,MAAM,EAAE,QAAQ,CAAC,MAAM;KACxB,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,OAAO,mBAAmB,CAAC,MAAM,CAAC,IAAI,IAAI,UAAU,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC;AACpF,CAAC"}
|
|
1
|
+
{"version":3,"file":"streamCall.js","sourceRoot":"","sources":["../../src/tool/streamCall.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAwB,MAAM,4BAA4B,CAAC;AACjF,OAAO,EAAE,gBAAgB,EAAuB,MAAM,2BAA2B,CAAC;AAClF,OAAO,EACL,eAAe,EACf,YAAY,EACZ,UAAU,EACV,eAAe,EACf,aAAa,GACd,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,GAIf,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnF,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,eAAe,GAChB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AA2DlD;;;;;;GAMG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,EAA4B,CAAC;AAEzD,MAAM,UAAU,aAAa,CAC3B,SAAiB,EACjB,KAAkB,EAClB,MAAuB,EACvB,SAAiB,EACjB,OAA0B;IAE1B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI;SACd,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;SACtB,IAAI,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CACxB,GAAG,EAAE,CAAC,SAAS,EACf,GAAG,EAAE,CAAC,SAAS,CAChB,CAAC;IACF,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACtC,KAAK,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;YAAE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,SAAiB,EACjB,KAAkB,EAClB,MAAuB,EACvB,SAAiB,EACjB,OAA0B;IAE1B,MAAM,OAAO,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEhD,wCAAwC;IACxC,mDAAmD;IACnD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAClC,MAAM,MAAM,GAAG,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE;QACpD,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,UAAU;KAClB,CAAC,CAAC;IACH,IAAI,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,UAAU,OAAO,CAAC,SAAS,EAAE,CAAC;IAChD,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;IAExB,2CAA2C;IAC3C,uCAAuC;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU;QAChC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IAClC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC;IAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,EACpE,OAAO,CACR,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,WAAW,GAA4B,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvG,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,MAAM,YAAY,GAChB,KAAK,KAAK,SAAS;QACnB,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK;QAC7B,WAAW,KAAK,IAAI;QACpB,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,IAAI,YAAY,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC/C,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE;gBAC7C,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;IACH,CAAC;IACD,MAAM,cAAc,GAClB,YAAY,IAAI,KAAK;QACnB,CAAC,CAAC,eAAe,CACb;YACE,SAAS;YACT,KAAK;YACL,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,cAAc,EAAE,MAAM;SACvB,EACD,WAAW,EACX,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CACpC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QACrB,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE;QACxE,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,QAAQ;QACR,aAAa,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACrC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,UAAU,EAAE,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,iBAAiB;QAC3D,eAAe,EAAE,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB;QAC3E,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,aAAa,EAAE,OAAO,CAAC,SAAS,GAAG,CAAC;QACpC,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,CAAC;QACxC,iBAAiB,EAAE,OAAO,CAAC,YAAY,GAAG,CAAC;QAC3C,WAAW,EACT,YAAY,IAAI,MAAM,KAAK,IAAI;YAC7B,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE;YAC/C,CAAC,CAAC,SAAS;QACf,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC;IAChC,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;IAC9C,OAAO,CAAC,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;IACtD,OAAO,CAAC,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC;IAEpD,oCAAoC;IACpC,MAAM,OAAO,GAAG,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;IACtE,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QAC9C,OAAO;YACL,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;YAClC,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAChD,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SAClF,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GACb,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QAC1E,CAAC,CAAC,aAAa,CACX,gBAAgB,EAChB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,OAAO,CAAC,MAAM,EACtB,OAAO,CAAC,OAAO,CAAC,MAAM,EACtB,OAAO,CAAC,OAAO,CAAC,QAAQ,CACzB;QACH,CAAC,CAAC,IAAI,GAAG,EAA4B,CAAC;IAC1C,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAA+B,CAAC;IACrE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;QACrC,qBAAqB,CAAC,GAAG,CACvB,GAAG,EACH,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;SACjB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAED,0CAA0C;IAC1C,iCAAiC;IACjC,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC;IACvD,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IACpE,YAAY,CACV,IAAI,EACJ,OAAO,EACP;QACE,SAAS,EAAE,UAAU;QACrB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,WAAW,EAAE,OAAO,CAAC,qBAAqB;KAC3C,EACD,GAAG,EAAE;QACH,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC;QACvB,OAAO,OAAO,CAAC,SAAS,CAAC;IAC3B,CAAC,EACD,MAAM,CAAC,QAAQ,CAChB,CAAC;IACF,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAExD,mCAAmC;IACnC,4CAA4C;IAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAG,IAAI,GAAG,CAC5B,IAAI,CAAC,KAAK,CAAC,QAAQ;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC;SAC5B,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CACxC,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;IAEzF,MAAM,KAAK,GAAiB;QAC1B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC9D,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;KACvE,CAAC;IACF,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE;QACrC,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,UAAU,EAAE,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,iBAAiB;QAC3D,eAAe,EAAE,MAAM,CAAC,iBAAiB,IAAI,OAAO,CAAC,sBAAsB;QAC3E,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;KAC7C,CAAC,CAAC;IAEH,gCAAgC;IAChC,MAAM,WAAW,GAAqB,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CACzF,CAAC;IACF,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CACrD,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACT,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC1C,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtE,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC3F,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC7B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IACD,iDAAiD;IACjD,oBAAoB;IACpB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;YAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7F,CAAC,CAAC,CAAC;IACH,oBAAoB;IACpB,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,EAAE,CAAC,IAAI,KAAK,KAAK;YAAE,SAAS;QAChC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzD,IAAI,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,6BAA6B;IAC7B,MAAM,aAAa,GAAqC,EAAE,CAAC;IAC3D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;QACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACpD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM;YAAE,SAAS;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CACnC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CACrD,CAAC;QACF,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5F,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG;gBAChC,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,QAAQ,EAAE,IAAI;gBACd,QAAQ,EAAE,IAAI,CAAC,aAAa;aAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAChB,OAAO,KAAK,IAAI;QACd,CAAC,CAAC,OAAO,CAAC,OAAO;QACjB,CAAC,CAAC;YACE,MAAM,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;YACtF,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,IAAI;YACX,aAAa,EAAE,IAAI;YACnB,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM;YACjD,SAAS,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;SACrF,CAAC;IACR,YAAY,CAAC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;IAEhE,OAAO;QACL,GAAG,MAAM;QACT,OAAO,EAAE,WAAW;QACpB,SAAS,EAAE,aAAa;QACxB,SAAS,EAAE,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE;QACzE,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,MAAM;QACrE,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,MAAM;QACnE,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,eAAe,EAAE,MAAM,CAAC,QAAQ;QAChC,KAAK,EAAE,YAAY;QACnB,aAAa;KACd,CAAC;AACJ,CAAC;AAED,2CAA2C;AAC3C,SAAS,iBAAiB,CAAC,QAAwB;IACjD,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3F,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjF,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;QAC3C,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,MAAM,EAAE,QAAQ,CAAC,MAAM;KACxB,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,OAAO,mBAAmB,CAAC,MAAM,CAAC,IAAI,IAAI,UAAU,EAAE,MAAM,CAAC,KAAK,IAAI,UAAU,CAAC,CAAC;AACpF,CAAC"}
|
package/lib/types.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* 与 source_crop_bbox 为笛卡尔像素序 [x, y, w, h] 或 [x_min, y_min, x_max, y_max]，
|
|
8
8
|
* 后者满足 x = 42*col, y = 42*row。source 坐标一律处于转正后的显示坐标系。
|
|
9
9
|
*/
|
|
10
|
+
import type { AudioPluginConfig, IntentAudioDirective, TranscriptSegment } from './audio/types.js';
|
|
10
11
|
export interface ProbeResult {
|
|
11
12
|
codedWidth: number;
|
|
12
13
|
codedHeight: number;
|
|
@@ -80,6 +81,8 @@ export interface IntentDirective {
|
|
|
80
81
|
max_packets?: number;
|
|
81
82
|
max_tokens_budget?: number;
|
|
82
83
|
reset?: boolean;
|
|
84
|
+
/** 音频台词侧车指令（字幕提取/ASR 语言/assistFocus 开关）。 */
|
|
85
|
+
audio?: IntentAudioDirective;
|
|
83
86
|
}
|
|
84
87
|
export interface SessionIntent {
|
|
85
88
|
intentId: string;
|
|
@@ -173,6 +176,8 @@ export interface HistoryEntry {
|
|
|
173
176
|
anchorBytes: Buffer | null;
|
|
174
177
|
/** F 载体规范化像素内容哈希（§6 降级轨；不含 F 时为 null）。 */
|
|
175
178
|
anchorFContentHash: string | null;
|
|
179
|
+
/** 冷区引用时 100% 保留的台词段（音频侧车写入；JSON 安全）。 */
|
|
180
|
+
audioSegments?: TranscriptSegment[] | undefined;
|
|
176
181
|
}
|
|
177
182
|
/** 发包历史环形队列：按包龄旧→新排列；热区为最近 2 包，冷区至多 maxColdRefs 包。 */
|
|
178
183
|
export interface HistoryRing {
|
|
@@ -191,6 +196,8 @@ export interface ColdRefPlan {
|
|
|
191
196
|
bytes: Buffer;
|
|
192
197
|
/** F 载体规范化像素内容哈希（§6 降级轨）。 */
|
|
193
198
|
contentHash?: string | undefined;
|
|
199
|
+
/** 源包台词段（Epoch 内逐字节复用）。 */
|
|
200
|
+
audioSegments?: TranscriptSegment[] | undefined;
|
|
194
201
|
}
|
|
195
202
|
/** Epoch Compaction 状态：窗口内冷区引用零变动（前缀稳定）。 */
|
|
196
203
|
export interface EpochState {
|
|
@@ -352,4 +359,6 @@ export interface PfStreamPluginConfig {
|
|
|
352
359
|
streamWorkMaxEdge: number;
|
|
353
360
|
epochCompactionWindow: number;
|
|
354
361
|
meteringProfile: MeteringProfile;
|
|
362
|
+
/** 音频台词侧车（三级顺位：字幕 → VAD+ASR → 跳过）。 */
|
|
363
|
+
audio: AudioPluginConfig;
|
|
355
364
|
}
|
package/locale/en.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"meta": {
|
|
3
|
+
"title": "Video understanding",
|
|
4
|
+
"description": "The plugin samples keyframes adaptively as the picture changes, then combines them with embedded subtitles or transcript text, so the agent can read visuals and dialogue together.\n\nTranscription works in two ways: a local model runs offline, or a cloud API key calls a hosted service. The local model can be downloaded on the plugin page."
|
|
5
|
+
}
|
|
6
|
+
}
|
package/locale/zh.json
ADDED
package/package.json
CHANGED
|
@@ -1,16 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-pf-stream",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "DeepSeek Harness video
|
|
3
|
+
"version": "0.3.1",
|
|
4
|
+
"description": "DeepSeek Harness video understanding plugin: adaptive dual-scale streaming sampling plus deterministic inspection, with subtitles or transcribed speech and a self-service setup panel.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/Weilv-D/dsh-pf-stream"
|
|
9
9
|
},
|
|
10
|
+
"homepage": "https://github.com/Weilv-D/dsh-pf-stream#readme",
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/Weilv-D/dsh-pf-stream/issues"
|
|
13
|
+
},
|
|
10
14
|
"main": "lib/index.js",
|
|
11
15
|
"types": "lib/index.d.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": "./lib/index.js",
|
|
18
|
+
"./client": "./client.js",
|
|
19
|
+
"./locale/*.json": "./locale/*.json",
|
|
20
|
+
"./icon": "./icon.svg"
|
|
21
|
+
},
|
|
12
22
|
"files": [
|
|
13
23
|
"lib",
|
|
24
|
+
"client.js",
|
|
25
|
+
"locale/zh.json",
|
|
26
|
+
"locale/en.json",
|
|
27
|
+
"icon.svg",
|
|
14
28
|
"cordis.patch.yml",
|
|
15
29
|
"LICENSE",
|
|
16
30
|
"README.md",
|
|
@@ -44,6 +58,13 @@
|
|
|
44
58
|
"dsh": {
|
|
45
59
|
"bundle": {
|
|
46
60
|
"patch": "./cordis.patch.yml"
|
|
61
|
+
},
|
|
62
|
+
"client": {
|
|
63
|
+
"platform": "web",
|
|
64
|
+
"immediately": true,
|
|
65
|
+
"inject": [
|
|
66
|
+
"slots"
|
|
67
|
+
]
|
|
47
68
|
}
|
|
48
69
|
},
|
|
49
70
|
"keywords": [
|