@lalalic/markcut 1.2.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -17
- package/README.md +7 -3
- package/SKILL.md +0 -2
- package/docs/edit-mode.md +1 -1
- package/docs/label-mode.md +6 -4
- package/docs/markdown-descriptive.md +34 -1
- package/docs/system-prompt-edit.md +19 -0
- package/package.json +1 -1
- package/src/config.mjs +8 -3
- package/src/descriptive/compiler.test.ts +42 -42
- package/src/descriptive/compiler.ts +41 -52
- package/src/descriptive/markdown.ts +8 -4
- package/src/descriptive/resolve.test.ts +14 -7
- package/src/descriptive/resolve.ts +148 -20
- package/src/player/browser.tsx +178 -54
- package/src/player/bundle/player.js +1149 -566
- package/src/player/components/EditControls.tsx +92 -0
- package/src/player/components/HeaderBar.tsx +60 -0
- package/src/player/components/LabelControls.tsx +367 -0
- package/src/player/components/SceneThumbnails.tsx +60 -0
- package/src/player/components/VariantBar.tsx +39 -0
- package/src/player/components/index.ts +5 -0
- package/src/player/pipeline.mjs +130 -66
- package/src/player/pipeline.ts +3 -1
- package/src/player/server-shared.mjs +5 -7
- package/src/player/server.mjs +336 -239
- package/src/render/cli.mjs +4 -6
- package/src/schema/index.ts +20 -23
- package/src/types/Audio.tsx +25 -33
- package/src/types/Component.tsx +18 -24
- package/src/types/Effect.tsx +31 -39
- package/src/types/Image.tsx +23 -30
- package/src/types/Include.tsx +70 -76
- package/src/types/Map.tsx +48 -44
- package/src/types/Rhythm.tsx +19 -27
- package/src/types/Video.tsx +40 -47
- package/src/utils/index.ts +23 -10
- package/src/vision/cli.mjs +6 -6
- package/templates/courseware/TEMPLATE.md +16 -10
- package/tests/fixtures/audio.json +4 -2
- package/tests/fixtures/basic.json +2 -1
- package/tests/fixtures/component-all.json +4 -2
- package/tests/fixtures/components.json +4 -2
- package/tests/fixtures/effects.json +12 -6
- package/tests/fixtures/full.json +8 -4
- package/tests/fixtures/map.json +2 -1
- package/tests/fixtures/md/dialogue.md +12 -0
- package/tests/fixtures/md/edge-cases.md +9 -0
- package/tests/fixtures/scenes.json +4 -2
- package/tests/fixtures/subtitle.json +6 -3
- package/tests/fixtures/subvideo.json +11 -6
- package/tests/fixtures/templates/courseware.md +61 -4
- package/tests/fixtures/tween-visual.json +2 -6
- package/tests/fixtures/video-series.json +6 -14
- package/tests/md-descriptive.test.ts +170 -0
- package/tests/render.test.ts +32 -16
- package/tests/schema.test.ts +6 -3
- package/tests/server.test.ts +9 -6
- package/tests/utils.ts +4 -4
- package/scripts/artlist-dl.mjs +0 -190
- package/src/player/label-server.mjs +0 -599
|
@@ -18,6 +18,8 @@ import { uid, walkDown } from "../utils/index";
|
|
|
18
18
|
|
|
19
19
|
export interface CompileOptions {
|
|
20
20
|
defaults?: Partial<Record<"image" | "video" | "audio" | "component" | "rhythm" | "include" | "map", number>>;
|
|
21
|
+
/** Google Maps API key, injected onto map nodes during compilation. */
|
|
22
|
+
googleMapsApiKey?: string;
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
/** A single effect spec — either a bare animation name or an object with options. */
|
|
@@ -65,6 +67,10 @@ export interface DescriptiveAudio extends DescriptiveBaseNode {
|
|
|
65
67
|
src?: string;
|
|
66
68
|
/** Narration text for TTS generation. When set without src, triggers TTS pipeline. */
|
|
67
69
|
script?: string;
|
|
70
|
+
/** Speaker name for multi-turn dialogue. Set by resolveDialogue when expanding
|
|
71
|
+
* `SpeakerName: text` format scripts. Used for per-speaker voice selection and
|
|
72
|
+
* subtitle prefix. */
|
|
73
|
+
speaker?: string;
|
|
68
74
|
volume?: number;
|
|
69
75
|
foreground?: boolean;
|
|
70
76
|
startFrom?: number;
|
|
@@ -183,6 +189,12 @@ export interface DescriptiveRoot {
|
|
|
183
189
|
imports?: never;
|
|
184
190
|
/** Raw imports block source (from \`\`\`imports code fence). Parsed by parseImportsBlock. */
|
|
185
191
|
importsBlock?: string;
|
|
192
|
+
/** Per-speaker voice mapping for multi-turn dialogue.
|
|
193
|
+
* Keys are speaker names (e.g. "Ray", "Alice"), values are TTS voice names
|
|
194
|
+
* (e.g. "en-US-GuyNeural", "en-US-JennyNeural").
|
|
195
|
+
* When a dialogue node has a matching `speaker` field, its TTS voice is
|
|
196
|
+
* substituted from this map. */
|
|
197
|
+
voices?: Record<string, string>;
|
|
186
198
|
children: DescriptiveNode[];
|
|
187
199
|
}
|
|
188
200
|
|
|
@@ -250,6 +262,7 @@ export function resolveAllTemplateVars(
|
|
|
250
262
|
|
|
251
263
|
interface CompileContext {
|
|
252
264
|
defaults: Record<"image" | "video" | "audio" | "component" | "rhythm" | "include" | "map" | "effect", number>;
|
|
265
|
+
googleMapsApiKey: string;
|
|
253
266
|
}
|
|
254
267
|
|
|
255
268
|
interface CompileResult {
|
|
@@ -422,24 +435,18 @@ function wrapWithEffects(
|
|
|
422
435
|
if (!rawEffects || rawEffects.length === 0) return result;
|
|
423
436
|
|
|
424
437
|
const effects = rawEffects.map(normalizeEffectSpec);
|
|
425
|
-
const innerStream = result.stream;
|
|
426
|
-
const
|
|
427
|
-
const
|
|
428
|
-
const absStart = firstAction.start ?? 0;
|
|
429
|
-
const absEnd = firstAction.end ?? result.duration;
|
|
438
|
+
const innerStream = result.stream as any;
|
|
439
|
+
const absStart = innerStream.start ?? 0;
|
|
440
|
+
const absEnd = innerStream.end ?? result.duration;
|
|
430
441
|
const duration = absEnd - absStart;
|
|
431
442
|
|
|
432
|
-
// Reset inner stream's
|
|
443
|
+
// Reset inner stream's timing to be relative (start=0) so the effect
|
|
433
444
|
// wrapper owns the absolute timing. The EffectWrapper renders children
|
|
434
|
-
// with their relative
|
|
445
|
+
// with their relative timing inside its own Sequence.
|
|
435
446
|
const resetStream = {
|
|
436
447
|
...innerStream,
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
id: uid(),
|
|
440
|
-
start: 0,
|
|
441
|
-
end: duration,
|
|
442
|
-
}],
|
|
448
|
+
start: 0,
|
|
449
|
+
end: duration,
|
|
443
450
|
durationInSeconds: duration,
|
|
444
451
|
} as any;
|
|
445
452
|
|
|
@@ -466,11 +473,8 @@ function wrapWithEffects(
|
|
|
466
473
|
animationIterationCount: spec.animationIterationCount ?? 1,
|
|
467
474
|
customKeyframes: spec.customKeyframes,
|
|
468
475
|
children: [currentStream],
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
start: effStart,
|
|
472
|
-
end: effEnd,
|
|
473
|
-
}],
|
|
476
|
+
start: effStart,
|
|
477
|
+
end: effEnd,
|
|
474
478
|
visible: true,
|
|
475
479
|
...pickOn(node),
|
|
476
480
|
} as Effect;
|
|
@@ -490,21 +494,12 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
490
494
|
style: node.style,
|
|
491
495
|
visible: node.visible ?? true,
|
|
492
496
|
isBackground: node.isBackground,
|
|
493
|
-
durationInSeconds: end,
|
|
494
|
-
...pickOn(node),
|
|
495
|
-
};
|
|
496
|
-
|
|
497
|
-
const action = {
|
|
498
|
-
id: uid(),
|
|
499
497
|
start,
|
|
500
498
|
end,
|
|
501
499
|
startFrom: node.type === "video" || node.type === "audio" ? node.startFrom : undefined,
|
|
502
500
|
endAt: node.type === "video" || node.type === "audio" ? node.endAt : undefined,
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
node.type === "video" || node.type === "audio" || node.type === "rhythm"
|
|
506
|
-
? node.volume
|
|
507
|
-
: undefined,
|
|
501
|
+
durationInSeconds: end,
|
|
502
|
+
...pickOn(node),
|
|
508
503
|
};
|
|
509
504
|
|
|
510
505
|
switch (node.type) {
|
|
@@ -517,7 +512,6 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
517
512
|
playbackRate: node.playbackRate,
|
|
518
513
|
width: node.width ?? 1080,
|
|
519
514
|
height: node.height ?? 1920,
|
|
520
|
-
actions: [action],
|
|
521
515
|
};
|
|
522
516
|
return { stream, duration: end };
|
|
523
517
|
}
|
|
@@ -528,7 +522,8 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
528
522
|
src: node.src,
|
|
529
523
|
volume: node.volume ?? 1,
|
|
530
524
|
foreground: node.foreground,
|
|
531
|
-
|
|
525
|
+
loop: node.loop,
|
|
526
|
+
speaker: node.speaker,
|
|
532
527
|
};
|
|
533
528
|
return { stream, duration: end };
|
|
534
529
|
}
|
|
@@ -538,7 +533,6 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
538
533
|
type: "image",
|
|
539
534
|
src: node.src,
|
|
540
535
|
fit: node.fit ?? "contain",
|
|
541
|
-
actions: [action],
|
|
542
536
|
};
|
|
543
537
|
return { stream, duration: end };
|
|
544
538
|
}
|
|
@@ -561,7 +555,6 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
561
555
|
type: "component",
|
|
562
556
|
jsx: node.jsx,
|
|
563
557
|
data: Object.keys(bindings).length ? bindings : undefined,
|
|
564
|
-
actions: [action],
|
|
565
558
|
};
|
|
566
559
|
return { stream, duration: end };
|
|
567
560
|
}
|
|
@@ -574,7 +567,6 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
574
567
|
volume: node.volume ?? 1,
|
|
575
568
|
spots: node.spots,
|
|
576
569
|
children: [],
|
|
577
|
-
actions: [action],
|
|
578
570
|
};
|
|
579
571
|
return { stream, duration: end };
|
|
580
572
|
}
|
|
@@ -590,7 +582,7 @@ function compileLeaf(node: Exclude<DescriptiveNode, DescriptiveContainer | Descr
|
|
|
590
582
|
mapType: node.mapType ?? "roadmap",
|
|
591
583
|
travelMode: node.travelMode ?? "DRIVING",
|
|
592
584
|
routeMarker: node.routeMarker ?? "🚗",
|
|
593
|
-
|
|
585
|
+
googleMapsApiKey: ctx.googleMapsApiKey,
|
|
594
586
|
};
|
|
595
587
|
return { stream, duration: end };
|
|
596
588
|
}
|
|
@@ -668,6 +660,7 @@ function compileScene(
|
|
|
668
660
|
type: "folder",
|
|
669
661
|
visible: true,
|
|
670
662
|
isSeries: true,
|
|
663
|
+
start: 0,
|
|
671
664
|
transition: sceneKind === "transitionSeries" ? resolved.name : undefined,
|
|
672
665
|
transitionTime: sceneKind === "transitionSeries" ? resolved.time : 0.5,
|
|
673
666
|
children: compiledChildren.map((c) => c.stream),
|
|
@@ -691,6 +684,7 @@ function compileScene(
|
|
|
691
684
|
style: node.style,
|
|
692
685
|
visible: node.visible ?? true,
|
|
693
686
|
isBackground: node.isBackground,
|
|
687
|
+
start,
|
|
694
688
|
children: sceneChildren,
|
|
695
689
|
durationInSeconds: end,
|
|
696
690
|
...pickOn(node),
|
|
@@ -732,13 +726,8 @@ function compileInclude(
|
|
|
732
726
|
src: node.src,
|
|
733
727
|
volume: node.volume ?? 1,
|
|
734
728
|
children: compiledChildren.map((c) => c.stream),
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
id: uid(),
|
|
738
|
-
start,
|
|
739
|
-
end,
|
|
740
|
-
},
|
|
741
|
-
],
|
|
729
|
+
start,
|
|
730
|
+
end,
|
|
742
731
|
durationInSeconds: end,
|
|
743
732
|
...pickOn(node),
|
|
744
733
|
};
|
|
@@ -802,14 +791,8 @@ function compileRhythm(
|
|
|
802
791
|
volume: node.volume ?? 1,
|
|
803
792
|
spots: node.spots,
|
|
804
793
|
children: compiledChildren,
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
id: uid(),
|
|
808
|
-
start,
|
|
809
|
-
end,
|
|
810
|
-
volume: node.volume,
|
|
811
|
-
},
|
|
812
|
-
],
|
|
794
|
+
start,
|
|
795
|
+
end,
|
|
813
796
|
durationInSeconds: end,
|
|
814
797
|
...pickOn(node),
|
|
815
798
|
};
|
|
@@ -833,6 +816,7 @@ function compileContainer(node: DescriptiveContainer, ctx: CompileContext, paren
|
|
|
833
816
|
style: node.style,
|
|
834
817
|
visible: node.visible ?? true,
|
|
835
818
|
isBackground: node.isBackground,
|
|
819
|
+
start: 0,
|
|
836
820
|
isSeries: node.type !== "parallel",
|
|
837
821
|
transition: node.type === "transitionSeries" ? resolved.name : undefined,
|
|
838
822
|
transitionTime: node.type === "transitionSeries" ? resolved.time : 0.5,
|
|
@@ -1191,11 +1175,18 @@ export function resolveVariantOverrides(
|
|
|
1191
1175
|
}
|
|
1192
1176
|
|
|
1193
1177
|
export function compileDescriptiveRoot(input: DescriptiveRoot, options: CompileOptions = {}): Root {
|
|
1178
|
+
const root: DescriptiveRoot = typeof input === "string" ? JSON.parse(input) : input;
|
|
1179
|
+
|
|
1180
|
+
const resolved = resolveTransition(root.layout ? root.transition ?? root.layout : root.transition, root.transitionTime);
|
|
1181
|
+
const rootKind: "series" | "parallel" | "transitionSeries" = root.layout === "parallel" ? "parallel" : "series";
|
|
1182
|
+
|
|
1183
|
+
const googleMapsApiKey = options.googleMapsApiKey ?? "";
|
|
1194
1184
|
const ctx: CompileContext = {
|
|
1195
1185
|
defaults: {
|
|
1196
1186
|
...DEFAULTS,
|
|
1197
1187
|
...(options.defaults ?? {}),
|
|
1198
1188
|
},
|
|
1189
|
+
googleMapsApiKey,
|
|
1199
1190
|
};
|
|
1200
1191
|
|
|
1201
1192
|
// Resolve frontmatter imports / inline component defs onto each component node.
|
|
@@ -1204,8 +1195,6 @@ export function compileDescriptiveRoot(input: DescriptiveRoot, options: CompileO
|
|
|
1204
1195
|
|
|
1205
1196
|
ensureUniqueIds(input.children, "root");
|
|
1206
1197
|
|
|
1207
|
-
const rootKind = input.layout ?? "series";
|
|
1208
|
-
const resolved = resolveTransition(input.transition, input.transitionTime);
|
|
1209
1198
|
const children = compileChildren(input.children, ctx, rootKind);
|
|
1210
1199
|
const duration = aggregateDuration(children, rootKind, resolved.time);
|
|
1211
1200
|
|
|
@@ -101,7 +101,7 @@ function preserveVariantAttrs(node: Record<string, unknown>, attrs: Record<strin
|
|
|
101
101
|
"spots", "waypoints", "routeColor", "routeWeight", "routeMarker",
|
|
102
102
|
"travelMode", "zoom", "center", "mapType", "data", "prompt",
|
|
103
103
|
"name", "title", "transition", "transitionTime", "layout",
|
|
104
|
-
"componentName", "props",
|
|
104
|
+
"componentName", "props", "speaker",
|
|
105
105
|
]);
|
|
106
106
|
for (const [k, v] of Object.entries(attrs)) {
|
|
107
107
|
if (!STANDARD.has(k)) {
|
|
@@ -214,6 +214,7 @@ function parseNodeLine(content: string, lineNum?: number): DescriptiveNode {
|
|
|
214
214
|
type: "audio",
|
|
215
215
|
id: attrs.id as any,
|
|
216
216
|
src,
|
|
217
|
+
speaker: attrs.speaker as string | undefined,
|
|
217
218
|
duration: attrs.duration as any,
|
|
218
219
|
start: attrs.start as any,
|
|
219
220
|
startFrom: attrs.startFrom as any,
|
|
@@ -292,14 +293,14 @@ function parseNodeLine(content: string, lineNum?: number): DescriptiveNode {
|
|
|
292
293
|
}
|
|
293
294
|
case "script": {
|
|
294
295
|
const raw = firstPositional ?? (attrs.script ? String(attrs.script) : undefined);
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
const text = isQuoted(raw) ? unquote(raw) : raw;
|
|
296
|
+
// Text may be empty — ~~~script code fence will provide it later
|
|
297
|
+
const text = raw ? (isQuoted(raw) ? unquote(raw) : raw) : undefined;
|
|
298
298
|
// Script is an alias for audio — creates an audio node with TTS-needed marker.
|
|
299
299
|
const scriptNode: any = {
|
|
300
300
|
type: "audio",
|
|
301
301
|
id: attrs.id as any,
|
|
302
302
|
script: text,
|
|
303
|
+
speaker: attrs.speaker as string | undefined,
|
|
303
304
|
volume: (attrs.volume as number | undefined) ?? 1,
|
|
304
305
|
start: attrs.start as any,
|
|
305
306
|
duration: attrs.duration as any,
|
|
@@ -667,6 +668,9 @@ function applyRootAttrs(root: DescriptiveRoot, attrs: Record<string, unknown>):
|
|
|
667
668
|
}
|
|
668
669
|
break;
|
|
669
670
|
}
|
|
671
|
+
case "voices":
|
|
672
|
+
root.voices = v as Record<string, string>;
|
|
673
|
+
break;
|
|
670
674
|
default:
|
|
671
675
|
throw new Error(`unknown root key: ${k}`);
|
|
672
676
|
}
|
|
@@ -119,7 +119,8 @@ describe("resolveSubtitles", () => {
|
|
|
119
119
|
type: "scene", name: "S", layout: "parallel",
|
|
120
120
|
children: [{
|
|
121
121
|
id: "a1", type: "audio", src: audioPath,
|
|
122
|
-
|
|
122
|
+
start: 5,
|
|
123
|
+
end: 8,
|
|
123
124
|
} as any],
|
|
124
125
|
durationInSeconds: 8,
|
|
125
126
|
} as any],
|
|
@@ -490,7 +491,8 @@ describe("resolveSubtitles — additional", () => {
|
|
|
490
491
|
stt: "custom-stt {input} --out {output}",
|
|
491
492
|
children: [{
|
|
492
493
|
type: "scene", name: "S", layout: "parallel",
|
|
493
|
-
children: [{ id: "a1", type: "audio", src: audioPath,
|
|
494
|
+
children: [{ id: "a1", type: "audio", src: audioPath, start: 0,
|
|
495
|
+
end: 2 } as any],
|
|
494
496
|
durationInSeconds: 2,
|
|
495
497
|
} as any],
|
|
496
498
|
};
|
|
@@ -511,7 +513,8 @@ describe("resolveSubtitles — additional", () => {
|
|
|
511
513
|
const root: DescriptiveRoot = {
|
|
512
514
|
children: [{
|
|
513
515
|
type: "scene", name: "S", layout: "parallel",
|
|
514
|
-
children: [{ id: "a1", type: "audio", src: audioPath,
|
|
516
|
+
children: [{ id: "a1", type: "audio", src: audioPath, start: 0,
|
|
517
|
+
end: 2 } as any],
|
|
515
518
|
durationInSeconds: 2,
|
|
516
519
|
} as any],
|
|
517
520
|
};
|
|
@@ -531,7 +534,8 @@ describe("resolveSubtitles — additional", () => {
|
|
|
531
534
|
const root: DescriptiveRoot = {
|
|
532
535
|
children: [{
|
|
533
536
|
type: "scene", name: "S", layout: "parallel",
|
|
534
|
-
children: [{ id: "a1", type: "audio", src: audioPath,
|
|
537
|
+
children: [{ id: "a1", type: "audio", src: audioPath, start: 0,
|
|
538
|
+
end: 2 } as any],
|
|
535
539
|
durationInSeconds: 2,
|
|
536
540
|
} as any],
|
|
537
541
|
};
|
|
@@ -558,8 +562,10 @@ describe("resolveSubtitles — additional", () => {
|
|
|
558
562
|
children: [{
|
|
559
563
|
type: "scene", name: "S", layout: "parallel",
|
|
560
564
|
children: [
|
|
561
|
-
{ id: "a1", type: "audio", src: a1,
|
|
562
|
-
|
|
565
|
+
{ id: "a1", type: "audio", src: a1, start: 0,
|
|
566
|
+
end: 2 },
|
|
567
|
+
{ id: "a2", type: "audio", src: a2, start: 3,
|
|
568
|
+
end: 5 },
|
|
563
569
|
],
|
|
564
570
|
durationInSeconds: 5,
|
|
565
571
|
}],
|
|
@@ -586,7 +592,8 @@ describe("resolveSubtitles — additional", () => {
|
|
|
586
592
|
const root: any = {
|
|
587
593
|
children: [{
|
|
588
594
|
type: "scene", name: "S", layout: "parallel",
|
|
589
|
-
children: [{ id: "a1", type: "audio", src: audioPath,
|
|
595
|
+
children: [{ id: "a1", type: "audio", src: audioPath, start: 0,
|
|
596
|
+
end: 2 }],
|
|
590
597
|
durationInSeconds: 2,
|
|
591
598
|
}],
|
|
592
599
|
};
|
|
@@ -261,6 +261,117 @@ export async function resolveMediaDurations(
|
|
|
261
261
|
return clone;
|
|
262
262
|
}
|
|
263
263
|
|
|
264
|
+
// ── Dialogue expansion ──────────────────────────────────────────────────────
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Detect if a script contains multi-turn dialogue in `SpeakerName: text` format.
|
|
268
|
+
*
|
|
269
|
+
* A script is a "dialogue" if at least 2 lines match the pattern `Name: text`.
|
|
270
|
+
* Single-line `SpeakerName: text` is NOT treated as dialogue — it's just a
|
|
271
|
+
* regular script with a speaker annotation.
|
|
272
|
+
*
|
|
273
|
+
* Example dialogue:
|
|
274
|
+
* ```
|
|
275
|
+
* Ray: Hello everyone and welcome
|
|
276
|
+
* Alice: Good day to you all
|
|
277
|
+
* Ray: Let's get started
|
|
278
|
+
* ```
|
|
279
|
+
*
|
|
280
|
+
* Each line becomes a separate audio node with:
|
|
281
|
+
* - `script`: just the text (without the "SpeakerName: " prefix)
|
|
282
|
+
* - `speaker`: the speaker name
|
|
283
|
+
*
|
|
284
|
+
* The dialogue lines are wrapped in a series container so they play sequentially
|
|
285
|
+
* regardless of the parent scene's layout.
|
|
286
|
+
*/
|
|
287
|
+
export function parseDialogueLines(script: string): Array<{ speaker: string; text: string }> {
|
|
288
|
+
const lines = script.split("\n").map(l => l.trim()).filter(Boolean);
|
|
289
|
+
const dialoguePattern = /^([\w\u4e00-\u9fff][\w\s\u4e00-\u9fff]*?):\s*(.+)$/;
|
|
290
|
+
|
|
291
|
+
const result: Array<{ speaker: string; text: string }> = [];
|
|
292
|
+
for (const line of lines) {
|
|
293
|
+
const match = line.match(dialoguePattern);
|
|
294
|
+
if (!match) return []; // Any non-matching line invalidates the dialogue format
|
|
295
|
+
result.push({ speaker: match[1]!.trim(), text: match[2]!.trim() });
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// Require at least 2 lines to be considered a dialogue
|
|
299
|
+
return result.length >= 2 ? result : [];
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* Walk the descriptive tree and expand multi-turn dialogue-formatted scripts
|
|
304
|
+
* into sequential per-line audio nodes.
|
|
305
|
+
*
|
|
306
|
+
* This runs BEFORE resolveScripts so each dialogue line gets its own TTS
|
|
307
|
+
* generation with the correct per-speaker voice.
|
|
308
|
+
*/
|
|
309
|
+
export function resolveDialogue(root: DescriptiveRoot): DescriptiveRoot {
|
|
310
|
+
const clone: DescriptiveRoot = JSON.parse(JSON.stringify(root));
|
|
311
|
+
|
|
312
|
+
// Collect dialogue nodes for expansion (walking and modifying simultaneously is tricky)
|
|
313
|
+
const expansions: Array<{
|
|
314
|
+
parent: any;
|
|
315
|
+
index: number;
|
|
316
|
+
originalNode: any;
|
|
317
|
+
lines: Array<{ speaker: string; text: string }>;
|
|
318
|
+
}> = [];
|
|
319
|
+
|
|
320
|
+
walkDown(clone as any, (node, parent) => {
|
|
321
|
+
if (node.type !== "audio") return;
|
|
322
|
+
if (!node.script || typeof node.script !== "string") return;
|
|
323
|
+
if (node.src) return; // Already has a real source
|
|
324
|
+
|
|
325
|
+
const lines = parseDialogueLines(node.script);
|
|
326
|
+
if (lines.length < 2) return; // Not a dialogue
|
|
327
|
+
|
|
328
|
+
if (!parent || !Array.isArray(parent.children)) return;
|
|
329
|
+
const idx = parent.children.indexOf(node);
|
|
330
|
+
if (idx === -1) return;
|
|
331
|
+
|
|
332
|
+
expansions.push({ parent, index: idx, originalNode: node, lines });
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
if (expansions.length === 0) return root;
|
|
336
|
+
|
|
337
|
+
// Apply expansions in reverse index order so splice indices stay valid
|
|
338
|
+
expansions.sort((a, b) => b.index - a.index);
|
|
339
|
+
|
|
340
|
+
for (const { parent, index, originalNode, lines } of expansions) {
|
|
341
|
+
const dialogueNodes: any[] = [];
|
|
342
|
+
|
|
343
|
+
for (let i = 0; i < lines.length; i++) {
|
|
344
|
+
const line = lines[i]!;
|
|
345
|
+
const { speaker, text } = line;
|
|
346
|
+
const lineNode: any = {
|
|
347
|
+
...originalNode,
|
|
348
|
+
id: originalNode.id ? `${originalNode.id}-line-${i}` : undefined,
|
|
349
|
+
script: text,
|
|
350
|
+
speaker,
|
|
351
|
+
start: 0,
|
|
352
|
+
duration: undefined, // Will be set by TTS duration probing
|
|
353
|
+
};
|
|
354
|
+
delete lineNode.src;
|
|
355
|
+
dialogueNodes.push(lineNode);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// Wrap dialogue lines in a series container for sequential playback
|
|
359
|
+
const seriesContainer: any = {
|
|
360
|
+
type: "series",
|
|
361
|
+
id: `${originalNode.id ?? "dialogue"}-series`,
|
|
362
|
+
children: dialogueNodes,
|
|
363
|
+
};
|
|
364
|
+
|
|
365
|
+
// Replace the original node with the series container
|
|
366
|
+
parent.children.splice(index, 1, seriesContainer);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
const expanded = expansions.length > 0 ? clone : root;
|
|
370
|
+
const count = expansions.reduce((sum, e) => sum + e.lines.length, 0);
|
|
371
|
+
console.log(` 💬 dialogue: expanded ${expansions.length} script${expansions.length > 1 ? "s" : ""} into ${count} lines`);
|
|
372
|
+
return expanded;
|
|
373
|
+
}
|
|
374
|
+
|
|
264
375
|
// ── Script → TTS → STT ─────────────────────────────────────────────────────
|
|
265
376
|
|
|
266
377
|
export interface ResolveScriptOptions {
|
|
@@ -292,23 +403,30 @@ export async function resolveScripts(
|
|
|
292
403
|
let cacheDirty = false;
|
|
293
404
|
|
|
294
405
|
// Collect all audio nodes that have script text but no src yet
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
walkDown(clone as any, (node, parent) => {
|
|
406
|
+
const allScriptNodes: Array<{ node: any; id: string }> = [];
|
|
407
|
+
walkDown(clone as any, (node) => {
|
|
298
408
|
if (node.type !== "audio") return;
|
|
299
409
|
if (!node.script || typeof node.script !== "string") return;
|
|
300
410
|
if (node.src) return; // already has real source
|
|
301
411
|
const id = node.id ?? `audio-${allScriptNodes.length}`;
|
|
302
|
-
allScriptNodes.push({ node, id
|
|
412
|
+
allScriptNodes.push({ node, id });
|
|
303
413
|
});
|
|
304
414
|
|
|
305
415
|
if (allScriptNodes.length > 0) {
|
|
306
416
|
console.log(` 🔊 TTS: generating ${allScriptNodes.length} script${allScriptNodes.length > 1 ? "s" : ""}...`);
|
|
307
417
|
}
|
|
308
418
|
|
|
309
|
-
for (const { node, id
|
|
310
|
-
//
|
|
311
|
-
|
|
419
|
+
for (const { node, id } of allScriptNodes) {
|
|
420
|
+
// TTS CLI from root config only
|
|
421
|
+
let ttsCli = clone.tts ?? options.ttsCli ?? DEFAULT_TTS_CLI;
|
|
422
|
+
|
|
423
|
+
// Per-speaker voice appends extra CLI flags from root voices config
|
|
424
|
+
if (node.speaker && clone.voices) {
|
|
425
|
+
const speakerVoice = clone.voices[node.speaker];
|
|
426
|
+
if (speakerVoice) {
|
|
427
|
+
ttsCli = `${ttsCli} ${speakerVoice}`;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
312
430
|
|
|
313
431
|
// Content-addressed filename: hash of script + CLI, so identical scripts
|
|
314
432
|
// across different source files share the same audio file.
|
|
@@ -318,7 +436,8 @@ export async function resolveScripts(
|
|
|
318
436
|
// Check cache — skip TTS if script + config unchanged AND audio file exists
|
|
319
437
|
const cached = checkCache(cache, `tts:${cacheKey}`, cacheKey);
|
|
320
438
|
let generated: string;
|
|
321
|
-
const
|
|
439
|
+
const speakerLabel = node.speaker ? `${node.speaker}: ` : "";
|
|
440
|
+
const label = `${speakerLabel}${firstWords(node.script, 8)}`;
|
|
322
441
|
if (cached) {
|
|
323
442
|
generated = cached;
|
|
324
443
|
console.log(` ✓ TTS: ${label} (cached)`);
|
|
@@ -378,15 +497,16 @@ export async function resolveSubtitles(
|
|
|
378
497
|
let cacheDirty = false;
|
|
379
498
|
|
|
380
499
|
// Collect { audioSrc, absoluteOffset } from the compiled tree
|
|
381
|
-
const clips: Array<{ audioSrc: string; offset: number }> = [];
|
|
500
|
+
const clips: Array<{ audioSrc: string; offset: number; speaker?: string }> = [];
|
|
382
501
|
|
|
383
502
|
/**
|
|
384
503
|
* Walk an array of sibling nodes, tracking cumulative offset for series layouts.
|
|
385
504
|
*
|
|
386
|
-
* In the compiled stream tree, container nodes (Folder/Scene) don't carry
|
|
387
|
-
* — their position on the timeline is implicit in the
|
|
388
|
-
* renderer which plays each child sequentially
|
|
389
|
-
* This function replicates that logic to compute
|
|
505
|
+
* In the compiled stream tree, container nodes (Folder/Scene) don't carry leaf
|
|
506
|
+
* timing (start/end) — their position on the timeline is implicit in the
|
|
507
|
+
* `<Series>`/`<TransitionSeries>` renderer which plays each child sequentially
|
|
508
|
+
* based on `durationInSeconds`. This function replicates that logic to compute
|
|
509
|
+
* absolute audio offsets.
|
|
390
510
|
*/
|
|
391
511
|
function walkSiblings(
|
|
392
512
|
nodes: any[],
|
|
@@ -404,13 +524,13 @@ export async function resolveSubtitles(
|
|
|
404
524
|
// In a series, each child starts after all previous siblings' durations.
|
|
405
525
|
// In parallel, all children share the parent offset.
|
|
406
526
|
const nodeStart = parentIsSeries ? seriesOffset : parentOffset;
|
|
407
|
-
// Leaf nodes
|
|
408
|
-
//
|
|
409
|
-
const actionStart = node.
|
|
527
|
+
// Leaf nodes carry a start offset relative to the container start
|
|
528
|
+
// (e.g., parallel layout with staggered start) directly on the base field.
|
|
529
|
+
const actionStart = node.start ?? 0;
|
|
410
530
|
const effectiveOffset = nodeStart + actionStart;
|
|
411
531
|
|
|
412
532
|
if (node.type === "audio" && node.src) {
|
|
413
|
-
clips.push({ audioSrc: node.src, offset: effectiveOffset });
|
|
533
|
+
clips.push({ audioSrc: node.src, offset: effectiveOffset, speaker: node.speaker });
|
|
414
534
|
}
|
|
415
535
|
|
|
416
536
|
// Recurse into children — determine their layout from the node type
|
|
@@ -449,7 +569,7 @@ export async function resolveSubtitles(
|
|
|
449
569
|
}
|
|
450
570
|
}
|
|
451
571
|
|
|
452
|
-
// Walk the compiled tree (with
|
|
572
|
+
// Walk the compiled tree (with base timing) for correct absolute offsets;
|
|
453
573
|
// fall back to the descriptive tree if no compiled tree provided.
|
|
454
574
|
const treeToWalk = options.compiled ?? (clone as any);
|
|
455
575
|
walkSiblings(
|
|
@@ -466,7 +586,7 @@ export async function resolveSubtitles(
|
|
|
466
586
|
|
|
467
587
|
let sttFailedCount = 0;
|
|
468
588
|
|
|
469
|
-
for (const { audioSrc, offset } of clips) {
|
|
589
|
+
for (const { audioSrc, offset, speaker } of clips) {
|
|
470
590
|
// Cache key: audio hash + STT CLI string
|
|
471
591
|
const audioHash = existsSync(audioSrc)
|
|
472
592
|
? createHash("sha1").update(readFileSync(audioSrc)).digest("hex").slice(0, 12)
|
|
@@ -518,7 +638,11 @@ export async function resolveSubtitles(
|
|
|
518
638
|
const sec = (s % 60).toFixed(3).padStart(6, "0");
|
|
519
639
|
return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${sec}`;
|
|
520
640
|
};
|
|
521
|
-
|
|
641
|
+
let text = lines.slice(lines.indexOf(tline) + 1).join("\n").trim();
|
|
642
|
+
// Prepend speaker prefix for dialogue cues
|
|
643
|
+
if (speaker) {
|
|
644
|
+
text = `${speaker}: ${text}`;
|
|
645
|
+
}
|
|
522
646
|
mergedLines.push(String(cueIndex++));
|
|
523
647
|
mergedLines.push(`${formatSec(toSec(a) + offset)} --> ${formatSec(toSec(z) + offset)}`);
|
|
524
648
|
mergedLines.push(text, "");
|
|
@@ -864,6 +988,10 @@ export async function resolveAll(
|
|
|
864
988
|
skip: options.skip,
|
|
865
989
|
});
|
|
866
990
|
|
|
991
|
+
// Step 4.5: Expand multi-turn dialogue (SpeakerName: text format)
|
|
992
|
+
// Must run before TTS so each dialogue line gets its own audio node
|
|
993
|
+
result = resolveDialogue(result);
|
|
994
|
+
|
|
867
995
|
// Step 5: Script → TTS
|
|
868
996
|
if (options.scriptOutputDir) {
|
|
869
997
|
result = await resolveScripts(result, {
|