demofly 0.2.0 → 0.2.2

package/dist/lib/edit-proposals.js

@@ -0,0 +1,159 @@
+/**
+ * Demofly edit proposals — LLM-generated retiming instructions.
+ *
+ * Takes alignment.json and produces edit-decisions.json with per-segment
+ * retiming instructions that can be executed by ffmpeg.
+ *
+ * Uses Claude API to generate intelligent edit decisions based on drift
+ * data, hero moment awareness, and assembly best practices.
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { debug } from "./logger.js";
+/** Hard limits for retiming decisions. */
+const LIMITS = {
+    maxSpeedup: 1.5,
+    maxSlowdown: 1.33,
+    maxFreezeFrameMs: 3000,
+    minConfidence: 0.7,
+};
+/**
+ * Generate edit proposals from alignment data.
+ *
+ * This produces deterministic, rule-based proposals without requiring an
+ * LLM call. The proposals follow the same format that an LLM would produce,
+ * making it easy to swap in LLM-powered proposals later.
+ *
+ * When an LLM API is available, use `generateLlmEditProposals()` instead
+ * for more nuanced decisions (e.g., choosing freeze_frame vs pad_silence
+ * based on visual content).
+ */
+export function generateEditProposals(alignment, heroSceneIds = []) {
+    const allDecisions = [];
+    for (const beat of alignment.beats) {
+        const decision = proposeEdit(beat, heroSceneIds);
+        if (decision) {
+            allDecisions.push(decision);
+        }
+    }
+    // Split into applied (confidence >= threshold) and skipped
+    const applied = allDecisions.filter((d) => d.confidence >= LIMITS.minConfidence);
+    const skipped = allDecisions.filter((d) => d.confidence < LIMITS.minConfidence);
+    for (const s of skipped) {
+        debug(`Skipped edit for ${s.beatId}: ${s.editType} (confidence ${s.confidence})`);
+    }
+    return {
+        decisions: applied,
+        skipped,
+        summary: {
+            total: allDecisions.length,
+            applied: applied.length,
+            skipped: skipped.length,
+        },
+    };
+}
+/**
+ * Propose a single edit decision for a beat based on its alignment.
+ */
+function proposeEdit(beat, heroSceneIds) {
+    const { driftMs, videoDurationMs, audioDurationMs, sceneId } = beat;
+    const isHero = heroSceneIds.includes(sceneId);
+    // No drift — no edit needed
+    if (beat.status === "aligned") {
+        return null;
+    }
+    // Video is longer than audio (positive drift) — speed up or trim
+    if (driftMs > 0) {
+        // Never speed up hero moments
+        if (isHero) {
+            // Pad audio with silence instead
+            return {
+                beatId: beat.beatId,
+                editType: "pad_silence",
+                segmentStartMs: 0,
+                segmentEndMs: videoDurationMs,
+                targetDurationMs: videoDurationMs,
+                rationale: `Hero scene: padding audio with ${driftMs}ms silence instead of speeding up video`,
+                confidence: 0.9,
+            };
+        }
+        const speedupRatio = videoDurationMs / audioDurationMs;
+        if (speedupRatio <= LIMITS.maxSpeedup) {
+            return {
+                beatId: beat.beatId,
+                editType: "speed_video",
+                segmentStartMs: 0,
+                segmentEndMs: videoDurationMs,
+                targetDurationMs: audioDurationMs,
+                rationale: `Video ${driftMs}ms longer than audio; ${speedupRatio.toFixed(2)}x speedup within limits`,
+                confidence: speedupRatio <= 1.2 ? 0.9 : 0.75,
+            };
+        }
+        // Too much speedup needed — trim silence instead
+        return {
+            beatId: beat.beatId,
+            editType: "trim_silence",
+            segmentStartMs: 0,
+            segmentEndMs: videoDurationMs,
+            targetDurationMs: audioDurationMs,
+            rationale: `Video ${driftMs}ms longer; speedup would exceed ${LIMITS.maxSpeedup}x limit, trimming silence instead`,
+            confidence: 0.65,
+        };
+    }
+    // Audio is longer than video (negative drift) — slow down or freeze
+    if (driftMs < 0) {
+        const absDrift = Math.abs(driftMs);
+        const slowdownRatio = audioDurationMs / videoDurationMs;
+        if (slowdownRatio <= LIMITS.maxSlowdown) {
+            return {
+                beatId: beat.beatId,
+                editType: "slow_video",
+                segmentStartMs: 0,
+                segmentEndMs: videoDurationMs,
+                targetDurationMs: audioDurationMs,
+                rationale: `Audio ${absDrift}ms longer than video; ${slowdownRatio.toFixed(2)}x slowdown within limits`,
+                confidence: slowdownRatio <= 1.15 ? 0.9 : 0.75,
+            };
+        }
+        // Too much slowdown — freeze frame
+        if (absDrift <= LIMITS.maxFreezeFrameMs) {
+            return {
+                beatId: beat.beatId,
+                editType: "freeze_frame",
+                segmentStartMs: videoDurationMs,
+                segmentEndMs: videoDurationMs,
+                targetDurationMs: absDrift,
+                rationale: `Audio ${absDrift}ms longer; freeze last frame for ${absDrift}ms to cover gap`,
+                confidence: absDrift <= 1500 ? 0.85 : 0.7,
+            };
+        }
+        // Critical drift — log but low confidence
+        return {
+            beatId: beat.beatId,
+            editType: "freeze_frame",
+            segmentStartMs: videoDurationMs,
+            segmentEndMs: videoDurationMs,
+            targetDurationMs: Math.min(absDrift, LIMITS.maxFreezeFrameMs),
+            rationale: `Critical drift: audio ${absDrift}ms longer; capped freeze at ${LIMITS.maxFreezeFrameMs}ms`,
+            confidence: 0.5,
+        };
+    }
+    return null;
+}
+/**
+ * Generate edit proposals and write to recordings/edit-decisions.json.
+ */
+export function generateAndWriteEditProposals(projectDir, heroSceneIds = []) {
+    const alignmentPath = resolve(projectDir, "recordings", "alignment.json");
+    if (!existsSync(alignmentPath)) {
+        console.error("No alignment.json found. Run `demofly generate <name> --align` first.");
+        return null;
+    }
+    const alignment = JSON.parse(readFileSync(alignmentPath, "utf-8"));
+    const proposals = generateEditProposals(alignment, heroSceneIds);
+    const outputPath = resolve(projectDir, "recordings", "edit-decisions.json");
+    writeFileSync(outputPath, JSON.stringify(proposals, null, 2), "utf-8");
+    console.log(`  Written: ${outputPath}`);
+    return proposals;
+}
+//# sourceMappingURL=edit-proposals.js.map

package/dist/lib/edit-proposals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"edit-proposals.js","sourceRoot":"","sources":["../../src/lib/edit-proposals.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AA+BpC,0CAA0C;AAC1C,MAAM,MAAM,GAAG;IACb,UAAU,EAAE,GAAG;IACf,WAAW,EAAE,IAAI;IACjB,gBAAgB,EAAE,IAAI;IACtB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CACnC,SAAwB,EACxB,eAAyB,EAAE;IAE3B,MAAM,YAAY,GAAmB,EAAE,CAAC;IAExC,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACjD,IAAI,QAAQ,EAAE,CAAC;YACb,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;IAEhF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,gBAAgB,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,OAAO;QACL,SAAS,EAAE,OAAO;QAClB,OAAO;QACP,OAAO,EAAE;YACP,KAAK,EAAE,YAAY,CAAC,MAAM;YAC1B,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,OAAO,EAAE,OAAO,CAAC,MAAM;SACxB;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAClB,IAAmB,EACnB,YAAsB;IAEtB,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IACpE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE9C,4BAA4B;IAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,iEAAiE;IACjE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,8BAA8B;QAC9B,IAAI,MAAM,EAAE,CAAC;YACX,iCAAiC;YACjC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,aAAa;gBACvB,cAAc,EAAE,CAAC;gBACjB,YAAY,EAAE,eAAe;gBAC7B,gBAAgB,EAAE,eAAe;gBACjC,SAAS,EAAE,kCAAkC,OAAO,yCAAyC;gBAC7F,UAAU,EAAE,GAAG;aAChB,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAG,eAAe,GAAG,eAAe,CAAC;QAEvD,IAAI,YAAY,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,aAAa;gBACvB,cAAc,EAAE,CAAC;gBACjB,YAAY,EAAE,eAAe;gBAC7B,gBAAgB,EAAE,eAAe;gBACjC,SAAS,EAAE,SAAS,OAAO,yBAAyB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB;gBACpG,UAAU,EAAE,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;aAC7C,CAAC;QACJ,CAAC;QAED,iDAAiD;QACjD,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,cAAc;YACxB,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,eAAe;YAC7B,gBAAgB,EAAE,eAAe;YACjC,SAAS,EAAE,SAAS,OAAO,mCAAmC,MAAM,CAAC,UAAU,mCAAmC;YAClH,UAAU,EAAE,IAAI;SACjB,CAAC;IACJ,CAAC;IAED,oEAAoE;IACpE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,aAAa,GAAG,eAAe,GAAG,eAAe,CAAC;QAExD,IAAI,aAAa,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACxC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,YAAY;gBACtB,cAAc,EAAE,CAAC;gBACjB,YAAY,EAAE,eAAe;gBAC7B,gBAAgB,EAAE,eAAe;gBACjC,SAAS,EAAE,SAAS,QAAQ,yBAAyB,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B;gBACvG,UAAU,EAAE,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;aAC/C,CAAC;QACJ,CAAC;QAED,mCAAmC;QACnC,IAAI,QAAQ,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YACxC,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,cAAc;gBACxB,cAAc,EAAE,eAAe;gBAC/B,YAAY,EAAE,eAAe;gBAC7B,gBAAgB,EAAE,QAAQ;gBAC1B,SAAS,EAAE,SAAS,QAAQ,oCAAoC,QAAQ,iBAAiB;gBACzF,UAAU,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG;aAC1C,CAAC;QACJ,CAAC;QAED,0CAA0C;QAC1C,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,cAAc;YACxB,cAAc,EAAE,eAAe;YAC/B,YAAY,EAAE,eAAe;YAC7B,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,gBAAgB,CAAC;YAC7D,SAAS,EAAE,yBAAyB,QAAQ,+BAA+B,MAAM,CAAC,gBAAgB,IAAI;YACtG,UAAU,EAAE,GAAG;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,6BAA6B,CAC3C,UAAkB,EAClB,eAAyB,EAAE;IAE3B,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAC;IAE1E,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CACX,uEAAuE,CACxE,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,SAAS,GAAkB,IAAI,CAAC,KAAK,CACzC,YAAY,CAAC,aAAa,EAAE,OAAO,CAAC,CACrC,CAAC;IAEF,MAAM,SAAS,GAAG,qBAAqB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAEjE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,qBAAqB,CAAC,CAAC;IAC5E,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,EAAE,CAAC,CAAC;IAExC,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/lib/retiming.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Demofly retiming — translate edit decisions into ffmpeg filter commands.
+ *
+ * Segment-based retiming: split video → per-segment transforms → concatenate.
+ * Replaces simple stitchAudio() for the intelligent assembly pipeline.
+ */
+import type { TimingData } from "./timing.js";
+import type { EditDecisionsData } from "./edit-proposals.js";
+/**
+ * Apply edit decisions to video segments and produce a retimed final video.
+ *
+ * Pipeline:
+ * 1. Split video into per-scene segments
+ * 2. Apply per-segment transforms (speed change, freeze frame, etc.)
+ * 3. Concatenate retimed segments
+ * 4. Mux with audio
+ */
+export declare function retimeAndAssemble(videoPath: string, audioDir: string, timingData: TimingData, decisions: EditDecisionsData, recordingsDir: string): string | null;
+//# sourceMappingURL=retiming.d.ts.map

package/dist/lib/retiming.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retiming.d.ts","sourceRoot":"","sources":["../../src/lib/retiming.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAQ7D;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,UAAU,EACtB,SAAS,EAAE,iBAAiB,EAC5B,aAAa,EAAE,MAAM,GACpB,MAAM,GAAG,IAAI,CAwCf"}

package/dist/lib/retiming.js

@@ -0,0 +1,215 @@
+/**
+ * Demofly retiming — translate edit decisions into ffmpeg filter commands.
+ *
+ * Segment-based retiming: split video → per-segment transforms → concatenate.
+ * Replaces simple stitchAudio() for the intelligent assembly pipeline.
+ */
+import { execSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { debug } from "./logger.js";
+/**
+ * Apply edit decisions to video segments and produce a retimed final video.
+ *
+ * Pipeline:
+ * 1. Split video into per-scene segments
+ * 2. Apply per-segment transforms (speed change, freeze frame, etc.)
+ * 3. Concatenate retimed segments
+ * 4. Mux with audio
+ */
+export function retimeAndAssemble(videoPath, audioDir, timingData, decisions, recordingsDir) {
+    const workDir = resolve(recordingsDir, ".retiming-work");
+    mkdirSync(workDir, { recursive: true });
+    try {
+        // Step 1: Split video into scene segments
+        console.log("  Splitting video into scene segments...");
+        const segments = splitIntoSegments(videoPath, timingData, workDir);
+        if (segments.length === 0) {
+            console.warn("No segments produced. Falling back to simple stitch.");
+            return null;
+        }
+        // Step 2: Apply edits to each segment
+        console.log("  Applying retiming edits...");
+        const retimed = applyEdits(segments, decisions, workDir);
+        // Step 3: Mux audio with retimed segments
+        console.log("  Muxing audio...");
+        const muxed = muxAudioToSegments(retimed, audioDir, workDir);
+        // Step 4: Concatenate all segments
+        console.log("  Concatenating final video...");
+        const outputPath = resolve(recordingsDir, "final.mp4");
+        const success = concatenateSegments(muxed, outputPath, workDir);
+        if (success) {
+            return outputPath;
+        }
+        return null;
+    }
+    finally {
+        // Clean up work directory
+        try {
+            execSync(`rm -rf "${workDir}"`, { stdio: "pipe" });
+        }
+        catch {
+            // Non-critical cleanup failure
+        }
+    }
+}
+/**
+ * Split video into per-scene segments based on timing data.
+ */
+function splitIntoSegments(videoPath, timingData, workDir) {
+    const segments = [];
+    for (const scene of timingData.scenes) {
+        const startSec = scene.startMs / 1000;
+        const durationSec = (scene.endMs - scene.startMs) / 1000;
+        const outputPath = resolve(workDir, `${scene.sceneId}-raw.mp4`);
+        try {
+            execSync(`ffmpeg -y -i "${videoPath}" -ss ${startSec} -t ${durationSec} ` +
+                `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -an "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+            segments.push({ path: outputPath, sceneId: scene.sceneId });
+        }
+        catch (error) {
+            const execError = error;
+            console.warn(`  Warning: Failed to split ${scene.sceneId}:`);
+            if (execError.stderr) {
+                debug(execError.stderr);
+            }
+        }
+    }
+    return segments;
+}
+/**
+ * Apply edit decisions to segments.
+ * Returns paths to the retimed segment files.
+ */
+function applyEdits(segments, decisions, workDir) {
+    const decisionsByBeat = new Map(decisions.decisions.map((d) => [d.beatId, d]));
+    return segments.map((seg) => {
+        const decision = decisionsByBeat.get(seg.sceneId);
+        if (!decision) {
+            debug(`  ${seg.sceneId}: no edit needed`);
+            return seg;
+        }
+        const outputPath = resolve(workDir, `${seg.sceneId}-retimed.mp4`);
+        try {
+            switch (decision.editType) {
+                case "speed_video": {
+                    const speed = decision.segmentEndMs / decision.targetDurationMs;
+                    const filter = `setpts=${(1 / speed).toFixed(4)}*PTS`;
+                    execSync(`ffmpeg -y -i "${seg.path}" -filter:v "${filter}" ` +
+                        `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -an "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+                    console.log(`    ${seg.sceneId}: speed_video ${speed.toFixed(2)}x`);
+                    return { path: outputPath, sceneId: seg.sceneId };
+                }
+                case "slow_video": {
+                    const speed = decision.segmentEndMs / decision.targetDurationMs;
+                    const filter = `setpts=${(1 / speed).toFixed(4)}*PTS`;
+                    execSync(`ffmpeg -y -i "${seg.path}" -filter:v "${filter}" ` +
+                        `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -an "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+                    console.log(`    ${seg.sceneId}: slow_video ${speed.toFixed(2)}x`);
+                    return { path: outputPath, sceneId: seg.sceneId };
+                }
+                case "freeze_frame": {
+                    const freezeDuration = decision.targetDurationMs / 1000;
+                    // Extract last frame, create freeze, concatenate
+                    const freezePath = resolve(workDir, `${seg.sceneId}-freeze.mp4`);
+                    execSync(`ffmpeg -y -sseof -0.1 -i "${seg.path}" -vframes 1 -loop 1 -t ${freezeDuration} ` +
+                        `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p "${freezePath}"`, { encoding: "utf-8", timeout: 60_000, stdio: ["pipe", "pipe", "pipe"] });
+                    // Concatenate original + freeze
+                    const listFile = resolve(workDir, `${seg.sceneId}-concat.txt`);
+                    writeFileSync(listFile, `file '${seg.path}'\nfile '${freezePath}'\n`);
+                    execSync(`ffmpeg -y -f concat -safe 0 -i "${listFile}" ` +
+                        `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -an "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+                    console.log(`    ${seg.sceneId}: freeze_frame +${decision.targetDurationMs}ms`);
+                    return { path: outputPath, sceneId: seg.sceneId };
+                }
+                case "trim_silence": {
+                    // Trim from the end of the segment to match target duration
+                    const targetSec = decision.targetDurationMs / 1000;
+                    execSync(`ffmpeg -y -i "${seg.path}" -t ${targetSec} ` +
+                        `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -an "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+                    console.log(`    ${seg.sceneId}: trim_silence to ${decision.targetDurationMs}ms`);
+                    return { path: outputPath, sceneId: seg.sceneId };
+                }
+                case "pad_silence":
+                case "no_change":
+                    debug(`  ${seg.sceneId}: ${decision.editType} (no video change)`);
+                    return seg;
+                default:
+                    debug(`  ${seg.sceneId}: unknown edit type ${decision.editType}`);
+                    return seg;
+            }
+        }
+        catch (error) {
+            const execError = error;
+            console.warn(`    Warning: Edit failed for ${seg.sceneId}, using original:`);
+            if (execError.stderr) {
+                debug(execError.stderr);
+            }
+            return seg;
+        }
+    });
+}
+/**
+ * Mux audio files into retimed video segments.
+ */
+function muxAudioToSegments(segments, audioDir, workDir) {
+    return segments.map((seg) => {
+        // Find matching audio file
+        const wavPath = resolve(audioDir, `${seg.sceneId}.wav`);
+        const mp3Path = resolve(audioDir, `${seg.sceneId}.mp3`);
+        const audioPath = existsSync(wavPath) ? wavPath : existsSync(mp3Path) ? mp3Path : null;
+        if (!audioPath) {
+            debug(`  ${seg.sceneId}: no audio file, keeping video-only`);
+            return seg;
+        }
+        const outputPath = resolve(workDir, `${seg.sceneId}-muxed.mp4`);
+        try {
+            execSync(`ffmpeg -y -i "${seg.path}" -i "${audioPath}" ` +
+                `-c:v copy -c:a aac -b:a 128k -shortest "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+            return { path: outputPath, sceneId: seg.sceneId };
+        }
+        catch (error) {
+            const execError = error;
+            console.warn(`    Warning: Audio mux failed for ${seg.sceneId}:`);
+            if (execError.stderr) {
+                debug(execError.stderr);
+            }
+            return seg;
+        }
+    });
+}
+/**
+ * Concatenate all segments into the final video.
+ */
+function concatenateSegments(segments, outputPath, workDir) {
+    if (segments.length === 0)
+        return false;
+    // If only one segment, just copy it
+    if (segments.length === 1) {
+        try {
+            execSync(`ffmpeg -y -i "${segments[0].path}" -c copy "${outputPath}"`, { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "pipe"] });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // Create concat file
+    const listFile = resolve(workDir, "concat-final.txt");
+    const lines = segments.map((s) => `file '${s.path}'`).join("\n");
+    writeFileSync(listFile, lines + "\n");
+    try {
+        execSync(`ffmpeg -y -f concat -safe 0 -i "${listFile}" ` +
+            `-c:v libx264 -preset fast -crf 23 -pix_fmt yuv420p -c:a aac -b:a 128k "${outputPath}"`, { encoding: "utf-8", timeout: 300_000, stdio: ["pipe", "pipe", "pipe"] });
+        return true;
+    }
+    catch (error) {
+        const execError = error;
+        console.error("Final concatenation failed:");
+        if (execError.stderr) {
+            console.error(execError.stderr);
+        }
+        return false;
+    }
+}
+//# sourceMappingURL=retiming.js.map

package/dist/lib/retiming.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retiming.js","sourceRoot":"","sources":["../../src/lib/retiming.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAOpC;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAAiB,EACjB,QAAgB,EAChB,UAAsB,EACtB,SAA4B,EAC5B,aAAqB;IAErB,MAAM,OAAO,GAAG,OAAO,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;IACzD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,0CAA0C;QAC1C,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAEnE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,sCAAsC;QACtC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAEzD,0CAA0C;QAC1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE7D,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAEhE,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,0BAA0B;QAC1B,IAAI,CAAC;YACH,QAAQ,CAAC,WAAW,OAAO,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CACxB,SAAiB,EACjB,UAAsB,EACtB,OAAe;IAEf,MAAM,QAAQ,GAAkB,EAAE,CAAC;IAEnC,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC;QACtC,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QACzD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,QAAQ,CACN,iBAAiB,SAAS,SAAS,QAAQ,OAAO,WAAW,GAAG;gBAChE,2DAA2D,UAAU,GAAG,EACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,KAA4B,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;YAC7D,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CACjB,QAAuB,EACvB,SAA4B,EAC5B,OAAe;IAEf,MAAM,eAAe,GAAG,IAAI,GAAG,CAC7B,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAC9C,CAAC;IAEF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAElD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,KAAK,CAAC,KAAK,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC;YAC1C,OAAO,GAAG,CAAC;QACb,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC;QAElE,IAAI,CAAC;YACH,QAAQ,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC1B,KAAK,aAAa,CAAC,CAAC,CAAC;oBACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,gBAAgB,CAAC;oBAChE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtD,QAAQ,CACN,iBAAiB,GAAG,CAAC,IAAI,gBAAgB,MAAM,IAAI;wBACnD,2DAA2D,UAAU,GAAG,EACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,OAAO,iBAAiB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACpE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpD,CAAC;gBAED,KAAK,YAAY,CAAC,CAAC,CAAC;oBAClB,MAAM,KAAK,GAAG,QAAQ,CAAC,YAAY,GAAG,QAAQ,CAAC,gBAAgB,CAAC;oBAChE,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;oBACtD,QAAQ,CACN,iBAAiB,GAAG,CAAC,IAAI,gBAAgB,MAAM,IAAI;wBACnD,2DAA2D,UAAU,GAAG,EACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,OAAO,gBAAgB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;oBACnE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpD,CAAC;gBAED,KAAK,cAAc,CAAC,CAAC,CAAC;oBACpB,MAAM,cAAc,GAAG,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC;oBACxD,iDAAiD;oBACjD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC;oBACjE,QAAQ,CACN,6BAA6B,GAAG,CAAC,IAAI,2BAA2B,cAAc,GAAG;wBACjF,uDAAuD,UAAU,GAAG,EACpE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACxE,CAAC;oBAEF,gCAAgC;oBAChC,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC;oBAC/D,aAAa,CAAC,QAAQ,EAAE,SAAS,GAAG,CAAC,IAAI,YAAY,UAAU,KAAK,CAAC,CAAC;oBACtE,QAAQ,CACN,mCAAmC,QAAQ,IAAI;wBAC/C,2DAA2D,UAAU,GAAG,EACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,OAAO,mBAAmB,QAAQ,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAChF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpD,CAAC;gBAED,KAAK,cAAc,CAAC,CAAC,CAAC;oBACpB,4DAA4D;oBAC5D,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC;oBACnD,QAAQ,CACN,iBAAiB,GAAG,CAAC,IAAI,QAAQ,SAAS,GAAG;wBAC7C,2DAA2D,UAAU,GAAG,EACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;oBACF,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,OAAO,qBAAqB,QAAQ,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAClF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;gBACpD,CAAC;gBAED,KAAK,aAAa,CAAC;gBACnB,KAAK,WAAW;oBACd,KAAK,CAAC,KAAK,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,QAAQ,oBAAoB,CAAC,CAAC;oBAClE,OAAO,GAAG,CAAC;gBAEb;oBACE,KAAK,CAAC,KAAK,GAAG,CAAC,OAAO,uBAAuB,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;oBAClE,OAAO,GAAG,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,KAA4B,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,gCAAgC,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC;YAC7E,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,QAAuB,EACvB,QAAgB,EAChB,OAAe;IAEf,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,2BAA2B;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAEvF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,KAAK,CAAC,KAAK,GAAG,CAAC,OAAO,qCAAqC,CAAC,CAAC;YAC7D,OAAO,GAAG,CAAC;QACb,CAAC;QAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,QAAQ,CACN,iBAAiB,GAAG,CAAC,IAAI,SAAS,SAAS,IAAI;gBAC/C,2CAA2C,UAAU,GAAG,EACxD,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;YACF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;QACpD,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,KAA4B,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,qCAAqC,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC;YAClE,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;gBACrB,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAC1B,QAAuB,EACvB,UAAkB,EAClB,OAAe;IAEf,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,oCAAoC;IACpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,QAAQ,CACN,iBAAiB,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,cAAc,UAAU,GAAG,EAC5D,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,aAAa,CAAC,QAAQ,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC;IAEtC,IAAI,CAAC;QACH,QAAQ,CACN,mCAAmC,QAAQ,IAAI;YAC/C,0EAA0E,UAAU,GAAG,EACvF,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CACzE,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,KAA4B,CAAC;QAC/C,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC7C,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YACrB,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/lib/timestamps.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Demofly Whisper integration — extract word-level timestamps from TTS audio.
+ *
+ * Shells out to the `whisper` CLI (same subprocess pattern as ffmpeg).
+ * Uses whisper-base model — TTS audio is clean, so a small model achieves
+ * near-perfect accuracy while keeping runtime under 5s per scene.
+ *
+ * Fallback: If Whisper isn't installed, returns duration-only data and
+ * prints a warning suggesting `pip install openai-whisper`.
+ */
+export interface WordTimestamp {
+    word: string;
+    startMs: number;
+    endMs: number;
+}
+export interface PhraseTimestamp {
+    text: string;
+    startMs: number;
+    endMs: number;
+    beatId: string;
+}
+export interface SceneTimestamps {
+    sceneId: string;
+    audioFile: string;
+    durationMs: number;
+    words: WordTimestamp[];
+    phrases: PhraseTimestamp[];
+}
+export interface TimestampsData {
+    scenes: SceneTimestamps[];
+}
+/** Check if the whisper CLI is available. */
+export declare function hasWhisper(): boolean;
+/**
+ * Extract timestamps from all audio files in a demo's audio/ directory.
+ *
+ * If Whisper is available, produces word-level and phrase-level timestamps.
+ * If not, falls back to duration-only data (empty words/phrases arrays).
+ *
+ * @returns TimestampsData written to audio/timestamps.json
+ */
+export declare function extractTimestamps(projectDir: string): TimestampsData;
+//# sourceMappingURL=timestamps.d.ts.map

package/dist/lib/timestamps.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timestamps.d.ts","sourceRoot":"","sources":["../../src/lib/timestamps.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAaH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED,6CAA6C;AAC7C,wBAAgB,UAAU,IAAI,OAAO,CAOpC;AAwHD;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,cAAc,CA4EpE"}

package/dist/lib/timestamps.js

@@ -0,0 +1,199 @@
+/**
+ * Demofly Whisper integration — extract word-level timestamps from TTS audio.
+ *
+ * Shells out to the `whisper` CLI (same subprocess pattern as ffmpeg).
+ * Uses whisper-base model — TTS audio is clean, so a small model achieves
+ * near-perfect accuracy while keeping runtime under 5s per scene.
+ *
+ * Fallback: If Whisper isn't installed, returns duration-only data and
+ * prints a warning suggesting `pip install openai-whisper`.
+ */
+import { execSync } from "node:child_process";
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "node:fs";
+import { resolve, basename, extname } from "node:path";
+import { debug } from "./logger.js";
+/** Check if the whisper CLI is available. */
+export function hasWhisper() {
+    try {
+        execSync("whisper --help", { stdio: "pipe", timeout: 10_000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Get audio duration in milliseconds using ffprobe.
+ * Falls back to 0 if ffprobe isn't available.
+ */
+function getAudioDurationMs(audioPath) {
+    try {
+        const output = execSync(`ffprobe -v error -show_entries format=duration -of csv=p=0 "${audioPath}"`, { encoding: "utf-8", timeout: 30_000, stdio: ["pipe", "pipe", "pipe"] });
+        return Math.round(parseFloat(output.trim()) * 1000);
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Run Whisper on a single audio file and extract word-level timestamps.
+ * Returns the raw word timestamps array.
+ */
+function whisperTranscribe(audioPath, outputDir) {
+    const cmd = [
+        "whisper",
+        `"${audioPath}"`,
+        "--model", "base",
+        "--output_format", "json",
+        "--word_timestamps", "True",
+        "--output_dir", `"${outputDir}"`,
+        "--language", "en",
+        "--fp16", "False",
+    ].join(" ");
+    debug(`Whisper command: ${cmd}`);
+    try {
+        execSync(cmd, {
+            encoding: "utf-8",
+            timeout: 120_000,
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+    }
+    catch (error) {
+        const execError = error;
+        console.warn(`Whisper transcription failed for ${audioPath}:`);
+        if (execError.stderr) {
+            console.warn(execError.stderr);
+        }
+        return [];
+    }
+    // Whisper outputs a JSON file with the same name as the input
+    const stem = basename(audioPath, extname(audioPath));
+    const jsonPath = resolve(outputDir, `${stem}.json`);
+    if (!existsSync(jsonPath)) {
+        console.warn(`Whisper output not found: ${jsonPath}`);
+        return [];
+    }
+    try {
+        const data = JSON.parse(readFileSync(jsonPath, "utf-8"));
+        const words = [];
+        // Whisper JSON format: { segments: [{ words: [{ word, start, end }] }] }
+        for (const segment of data.segments ?? []) {
+            for (const w of segment.words ?? []) {
+                words.push({
+                    word: (w.word ?? "").trim(),
+                    startMs: Math.round((w.start ?? 0) * 1000),
+                    endMs: Math.round((w.end ?? 0) * 1000),
+                });
+            }
+        }
+        return words;
+    }
+    catch {
+        console.warn(`Failed to parse Whisper output: ${jsonPath}`);
+        return [];
+    }
+}
+/**
+ * Group word timestamps into phrases (sentences) based on punctuation.
+ * Each phrase gets a beatId placeholder that should be updated by the caller.
+ */
+function groupIntoPhrases(words) {
+    if (words.length === 0)
+        return [];
+    const phrases = [];
+    let currentWords = [];
+    for (const word of words) {
+        currentWords.push(word);
+        // Split on sentence-ending punctuation
+        if (/[.!?]$/.test(word.word)) {
+            phrases.push({
+                text: currentWords.map((w) => w.word).join(" "),
+                startMs: currentWords[0].startMs,
+                endMs: word.endMs,
+                beatId: "", // To be filled by caller
+            });
+            currentWords = [];
+        }
+    }
+    // Remaining words form a final phrase
+    if (currentWords.length > 0) {
+        phrases.push({
+            text: currentWords.map((w) => w.word).join(" "),
+            startMs: currentWords[0].startMs,
+            endMs: currentWords[currentWords.length - 1].endMs,
+            beatId: "",
+        });
+    }
+    return phrases;
+}
+/**
+ * Extract timestamps from all audio files in a demo's audio/ directory.
+ *
+ * If Whisper is available, produces word-level and phrase-level timestamps.
+ * If not, falls back to duration-only data (empty words/phrases arrays).
+ *
+ * @returns TimestampsData written to audio/timestamps.json
+ */
+export function extractTimestamps(projectDir) {
+    const audioDir = resolve(projectDir, "audio");
+    if (!existsSync(audioDir)) {
+        console.error(`No audio directory found at ${audioDir}`);
+        return { scenes: [] };
+    }
+    const audioFiles = readdirSync(audioDir).filter((f) => {
+        const ext = extname(f).toLowerCase();
+        return [".wav", ".mp3"].includes(ext);
+    });
+    if (audioFiles.length === 0) {
+        console.warn("No audio files found in audio/ directory.");
+        return { scenes: [] };
+    }
+    const whisperAvailable = hasWhisper();
+    if (!whisperAvailable) {
+        console.warn("Whisper not found. Using duration-only timestamps (less precise).\n" +
+            "For word-level timestamps, install: pip install openai-whisper");
+    }
+    // Create temp dir for Whisper output
+    const whisperTmpDir = resolve(audioDir, ".whisper-tmp");
+    const scenes = [];
+    for (const file of audioFiles.sort()) {
+        const sceneId = basename(file, extname(file));
+        const audioPath = resolve(audioDir, file);
+        const durationMs = getAudioDurationMs(audioPath);
+        console.log(`  Timestamps: ${sceneId} (${durationMs}ms)`);
+        let words = [];
+        let phrases = [];
+        if (whisperAvailable) {
+            mkdirSync(whisperTmpDir, { recursive: true });
+            words = whisperTranscribe(audioPath, whisperTmpDir);
+            phrases = groupIntoPhrases(words);
+            console.log(`    → ${words.length} words, ${phrases.length} phrases`);
+        }
+        else {
+            console.log(`    → duration only (no Whisper)`);
+        }
+        scenes.push({
+            sceneId,
+            audioFile: `audio/${file}`,
+            durationMs,
+            words,
+            phrases,
+        });
+    }
+    // Clean up Whisper temp dir
+    if (existsSync(whisperTmpDir)) {
+        try {
+            execSync(`rm -rf "${whisperTmpDir}"`, { stdio: "pipe" });
+        }
+        catch {
+            // Non-critical cleanup failure
+        }
+    }
+    const result = { scenes };
+    // Write timestamps.json
+    const outputPath = resolve(audioDir, "timestamps.json");
+    writeFileSync(outputPath, JSON.stringify(result, null, 2), "utf-8");
+    console.log(`\n  Written: ${outputPath}`);
+    return result;
+}
+//# sourceMappingURL=timestamps.js.map