chainlesschain 0.47.6 → 0.47.7

package/src/skills/video-editing/extractors/audio-extractor.js

@@ -0,0 +1,212 @@
+/**
+ * audio-extractor.js — 音频解构：ASR + beat/能量分析
+ *
+ * ASR 后端：
+ *  - litellm (通过 LLM provider 的 ASR 能力)
+ *  - whisper_cpp (本地)
+ *  - ffmpeg-subtitle (如果用户已有 .srt 文件直接跳过)
+ *
+ * Beat 分析：
+ *  - madmom Python sidecar (Phase 4)
+ *  - 简化模式：ffmpeg volumedetect + 等间距伪 beat (MVP)
+ */
+
+import { spawn } from "child_process";
+import { promises as fs } from "fs";
+import path from "path";
+
+export async function extractSubtitle(
+  audioPath,
+  outputDir,
+  { existingSrt, llmCall } = {},
+) {
+  const srtPath = path.join(outputDir, "subtitle.srt");
+
+  if (existingSrt) {
+    await fs.copyFile(existingSrt, srtPath);
+    return parseSrt(await fs.readFile(srtPath, "utf-8"));
+  }
+
+  if (llmCall) {
+    const result = await llmCall({ type: "asr", audioPath });
+    if (result && result.srt) {
+      await fs.writeFile(srtPath, result.srt);
+      return parseSrt(result.srt);
+    }
+  }
+
+  return [];
+}
+
+export function parseSrt(content) {
+  const blocks = content.trim().split(/\n\n+/);
+  return blocks
+    .map((block) => {
+      const lines = block.split("\n");
+      if (lines.length < 3) return null;
+      const timeMatch = lines[1].match(
+        /(\d{2}:\d{2}:\d{2}[,.]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,.]\d{3})/,
+      );
+      if (!timeMatch) return null;
+      return {
+        start: srtTimeToSeconds(timeMatch[1]),
+        end: srtTimeToSeconds(timeMatch[2]),
+        text: lines.slice(2).join(" "),
+      };
+    })
+    .filter(Boolean);
+}
+
+function srtTimeToSeconds(t) {
+  const [h, m, rest] = t.split(":");
+  const [s, ms] = rest.replace(",", ".").split(".");
+  return (
+    parseInt(h) * 3600 +
+    parseInt(m) * 60 +
+    parseInt(s) +
+    parseInt(ms || 0) / 1000
+  );
+}
+
+export async function analyzeBeats(
+  audioPath,
+  outputDir,
+  { useMadmom = false } = {},
+) {
+  if (useMadmom) {
+    return analyzeBeatsMadmom(audioPath, outputDir);
+  }
+  return analyzeBeatsSimple(audioPath, outputDir);
+}
+
+async function analyzeBeatsSimple(audioPath, outputDir) {
+  const duration = await getAudioDuration(audioPath);
+  const minSegment = 3.0;
+  const segments = [];
+  let t = 0;
+  let idx = 0;
+  while (t < duration) {
+    const segEnd = Math.min(t + minSegment + Math.random() * 2, duration);
+    segments.push({
+      idx: idx++,
+      start: parseFloat(t.toFixed(3)),
+      end: parseFloat(segEnd.toFixed(3)),
+      label: classifySegment(t, duration),
+    });
+    t = segEnd;
+  }
+  const beatPath = path.join(outputDir, "audio_beats.json");
+  await fs.writeFile(beatPath, JSON.stringify({ duration, segments }, null, 2));
+  return { duration, segments };
+}
+
+function classifySegment(t, total) {
+  const pct = t / total;
+  if (pct < 0.1) return "intro";
+  if (pct < 0.3) return "build-up";
+  if (pct < 0.7) return "chorus";
+  if (pct < 0.9) return "bridge";
+  return "outro";
+}
+
+async function analyzeBeatsMadmom(audioPath, outputDir) {
+  return new Promise((resolve, reject) => {
+    const thisDir = path.dirname(
+      new URL(import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1"),
+    );
+    const script = path.join(
+      thisDir,
+      "..",
+      "..",
+      "..",
+      "..",
+      "scripts",
+      "madmom-beats.py",
+    );
+    const proc = spawn("python", [script, audioPath], {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    let stdout = "";
+    let stderr = "";
+    proc.stdout.on("data", (d) => (stdout += d.toString("utf8")));
+    proc.stderr.on("data", (d) => (stderr += d.toString("utf8")));
+    proc.on("close", async (code) => {
+      if (code !== 0)
+        return reject(
+          new Error(`madmom sidecar exit ${code}: ${stderr.slice(-300)}`),
+        );
+      try {
+        const data = JSON.parse(stdout);
+        if (data.error) return reject(new Error(data.error));
+        const beatPath = path.join(outputDir, "audio_beats.json");
+        await fs.writeFile(beatPath, JSON.stringify(data, null, 2));
+        resolve(data);
+      } catch (e) {
+        reject(new Error(`madmom output parse failed: ${e.message}`));
+      }
+    });
+    proc.on("error", reject);
+  });
+}
+
+export async function getAudioDuration(audioPath) {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "-i",
+      audioPath,
+      "-show_entries",
+      "format=duration",
+      "-v",
+      "quiet",
+      "-of",
+      "csv=p=0",
+    ];
+    const proc = spawn("ffprobe", args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    proc.stdout.on("data", (d) => (stdout += d));
+    proc.on("close", (code) => {
+      if (code !== 0) return reject(new Error("ffprobe failed"));
+      resolve(parseFloat(stdout.trim()) || 0);
+    });
+    proc.on("error", reject);
+  });
+}
+
+export async function captionAudioSegments(segments, { llmCall } = {}) {
+  if (!llmCall) return segments;
+  const result = await llmCall({ type: "audio-caption", segments });
+  return result || segments;
+}
+
+export async function runAudioExtractor(audioPath, outputDir, options = {}) {
+  await fs.mkdir(outputDir, { recursive: true });
+
+  const subtitles = await extractSubtitle(audioPath, outputDir, {
+    existingSrt: options.existingSrt,
+    llmCall: options.llmCall,
+  });
+
+  const beats = await analyzeBeats(audioPath, outputDir, {
+    useMadmom: options.useMadmom,
+  });
+
+  const captionedSegments = await captionAudioSegments(beats.segments, {
+    llmCall: options.llmCall,
+  });
+
+  const captionPath = path.join(outputDir, "audio_caption.json");
+  await fs.writeFile(
+    captionPath,
+    JSON.stringify(
+      {
+        duration: beats.duration,
+        segments: captionedSegments,
+        subtitles,
+      },
+      null,
+      2,
+    ),
+  );
+
+  return { subtitles, beats, captionPath };
+}

package/src/skills/video-editing/extractors/subtitle-extractor.js

@@ -0,0 +1,90 @@
+/**
+ * subtitle-extractor.js — 独立 SRT/VTT 字幕提取 + 角色识别
+ */
+
+import { promises as fs } from "fs";
+import path from "path";
+import { parseSrt } from "./audio-extractor.js";
+
+export async function loadSubtitle(filePath) {
+  const content = await fs.readFile(filePath, "utf-8");
+  const ext = path.extname(filePath).toLowerCase();
+
+  if (ext === ".srt") return parseSrt(content);
+  if (ext === ".vtt") return parseVtt(content);
+
+  throw new Error(`Unsupported subtitle format: ${ext}`);
+}
+
+function parseVtt(content) {
+  const lines = content.replace(/\r\n/g, "\n").split("\n");
+  const entries = [];
+  let i = 0;
+  while (i < lines.length) {
+    if (lines[i].includes("-->")) {
+      const timeMatch = lines[i].match(
+        /(\d{2}:\d{2}:\d{2}\.\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}\.\d{3})/,
+      );
+      if (timeMatch) {
+        const textLines = [];
+        i++;
+        while (i < lines.length && lines[i].trim() !== "") {
+          textLines.push(lines[i].trim());
+          i++;
+        }
+        entries.push({
+          start: vttTimeToSeconds(timeMatch[1]),
+          end: vttTimeToSeconds(timeMatch[2]),
+          text: textLines.join(" "),
+        });
+      }
+    }
+    i++;
+  }
+  return entries;
+}
+
+function vttTimeToSeconds(t) {
+  const [h, m, s] = t.split(":");
+  return parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
+}
+
+export async function identifyCharacters(subtitles, { llmCall } = {}) {
+  if (!llmCall || subtitles.length === 0) {
+    return { speakers: {} };
+  }
+
+  const grouped = {};
+  for (const entry of subtitles) {
+    const speakerMatch = entry.text.match(/^\[([^\]]+)\]\s*/);
+    const speaker = speakerMatch ? speakerMatch[1] : "UNKNOWN";
+    if (!grouped[speaker]) grouped[speaker] = [];
+    grouped[speaker].push(entry.text.replace(/^\[[^\]]+\]\s*/, ""));
+  }
+
+  const result = await llmCall({
+    type: "character-identify",
+    speakers: grouped,
+  });
+
+  return result || { speakers: {} };
+}
+
+export async function runSubtitleExtractor(
+  subtitlePath,
+  outputDir,
+  options = {},
+) {
+  const subtitles = await loadSubtitle(subtitlePath);
+  const characters = await identifyCharacters(subtitles, {
+    llmCall: options.llmCall,
+  });
+
+  const outputPath = path.join(outputDir, "subtitle_analysis.json");
+  await fs.writeFile(
+    outputPath,
+    JSON.stringify({ subtitles, characters }, null, 2),
+  );
+
+  return { subtitles, characters, outputPath };
+}

package/src/skills/video-editing/extractors/video-extractor.js

@@ -0,0 +1,137 @@
+/**
+ * video-extractor.js — 视频解构：ffmpeg 抽帧 + VLM caption
+ *
+ * 输出:
+ *  - frames/      — 抽帧 png (按 VIDEO_FPS)
+ *  - scene.json   — 场景切分 + caption
+ */
+
+import { spawn } from "child_process";
+import { promises as fs } from "fs";
+import path from "path";
+
+const DEFAULT_FPS = 2;
+
+export async function extractFrames(
+  videoPath,
+  outputDir,
+  { fps = DEFAULT_FPS } = {},
+) {
+  const framesDir = path.join(outputDir, "frames");
+  await fs.mkdir(framesDir, { recursive: true });
+
+  return new Promise((resolve, reject) => {
+    const args = [
+      "-i",
+      videoPath,
+      "-vf",
+      `fps=${fps}`,
+      "-q:v",
+      "2",
+      path.join(framesDir, "frame_%06d.png"),
+    ];
+    const proc = spawn("ffmpeg", args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stderr = "";
+    proc.stderr.on("data", (d) => (stderr += d));
+    proc.on("close", (code) => {
+      if (code !== 0)
+        return reject(new Error(`ffmpeg exit ${code}: ${stderr.slice(-500)}`));
+      resolve(framesDir);
+    });
+    proc.on("error", reject);
+  });
+}
+
+export async function detectScenes(
+  videoPath,
+  outputDir,
+  { threshold = 0.3 } = {},
+) {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "-i",
+      videoPath,
+      "-vf",
+      `select='gt(scene,${threshold})',showinfo`,
+      "-vsync",
+      "vfr",
+      "-f",
+      "null",
+      "-",
+    ];
+    const proc = spawn("ffmpeg", args, { stdio: ["ignore", "pipe", "pipe"] });
+    let stderr = "";
+    proc.stderr.on("data", (d) => (stderr += d));
+    proc.on("close", async (code) => {
+      if (code !== 0) return reject(new Error(`scene detect exit ${code}`));
+      const timestamps = parseSceneTimestamps(stderr);
+      const scenePath = path.join(outputDir, "scene.json");
+      await fs.writeFile(
+        scenePath,
+        JSON.stringify({ scenes: timestamps }, null, 2),
+      );
+      resolve(timestamps);
+    });
+    proc.on("error", reject);
+  });
+}
+
+function parseSceneTimestamps(ffmpegStderr) {
+  const scenes = [];
+  const regex = /pts_time:(\d+\.?\d*)/g;
+  let m;
+  while ((m = regex.exec(ffmpegStderr)) !== null) {
+    scenes.push({ time: parseFloat(m[1]) });
+  }
+  return scenes;
+}
+
+export async function captionFrames(
+  framesDir,
+  { llmCall, batchSize = 32 } = {},
+) {
+  const files = (await fs.readdir(framesDir))
+    .filter((f) => f.endsWith(".png"))
+    .sort();
+
+  const captions = [];
+  for (let i = 0; i < files.length; i += batchSize) {
+    const batch = files.slice(i, i + batchSize);
+    const framePaths = batch.map((f) => path.join(framesDir, f));
+
+    if (llmCall) {
+      const result = await llmCall({
+        type: "scene-caption",
+        frames: framePaths,
+        frameIndices: batch.map((_, j) => i + j),
+      });
+      captions.push(...(Array.isArray(result) ? result : [result]));
+    } else {
+      for (const f of batch) {
+        captions.push({ frame: f, caption: null });
+      }
+    }
+  }
+  return captions;
+}
+
+export async function runVideoExtractor(videoPath, outputDir, options = {}) {
+  await fs.mkdir(outputDir, { recursive: true });
+
+  const fps = options.fps || DEFAULT_FPS;
+  const framesDir = await extractFrames(videoPath, outputDir, { fps });
+
+  const scenes = await detectScenes(videoPath, outputDir, {
+    threshold: options.sceneThreshold,
+  });
+
+  const captions = await captionFrames(framesDir, {
+    llmCall: options.llmCall,
+    batchSize: options.batchSize,
+  });
+
+  const captionPath = path.join(outputDir, "video_caption.json");
+  await fs.writeFile(captionPath, JSON.stringify({ fps, captions }, null, 2));
+
+  return { framesDir, scenes, captions, captionPath };
+}

package/src/skills/video-editing/parallel-orchestrator.js

@@ -0,0 +1,212 @@
+/**
+ * parallel-orchestrator.js — Section 级并行 Editor + 冲突检测 + 重跑
+ *
+ * 移植自 CutClaw ParallelShotOrchestrator:
+ *   1. 每个 section 并行 spawn editor 循环
+ *   2. 合并后检测时间区间冲突
+ *   3. 质量打分: 0.6 * protagonist_ratio + 0.4 * duration_accuracy
+ *   4. 输者重跑，注入 forbidden_time_ranges
+ */
+
+import { EventEmitter } from "events";
+
+export class ParallelShotOrchestrator extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.maxConcurrency = options.maxConcurrency || 4;
+    this.maxReruns = options.maxReruns || 3;
+    this.qualityWeights = options.qualityWeights || {
+      protagonist: 0.6,
+      duration: 0.4,
+    };
+  }
+
+  /**
+   * Run editor loops in parallel by section, resolve conflicts, rerun losers.
+   *
+   * @param {object[]} sections - shot_plan.sections
+   * @param {Function} runShotFn - async (section, shot, context) => result
+   * @param {object} context - shared context (assetDir, llmCall, etc.)
+   * @returns {object[]} merged shot_point entries
+   */
+  async run(sections, runShotFn, context) {
+    const allResults = [];
+
+    for (let i = 0; i < sections.length; i += this.maxConcurrency) {
+      const batch = sections.slice(i, i + this.maxConcurrency);
+      this.emit("event", {
+        type: "parallel.batch_start",
+        batchIdx: Math.floor(i / this.maxConcurrency),
+        sectionCount: batch.length,
+        ts: Date.now(),
+      });
+
+      const batchResults = await Promise.all(
+        batch.map((section) => this._runSection(section, runShotFn, context)),
+      );
+
+      allResults.push(...batchResults.flat());
+    }
+
+    const resolved = await this._resolveConflicts(
+      allResults,
+      runShotFn,
+      context,
+    );
+    return resolved;
+  }
+
+  async _runSection(section, runShotFn, context) {
+    const results = [];
+    for (const shot of section.shots || []) {
+      try {
+        const result = await runShotFn(section, shot, {
+          ...context,
+          forbiddenTimeRanges: [],
+        });
+        if (result) {
+          results.push({
+            ...result,
+            section_idx: section.section_idx,
+            shot_idx: shot.shot_idx,
+            target_duration: shot.target_duration || 3,
+          });
+        }
+      } catch (err) {
+        this.emit("event", {
+          type: "parallel.shot_error",
+          section_idx: section.section_idx,
+          shot_idx: shot.shot_idx,
+          error: err.message,
+          ts: Date.now(),
+        });
+      }
+    }
+    return results;
+  }
+
+  async _resolveConflicts(results, runShotFn, context) {
+    let resolved = [...results];
+
+    for (let round = 0; round < this.maxReruns; round++) {
+      const conflicts = detectConflicts(resolved);
+      if (conflicts.length === 0) break;
+
+      this.emit("event", {
+        type: "parallel.conflict_detected",
+        round,
+        conflictCount: conflicts.length,
+        ts: Date.now(),
+      });
+
+      const toRerun = [];
+      for (const [a, b] of conflicts) {
+        const scoreA = this._qualityScore(a);
+        const scoreB = this._qualityScore(b);
+
+        const loser = scoreA >= scoreB ? b : a;
+        const winner = scoreA >= scoreB ? a : b;
+
+        this.emit("event", {
+          type: "parallel.conflict_resolved",
+          winner: {
+            section: winner.section_idx,
+            shot: winner.shot_idx,
+            score: Math.max(scoreA, scoreB),
+          },
+          loser: {
+            section: loser.section_idx,
+            shot: loser.shot_idx,
+            score: Math.min(scoreA, scoreB),
+          },
+          ts: Date.now(),
+        });
+
+        toRerun.push({
+          entry: loser,
+          forbiddenTimeRanges: this._extractTimeRanges(winner),
+        });
+
+        resolved = resolved.filter((r) => r !== loser);
+      }
+
+      for (const { entry, forbiddenTimeRanges } of toRerun) {
+        const section = {
+          section_idx: entry.section_idx,
+          shots: [
+            {
+              shot_idx: entry.shot_idx,
+              target_duration: entry.target_duration,
+            },
+          ],
+          music_segment: entry.music_segment,
+        };
+        try {
+          const rerunResult = await this._runSection(section, runShotFn, {
+            ...context,
+            forbiddenTimeRanges,
+          });
+          resolved.push(...rerunResult);
+        } catch {
+          // skip failed reruns
+        }
+      }
+    }
+
+    return resolved.sort(
+      (a, b) => a.section_idx - b.section_idx || a.shot_idx - b.shot_idx,
+    );
+  }
+
+  _qualityScore(entry) {
+    const protagonist = entry.protagonist_ratio ?? 0.5;
+    const targetDur = entry.target_duration || 3;
+    const actualDur = entry.total_duration || 0;
+    const durAccuracy =
+      targetDur > 0
+        ? 1 - Math.min(Math.abs(actualDur - targetDur) / targetDur, 1)
+        : 0.5;
+
+    return (
+      this.qualityWeights.protagonist * protagonist +
+      this.qualityWeights.duration * durAccuracy
+    );
+  }
+
+  _extractTimeRanges(entry) {
+    return (entry.clips || []).map((c) => ({
+      start: c.start,
+      end: c.end,
+    }));
+  }
+}
+
+/**
+ * Detect time-range overlaps between shot_point entries.
+ * Returns pairs [a, b] where a and b have overlapping clips.
+ */
+export function detectConflicts(results) {
+  const conflicts = [];
+  const allClips = results.map((r) => ({
+    entry: r,
+    ranges: (r.clips || []).map((c) => [c.start, c.end]),
+  }));
+
+  for (let i = 0; i < allClips.length; i++) {
+    for (let j = i + 1; j < allClips.length; j++) {
+      if (rangesOverlap(allClips[i].ranges, allClips[j].ranges)) {
+        conflicts.push([allClips[i].entry, allClips[j].entry]);
+      }
+    }
+  }
+  return conflicts;
+}
+
+function rangesOverlap(rangesA, rangesB) {
+  for (const [aStart, aEnd] of rangesA) {
+    for (const [bStart, bEnd] of rangesB) {
+      if (aStart < bEnd && aEnd > bStart) return true;
+    }
+  }
+  return false;
+}