@rubytech/create-realagent-code 0.1.601 → 0.1.603

package/payload/server/chunk-CYU75QAV.js

@@ -0,0 +1,377 @@
+import { createRequire as __createRequire } from 'node:module'; const require = __createRequire(import.meta.url);
+
+// app/lib/stt/index.ts
+import { execFile } from "child_process";
+import { unlink, stat } from "fs/promises";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var TAG = "[stt]";
+var WHISPER_BINARY = process.env.WHISPER_BINARY ?? "/opt/whisper.cpp/build/bin/whisper-cli";
+var WHISPER_MODEL = process.env.WHISPER_MODEL ?? "/opt/whisper.cpp/models/ggml-base.bin";
+var WHISPER_TIMEOUT_MS = 20 * 60 * 1e3;
+var SPECIAL_TOKEN_RE = /\[_[A-Z]+(?:_\d+)?_?\]/g;
+function stripSpecialTokens(input) {
+  const matches = input.match(SPECIAL_TOKEN_RE);
+  const count = matches ? matches.length : 0;
+  if (!count) return { text: input, count: 0 };
+  const text = input.replace(SPECIAL_TOKEN_RE, "").replace(/[ \t]{2,}/g, " ").trim();
+  return { text, count };
+}
+var SEGMENT_RE = /^\[(\d{2}):(\d{2}):(\d{2})\.(\d{3})\s*-->\s*(\d{2}):(\d{2}):(\d{2})\.(\d{3})\]\s*(.*)$/;
+function toMs(h, m, s, ms) {
+  return ((Number(h) * 60 + Number(m)) * 60 + Number(s)) * 1e3 + Number(ms);
+}
+function parseWhisperSegments(stdout) {
+  const segments = [];
+  for (const line of stdout.split("\n")) {
+    const m = SEGMENT_RE.exec(line.trim());
+    if (!m) continue;
+    const { text } = stripSpecialTokens(m[9].trim());
+    if (!text) continue;
+    segments.push({
+      startMs: toMs(m[1], m[2], m[3], m[4]),
+      endMs: toMs(m[5], m[6], m[7], m[8]),
+      text
+    });
+  }
+  return segments;
+}
+async function transcribeWithTimestamps(audioPath, mimetype) {
+  const startMs = Date.now();
+  const wavPath = audioPath.replace(/\.[^.]+$/, "") + ".video.wav";
+  try {
+    await execFileAsync("ffmpeg", [
+      "-i",
+      audioPath,
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      "-c:a",
+      "pcm_s16le",
+      "-y",
+      wavPath
+    ], { timeout: 6e4 });
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG} timestamps failed: ffmpeg conversion error=${reason}`);
+    return void 0;
+  }
+  try {
+    const { stdout } = await execFileAsync(WHISPER_BINARY, [
+      "-m",
+      WHISPER_MODEL,
+      "-f",
+      wavPath,
+      "-l",
+      "auto"
+    ], { timeout: WHISPER_TIMEOUT_MS, maxBuffer: 10 * 1024 * 1024 });
+    const segments = parseWhisperSegments(stdout);
+    const durationMs = Date.now() - startMs;
+    if (segments.length === 0) {
+      console.error(
+        `${TAG} timestamps: no segments parsed duration_ms=${durationMs} mimetype=${mimetype}`
+      );
+      return void 0;
+    }
+    console.error(
+      `${TAG} timestamps done segments=${segments.length} duration_ms=${durationMs}`
+    );
+    return segments;
+  } catch (err) {
+    const durationMs = Date.now() - startMs;
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG} timestamps failed duration_ms=${durationMs} error=${reason}`);
+    return void 0;
+  } finally {
+    unlink(wavPath).catch(() => {
+    });
+  }
+}
+async function transcribe(audioPath, mimetype) {
+  const startMs = Date.now();
+  let audioBytes = 0;
+  try {
+    const s = await stat(audioPath);
+    audioBytes = s.size;
+  } catch {
+    console.error(`${TAG} failed: file not readable path=${audioPath}`);
+    return void 0;
+  }
+  console.error(
+    `${TAG} start provider=whisper-local audio_bytes=${audioBytes} mimetype=${mimetype} path=${audioPath}`
+  );
+  const wavPath = audioPath.replace(/\.[^.]+$/, "") + ".wav";
+  try {
+    await execFileAsync("ffmpeg", [
+      "-i",
+      audioPath,
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      "-c:a",
+      "pcm_s16le",
+      "-y",
+      // overwrite without prompting
+      wavPath
+    ], { timeout: 3e4 });
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG} failed: ffmpeg conversion error=${reason}`);
+    return void 0;
+  }
+  try {
+    const { stdout, stderr } = await execFileAsync(WHISPER_BINARY, [
+      "-m",
+      WHISPER_MODEL,
+      "-f",
+      wavPath,
+      "--no-timestamps",
+      "-l",
+      "auto"
+    ], { timeout: WHISPER_TIMEOUT_MS, maxBuffer: 10 * 1024 * 1024 });
+    const durationMs = Date.now() - startMs;
+    const { text, count: stripped } = stripSpecialTokens(stdout.trim());
+    if (!text) {
+      console.error(`${TAG} failed: whisper returned empty output duration_ms=${durationMs}`);
+      return void 0;
+    }
+    const langMatch = stderr.match(/auto-detected language:\s*(\w+)/);
+    const language = langMatch?.[1] ?? "unknown";
+    const words = text.split(/\s+/).filter(Boolean).length;
+    console.error(
+      `${TAG} done provider=whisper-local duration_ms=${durationMs} words=${words} stripped=${stripped} lang=${language}`
+    );
+    return { text, language, durationMs };
+  } catch (err) {
+    const durationMs = Date.now() - startMs;
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG} failed provider=whisper-local duration_ms=${durationMs} error=${reason}`);
+    return void 0;
+  } finally {
+    unlink(wavPath).catch(() => {
+    });
+  }
+}
+
+// app/lib/video/frames.ts
+import { execFile as execFile2 } from "child_process";
+import { join } from "path";
+import { promisify as promisify2 } from "util";
+var execFileAsync2 = promisify2(execFile2);
+var TAG2 = "[video]";
+var SCENE_THRESHOLD = 0.3;
+var DENSITY_INTERVAL_S = 4;
+var MIN_FRAMES = 3;
+var MAX_FRAMES = 24;
+var DEDUPE_TOLERANCE_S = 0.5;
+function round3(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+function selectFrameTimestamps(durationSec, sceneTimes) {
+  if (!Number.isFinite(durationSec) || durationSec <= 0) return [];
+  const floorTimes = [];
+  for (let t = 0; t < durationSec; t += DENSITY_INTERVAL_S) floorTimes.push(round3(t));
+  const inRange = sceneTimes.filter((t) => Number.isFinite(t) && t >= 0 && t < durationSec);
+  const sorted = [...floorTimes, ...inRange].sort((a, b) => a - b);
+  const merged = [];
+  for (const t of sorted) {
+    if (merged.length === 0 || t - merged[merged.length - 1] >= DEDUPE_TOLERANCE_S) {
+      merged.push(round3(t));
+    }
+  }
+  if (merged.length < MIN_FRAMES) {
+    const step = durationSec / MIN_FRAMES;
+    return Array.from({ length: MIN_FRAMES }, (_, i) => round3(i * step));
+  }
+  if (merged.length > MAX_FRAMES) {
+    const last = merged.length - 1;
+    const picked = [];
+    for (let i = 0; i < MAX_FRAMES; i++) {
+      picked.push(merged[Math.round(i * last / (MAX_FRAMES - 1))]);
+    }
+    return picked;
+  }
+  return merged;
+}
+var defaultExec = async (file, args) => {
+  const { stdout, stderr } = await execFileAsync2(file, args, {
+    timeout: 12e4,
+    maxBuffer: 10 * 1024 * 1024
+  });
+  return { stdout, stderr };
+};
+var PTS_TIME_RE = /pts_time:([0-9]+(?:\.[0-9]+)?)/g;
+async function detectSceneTimes(path, execImpl = defaultExec) {
+  try {
+    const { stdout, stderr } = await execImpl("ffmpeg", [
+      "-v",
+      "error",
+      "-i",
+      path,
+      "-vf",
+      `select='gt(scene,${SCENE_THRESHOLD})',metadata=print:file=-`,
+      "-an",
+      "-f",
+      "null",
+      "-"
+    ]);
+    const times = [];
+    for (const m of `${stdout}
+${stderr}`.matchAll(PTS_TIME_RE)) {
+      const t = Number(m[1]);
+      if (Number.isFinite(t)) times.push(t);
+    }
+    return times.sort((a, b) => a - b);
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG2} scene detect failed error=${reason} path=${path}`);
+    return [];
+  }
+}
+async function extractFrames(path, timestamps, outDir, execImpl = defaultExec) {
+  const stem = `video-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const refs = [];
+  for (const tSec of timestamps) {
+    const out = join(outDir, `${stem}-${tSec.toFixed(3).replace(".", "_")}.jpg`);
+    try {
+      await execImpl("ffmpeg", [
+        "-v",
+        "error",
+        "-ss",
+        String(tSec),
+        "-i",
+        path,
+        "-frames:v",
+        "1",
+        "-q:v",
+        "4",
+        "-y",
+        out
+      ]);
+      refs.push({ tSec, path: out });
+    } catch (err) {
+      const reason = err instanceof Error ? err.message : String(err);
+      console.error(`${TAG2} frame extract failed t=${tSec} error=${reason}`);
+    }
+  }
+  return refs;
+}
+
+// app/lib/video/probe.ts
+import { execFile as execFile3 } from "child_process";
+import { promisify as promisify3 } from "util";
+var execFileAsync3 = promisify3(execFile3);
+var TAG3 = "[video]";
+var defaultExec2 = async (file, args) => {
+  const { stdout } = await execFileAsync3(file, args, { timeout: 3e4 });
+  return { stdout };
+};
+async function probeVideo(path, execImpl = defaultExec2) {
+  try {
+    const { stdout } = await execImpl("ffprobe", [
+      "-v",
+      "error",
+      "-show_entries",
+      "format=duration",
+      "-show_entries",
+      "stream=codec_type",
+      "-of",
+      "json",
+      path
+    ]);
+    const parsed = JSON.parse(stdout);
+    const durationSec = Number(parsed.format?.duration);
+    if (!Number.isFinite(durationSec) || durationSec <= 0) {
+      console.error(`${TAG3} probe failed: unparseable duration path=${path}`);
+      return void 0;
+    }
+    const kinds = (parsed.streams ?? []).map((s) => s.codec_type);
+    return {
+      durationSec,
+      hasAudio: kinds.includes("audio"),
+      hasVideo: kinds.includes("video")
+    };
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    console.error(`${TAG3} probe failed: ffprobe error=${reason} path=${path}`);
+    return void 0;
+  }
+}
+
+// app/lib/video/timeline.ts
+function stamp(totalSec) {
+  const s = Math.floor(totalSec % 60);
+  const m = Math.floor(totalSec / 60 % 60);
+  const h = Math.floor(totalSec / 3600);
+  const mm = String(m).padStart(2, "0");
+  const ss = String(s).padStart(2, "0");
+  return h > 0 ? `${h}:${mm}:${ss}` : `${mm}:${ss}`;
+}
+function renderTimeline(frames, segments) {
+  const events = [];
+  for (const s of segments) {
+    const tSec = s.startMs / 1e3;
+    events.push({ tSec, rank: 0, line: `[${stamp(tSec)}] ${s.text}` });
+  }
+  for (const f of frames) {
+    events.push({ tSec: f.tSec, rank: 1, line: `[${stamp(f.tSec)}] frame \u2192 ${f.path}` });
+  }
+  events.sort((a, b) => a.tSec - b.tSec || a.rank - b.rank);
+  const header = segments.length > 0 ? `Video timeline \u2014 ${frames.length} frame(s) and the spoken transcript, on the video's own clock.` : `Video timeline \u2014 ${frames.length} frame(s); no speech was transcribed from this video.`;
+  return [
+    header,
+    "Read each frame path below to see that moment.",
+    "",
+    ...events.map((e) => e.line)
+  ].join("\n");
+}
+
+// app/lib/video/index.ts
+var TAG4 = "[video]";
+var MAX_DURATION_S = 300;
+var MEDIA_DIR = "/tmp/maxy-media";
+var defaults = {
+  probe: probeVideo,
+  detectScenes: detectSceneTimes,
+  extract: extractFrames,
+  transcribe: transcribeWithTimestamps,
+  outDir: MEDIA_DIR
+};
+async function extractVideo(path, mimetype, deps = {}) {
+  const d = { ...defaults, ...deps };
+  const startedMs = Date.now();
+  const probe = await d.probe(path);
+  if (!probe) {
+    console.error(`${TAG4} skipped=probe-failed path=${path}`);
+    return void 0;
+  }
+  if (!probe.hasVideo) {
+    console.error(`${TAG4} skipped=no-video-stream path=${path}`);
+    return void 0;
+  }
+  if (probe.durationSec > MAX_DURATION_S) {
+    console.error(
+      `${TAG4} skipped=duration duration_s=${probe.durationSec.toFixed(1)} max_s=${MAX_DURATION_S} path=${path}`
+    );
+    return void 0;
+  }
+  const sceneTimes = await d.detectScenes(path);
+  const timestamps = selectFrameTimestamps(probe.durationSec, sceneTimes);
+  const frames = await d.extract(path, timestamps, d.outDir);
+  if (frames.length === 0) {
+    console.error(`${TAG4} skipped=no-frames path=${path}`);
+    return void 0;
+  }
+  const segments = probe.hasAudio ? await d.transcribe(path, mimetype) : void 0;
+  console.error(
+    `${TAG4} done frames=${frames.length} segments=${segments?.length ?? 0} duration_ms=${Date.now() - startedMs} video_s=${probe.durationSec.toFixed(1)}`
+  );
+  return renderTimeline(frames, segments ?? []);
+}
+
+export {
+  transcribe,
+  extractVideo
+};