kiku-agent 1.2.0 → 1.3.1

package/server/agent/youtubeTranscript.js

@@ -0,0 +1,209 @@
+import { spawn } from 'child_process';
+import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ensureYouTubeCookies, isCookieFileValid } from './youtubeAuth.js';
+
+const YTDLP_CMD = process.env.KIKU_YTDLP || 'yt-dlp';
+
+const BASE_SUB_LANGS = ['th', 'en', 'zh-Hans', 'zh', 'zh-Hant', 'zh-CN', 'zh-TW', 'en-US'];
+const LANG_PREFERENCES = {
+  th: ['th', 'th-TH'],
+  en: ['en', 'en-US', 'en-GB'],
+  zh: ['zh-Hans', 'zh-CN', 'zh', 'zh-Hant', 'zh-TW'],
+};
+
+export function extractYouTubeVideoId(input) {
+  if (!input || typeof input !== 'string') return null;
+  const trimmed = input.trim();
+  if (/^[\w-]{11}$/.test(trimmed)) return trimmed;
+  const patterns = [
+    /youtube\.com\/watch\?(?:[^#]*&)?v=([\w-]{11})/i,
+    /youtu\.be\/([\w-]{11})/i,
+    /youtube\.com\/shorts\/([\w-]{11})/i,
+    /youtube\.com\/embed\/([\w-]{11})/i,
+    /youtube\.com\/live\/([\w-]{11})/i,
+  ];
+  for (const re of patterns) {
+    const match = trimmed.match(re);
+    if (match) return match[1];
+  }
+  return null;
+}
+
+
+export function parseSubtitleToText(content) {
+  if (!content || typeof content !== 'string') return '';
+  const text = content.startsWith('\uFEFF') ? content.slice(1) : content;
+  const lines = text.split(/\r?\n/);
+
+  const out = [];
+  const seen = new Set();
+  for (let i = 0; i < lines.length; i++) {
+    let l = lines[i].trim();
+    if (!l) continue;
+    if (/^WEBVTT/i.test(l) || /^(NOTE|STYLE|REGION)\b/i.test(l)) continue;
+    if (l.includes('-->')) continue;
+   
+    if (/^\d+$/.test(l)) {
+      let j = i + 1;
+      while (j < lines.length && !lines[j].trim()) j++;
+      if (j < lines.length && lines[j].includes('-->')) continue;
+    }
+    l = l.replace(/<[^>]+>/g, '');
+    l = l
+      .replace(/&amp;/g, '&')
+      .replace(/&#39;/g, "'")
+      .replace(/&apos;/g, "'")
+      .replace(/&quot;/g, '"')
+      .replace(/&gt;/g, '>')
+      .replace(/&lt;/g, '<')
+      .replace(/&nbsp;/g, ' ');
+    if (!l) continue;
+    const key = l.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(l);
+  }
+  return out.join(' ');
+}
+
+async function resolveCookieArgs() {
+  if (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES)) {
+    return ['--cookies', process.env.KIKU_YT_COOKIES];
+  }
+  if (process.env.KIKU_YT_BROWSER) {
+    return ['--cookies-from-browser', process.env.KIKU_YT_BROWSER];
+  }
+  try {
+    const status = await ensureYouTubeCookies({ silent: true });
+    if (status?.ok) {
+      if (status.cookiesPath) return ['--cookies', status.cookiesPath];
+      if (status.browser) return ['--cookies-from-browser', status.browser];
+    }
+  } catch { }
+  return [];
+}
+
+function runYtDlp(args, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(YTDLP_CMD, args, { windowsHide: true });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL'); } catch { }
+      reject(new Error(`yt-dlp timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    proc.stdout.on('data', d => { stdout += d.toString(); });
+    proc.stderr.on('data', d => { stderr += d.toString(); });
+    proc.on('error', err => { clearTimeout(timer); reject(err); });
+    proc.on('close', code => { clearTimeout(timer); resolve({ code, stdout, stderr }); });
+  });
+}
+
+function listSubtitleFiles(dir) {
+  try {
+    return readdirSync(dir)
+      .map(name => {
+        const match = name.match(/^transcript\.(.+)\.(vtt|srt)$/i);
+        return match ? { file: join(dir, name), lang: match[1] } : null;
+      })
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+function pickBestSubtitleFile(files, prefs) {
+  const score = (entry) => {
+    const lang = entry.lang.toLowerCase();
+    let best = prefs.length > 0 ? 100 : 0;
+    prefs.forEach((p, i) => {
+      const pLower = p.toLowerCase();
+      if (lang === pLower) best = Math.min(best, i);
+      else if (lang.startsWith(pLower)) best = Math.min(best, i + 0.5);
+    });
+    if (/(^|\.)orig$/i.test(lang) || /-orig$/i.test(lang)) best += 10;
+    return best;
+  };
+  return [...files].sort((a, b) => score(a) - score(b))[0];
+}
+
+export async function fetchYoutubeTranscript({ url, videoId, lang, maxChars } = {}) {
+  const id = videoId ? String(videoId).trim() : extractYouTubeVideoId(url);
+  if (!id || !/^[\w-]{11}$/.test(id)) {
+    return {
+      status: 'error',
+      message: 'Provide a valid YouTube URL (watch/youtu.be/shorts/embed/live) or an 11-character videoId.',
+    };
+  }
+
+  const dir = mkdtempSync(join(tmpdir(), 'kiku-transcript-'));
+  try {
+    const prefs = LANG_PREFERENCES[String(lang || '').toLowerCase()] || [];
+    const subLangs = [...new Set([...prefs, ...BASE_SUB_LANGS])].join(',');
+    const baseArgs = [
+      '--skip-download', '--no-warnings', '--quiet',
+      '--write-subs', '--write-auto-subs',
+      '--sub-format', 'vtt/srt/best',
+      '--sub-langs', subLangs,
+      '-o', join(dir, 'transcript.%(ext)s'),
+    ];
+    const watchUrl = `https://www.youtube.com/watch?v=${id}`;
+
+    const cookieArgs = await resolveCookieArgs();
+    let attempt = { code: 1, stdout: '', stderr: '' };
+    try {
+      attempt = await runYtDlp([...baseArgs, ...cookieArgs, watchUrl], 60_000);
+    } catch (err) {
+      attempt = { code: 1, stdout: '', stderr: err.message };
+    }
+
+    let files = listSubtitleFiles(dir);
+    if (files.length === 0 && cookieArgs.length > 0) {
+      try {
+        attempt = await runYtDlp([...baseArgs, watchUrl], 60_000);
+      } catch (err) {
+        attempt = { code: 1, stdout: '', stderr: err.message };
+      }
+      files = listSubtitleFiles(dir);
+    }
+
+    if (files.length === 0) {
+      const blocked = /sign in|confirm you|cookies|age/i.test(attempt.stderr || '');
+      return {
+        status: 'not_found',
+        videoId: id,
+        url: watchUrl,
+        message: blocked
+          ? `YouTube refused the caption request for ${id} (sign-in/cookies or age gate). Set KIKU_YT_COOKIES or KIKU_YT_BROWSER to enable authenticated caption downloads.`
+          : `No captions are available for this video (${id}), or yt-dlp could not reach it. Verify the video exists and has captions enabled.`,
+      };
+    }
+
+    const best = pickBestSubtitleFile(files, prefs);
+    const text = parseSubtitleToText(readFileSync(best.file, 'utf8'));
+    if (!text) {
+      return {
+        status: 'not_found',
+        videoId: id,
+        url: watchUrl,
+        message: `Captions were downloaded for ${id} but contained no readable text.`,
+      };
+    }
+
+    const max = Math.max(500, Math.min(Number(maxChars) || 6000, 20000));
+    const truncated = text.length > max;
+    return {
+      status: 'success',
+      videoId: id,
+      url: watchUrl,
+      lang: best.lang,
+      charCount: text.length,
+      truncated,
+      text: truncated ? `${text.slice(0, max)}…[truncated]` : text,
+    };
+  } finally {
+    try { rmSync(dir, { recursive: true, force: true }); } catch { }
+  }
+}