kiku-agent 1.2.0 → 1.3.0

package/server/agent/youtubeAuth.js

@@ -1,24 +1,32 @@
 import fs from 'fs';
 import path from 'path';
-import { spawn, execSync } from 'child_process';
+import { spawn, execFile } from 'child_process';
+import { promisify } from 'util';
 import { fileURLToPath } from 'url';
 import { KIKU_DATA_DIR } from './appPaths.js';
+import { PACKAGE_ENV_PATH, USER_ENV_PATH, upsertEnvKey } from './envManager.js';
+import { bootstrapEnv } from './envBootstrap.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const ENV_PATH = path.resolve(__dirname, '..', '..', '.env');
+
+bootstrapEnv();
+
+const execFileAsync = promisify(execFile);
+const getPrimaryEnvPath = () => (fs.existsSync(PACKAGE_ENV_PATH) ? PACKAGE_ENV_PATH : USER_ENV_PATH);
 const AUTH_CONFIG_PATH = path.join(KIKU_DATA_DIR, 'yt_auth.json');
-const SECURE_COOKIE_PATH = path.join(KIKU_DATA_DIR, 'cookies.txt');
-loadEnv();
-const YTDLP_CMD = process.env.KIKU_YTDLP || 'yt-dlp';
+const COOKIE_FILE_PATH = path.join(KIKU_DATA_DIR, 'cookies.txt');
+
+const YTDLP_CMD = () => process.env.KIKU_YTDLP || 'yt-dlp';
 
 
 const BROWSERS = ['firefox', 'chrome', 'edge', 'brave'];
+let browserDetectionCache = null;
 
-function findBrowserExe(browser) {
+async function findBrowserExeAsync(browser, { execImpl = execFileAsync } = {}) {
   const isWin = process.platform === 'win32';
   const probes = {
     firefox: {
-      cmd: isWin ? 'firefox' : 'firefox',
+      cmd: 'firefox',
       paths: [
         'C:\\Program Files\\Mozilla Firefox\\firefox.exe',
         'C:\\Program Files (x86)\\Mozilla Firefox\\firefox.exe',
@@ -53,7 +61,7 @@ function findBrowserExe(browser) {
   const p = probes[browser];
   if (!p) return null;
   try {
-    execSync(isWin ? `where ${p.cmd}` : `which ${p.cmd}`, { stdio: 'ignore', windowsHide: true });
+    await execImpl(isWin ? 'where' : 'which', [p.cmd], { windowsHide: true, timeout: 1500 });
     return p.cmd;
   } catch {}
 
@@ -65,12 +73,18 @@ function findBrowserExe(browser) {
   return null;
 }
 
-function detectBrowsers() {
-  return BROWSERS.filter(b => findBrowserExe(b) !== null);
+async function detectBrowsersCached({ execImpl } = {}) {
+  if (browserDetectionCache) return browserDetectionCache;
+  const found = [];
+  for (const b of BROWSERS) {
+    if (await findBrowserExeAsync(b, { execImpl })) found.push(b);
+  }
+  browserDetectionCache = found;
+  return found;
 }
 
-function openBrowser(url, browser) {
-  const exe = browser ? findBrowserExe(browser) : null;
+async function openBrowser(url, browser) {
+  const exe = browser ? await findBrowserExeAsync(browser) : null;
   const isWin = process.platform === 'win32';
   try {
     if (exe) {
@@ -105,8 +119,7 @@ function writeAuthConfig(updates) {
   } catch {}
 }
 
-function loadEnv() {
-
+function refreshYtAuthEnv() {
   const userConfig = readAuthConfig();
   if (userConfig.KIKU_YT_BROWSER && process.env.KIKU_YT_BROWSER === undefined) {
     process.env.KIKU_YT_BROWSER = userConfig.KIKU_YT_BROWSER;
@@ -114,62 +127,40 @@ function loadEnv() {
   if (userConfig.KIKU_YT_COOKIES && process.env.KIKU_YT_COOKIES === undefined) {
     process.env.KIKU_YT_COOKIES = userConfig.KIKU_YT_COOKIES;
   }
-
-
-  try {
-    if (!fs.existsSync(ENV_PATH)) return;
-    const text = fs.readFileSync(ENV_PATH, 'utf8');
-    for (const line of text.split('\n')) {
-      const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
-      if (!m) continue;
-      const key = m[1].trim();
-      const val = m[2].trim().replace(/^["']|["']$/g, '');
-      if (process.env[key] === undefined) process.env[key] = val;
-    }
-  } catch {}
-}
-
-function readEnvMap() {
-  const map = {};
-  try {
-    if (!fs.existsSync(ENV_PATH)) return map;
-    const text = fs.readFileSync(ENV_PATH, 'utf8');
-    for (const line of text.split('\n')) {
-      const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
-      if (m) map[m[1]] = m[2].replace(/^["']|["']$/g, '');
-    }
-  } catch {}
-  return map;
 }
 
 function writeEnv(updates) {
-
-  for (const [k, v] of Object.entries(updates)) {
-    process.env[k] = v;
+  const cleanUpdates = {};
+  for (const [k, v] of Object.entries(updates || {})) {
+    const cleanKey = String(k || '').replace(/[\r\n]+/g, '').trim();
+    const cleanVal = String(v ?? '').replace(/[\r\n]+/g, '').trim();
+    if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(cleanKey)) {
+      cleanUpdates[cleanKey] = cleanVal;
+      process.env[cleanKey] = cleanVal;
+    }
   }
-  writeAuthConfig(updates);
+  writeAuthConfig(cleanUpdates);
 
- 
   try {
-    const map = readEnvMap();
-    Object.assign(map, updates);
-    const out = Object.entries(map).map(([k, v]) => `${k}=${v}`).join('\n') + '\n';
-    fs.writeFileSync(ENV_PATH, out, 'utf8');
+    const envFile = getPrimaryEnvPath();
+    for (const [key, value] of Object.entries(cleanUpdates)) {
+      upsertEnvKey(envFile, key, value);
+    }
   } catch {}
 }
 
-function hasSession(browser) {
+function hasSession(browser, { timeoutMs = 6000 } = {}) {
   return new Promise((resolve) => {
-    const timeoutMs = 8000;
+    const probeUrl = process.env.KIKU_YT_PROBE_URL || 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
     const args = [
       '--cookies-from-browser', browser,
       '--print', '%(title)s',
       '--no-warnings',
       '--no-playlist',
       '--no-check-certificates',
-      'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+      probeUrl
     ];
-    const p = spawn(YTDLP_CMD, args, {
+    const p = spawn(YTDLP_CMD(), args, {
       windowsHide: true,
       stdio: ['ignore', 'pipe', 'pipe']
     });
@@ -182,17 +173,18 @@ function hasSession(browser) {
       if (!resolved) {
         resolved = true;
         try { p.kill('SIGKILL'); } catch {}
-        resolve(false);
+        resolve({ ok: false, reason: 'timeout' });
       }
     }, timeoutMs);
 
     p.stdout?.on('data', d => { out += d.toString(); });
     p.stderr?.on('data', d => { err += d.toString(); });
-    p.on('error', () => {
+    p.on('error', (spawnErr) => {
       if (!resolved) {
         resolved = true;
         clearTimeout(timer);
-        resolve(false);
+        const missing = spawnErr?.code === 'ENOENT';
+        resolve({ ok: false, reason: missing ? 'yt-dlp-missing' : 'spawn-error', detail: spawnErr?.message });
       }
     });
     p.on('close', (code) => {
@@ -201,7 +193,11 @@ function hasSession(browser) {
       clearTimeout(timer);
       const title = out.trim();
       const blocked = /403|Forbidden|Sign in|confirm you'?re not a bot|Login Required|Private Video|database is locked/i.test(err);
-      resolve(code === 0 && title.length > 0 && !blocked);
+      if (code === 0 && title.length > 0 && !blocked) {
+        resolve({ ok: true, reason: 'session-found' });
+      } else {
+        resolve({ ok: false, reason: blocked ? 'blocked' : 'no-session', detail: err.slice(0, 200) });
+      }
     });
   });
 }
@@ -248,18 +244,17 @@ export function isCookieFileValid(filePath) {
 }
 
 async function _ensureYouTubeCookiesImpl(opts = {}) {
-  loadEnv();
+  refreshYtAuthEnv();
 
   const cookiesCandidate = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
     ? process.env.KIKU_YT_COOKIES
-    : (isCookieFileValid(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null);
+    : (isCookieFileValid(COOKIE_FILE_PATH) ? COOKIE_FILE_PATH : null);
   if (cookiesCandidate && isCookieFileValid(cookiesCandidate)) {
     _ensured = { ok: true, source: 'file', cookiesPath: cookiesCandidate };
     return _ensured;
   }
 
- 
-  const available = detectBrowsers();
+  const available = await detectBrowsersCached();
   if (!available.length) {
     _ensured = { ok: false, reason: 'no-browser', detail: 'No supported browser found (Firefox, Chrome, Edge, Brave)' };
     _lastFailTime = Date.now();
@@ -271,12 +266,21 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
     ? [preferred, ...available.filter(b => b !== preferred)]
     : available;
 
+  let sawYtdlpMissing = false;
   for (const b of candidates) {
-    if (await hasSession(b)) {
+    const probe = await hasSession(b, { timeoutMs: opts.probeTimeoutMs });
+    if (probe.ok) {
       writeEnv({ KIKU_YT_BROWSER: b });
       _ensured = { ok: true, source: 'existing-session', browser: b };
       return _ensured;
     }
+    if (probe.reason === 'yt-dlp-missing') sawYtdlpMissing = true;
+  }
+
+  if (sawYtdlpMissing) {
+    _ensured = { ok: false, reason: 'yt-dlp-missing', detail: `yt-dlp was not found (set KIKU_YTDLP in .env, e.g. ${YTDLP_CMD()})` };
+    _lastFailTime = Date.now();
+    return _ensured;
   }
 
  
@@ -310,7 +314,8 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
     }
 
     for (const b of available) {
-      if (await hasSession(b)) {
+      const probe = await hasSession(b, { timeoutMs: opts.probeTimeoutMs });
+      if (probe.ok) {
         writeEnv({ KIKU_YT_BROWSER: b });
         if (process.stdout.isTTY) {
           process.stdout.write('\r' + ' '.repeat(65) + '\r');
@@ -335,9 +340,9 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
 }
 
 export async function exportCookiesToFile() {
-  loadEnv();
+  refreshYtAuthEnv();
 
-  const available = detectBrowsers();
+  const available = await detectBrowsersCached();
   const preferred = process.env.KIKU_YT_BROWSER;
   const candidates = preferred && available.includes(preferred)
     ? [preferred, ...available.filter(b => b !== preferred)]
@@ -348,18 +353,18 @@ export async function exportCookiesToFile() {
       fs.mkdirSync(KIKU_DATA_DIR, { recursive: true });
       const args = [
         '--cookies-from-browser', b,
-        '--cookies', SECURE_COOKIE_PATH,
+        '--cookies', COOKIE_FILE_PATH,
         '--no-warnings',
         '--skip-download',
-        'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
+        process.env.KIKU_YT_PROBE_URL || 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
       ];
       await new Promise((resolve, reject) => {
-        const p = spawn(YTDLP_CMD, args, { windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] });
+        const p = spawn(YTDLP_CMD(), args, { windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] });
         let err = '';
         p.stderr?.on('data', d => { err += d.toString(); });
         p.on('error', reject);
         p.on('close', code => {
-          if (code === 0 && fs.existsSync(SECURE_COOKIE_PATH)) {
+          if (code === 0 && fs.existsSync(COOKIE_FILE_PATH)) {
             resolve();
           } else {
             reject(new Error(err || `yt-dlp exited ${code}`));
@@ -367,8 +372,11 @@ export async function exportCookiesToFile() {
         });
       });
 
-      writeEnv({ KIKU_YT_COOKIES: SECURE_COOKIE_PATH });
-      return { ok: true, browser: b, cookiesPath: SECURE_COOKIE_PATH };
+      if (process.platform !== 'win32') {
+        try { fs.chmodSync(COOKIE_FILE_PATH, 0o600); } catch {}
+      }
+      writeEnv({ KIKU_YT_COOKIES: COOKIE_FILE_PATH });
+      return { ok: true, browser: b, cookiesPath: COOKIE_FILE_PATH };
     } catch {}
   }
 
@@ -384,12 +392,13 @@ export function resetCookieCheck() {
 export function getCookieStatus() {
   const envCookies = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
     ? process.env.KIKU_YT_COOKIES
-    : (isCookieFileValid(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null);
+    : (isCookieFileValid(COOKIE_FILE_PATH) ? COOKIE_FILE_PATH : null);
   return {
     ensured: _ensured,
     envBrowser: process.env.KIKU_YT_BROWSER || null,
     envCookies,
-    envPath: ENV_PATH,
+    envPath: getPrimaryEnvPath(),
+    userEnvPath: USER_ENV_PATH,
     authConfigPath: AUTH_CONFIG_PATH
   };
 }

package/server/agent/youtubeTranscript.js

@@ -0,0 +1,209 @@
+import { spawn } from 'child_process';
+import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ensureYouTubeCookies, isCookieFileValid } from './youtubeAuth.js';
+
+const YTDLP_CMD = process.env.KIKU_YTDLP || 'yt-dlp';
+
+const BASE_SUB_LANGS = ['th', 'en', 'zh-Hans', 'zh', 'zh-Hant', 'zh-CN', 'zh-TW', 'en-US'];
+const LANG_PREFERENCES = {
+  th: ['th', 'th-TH'],
+  en: ['en', 'en-US', 'en-GB'],
+  zh: ['zh-Hans', 'zh-CN', 'zh', 'zh-Hant', 'zh-TW'],
+};
+
+export function extractYouTubeVideoId(input) {
+  if (!input || typeof input !== 'string') return null;
+  const trimmed = input.trim();
+  if (/^[\w-]{11}$/.test(trimmed)) return trimmed;
+  const patterns = [
+    /youtube\.com\/watch\?(?:[^#]*&)?v=([\w-]{11})/i,
+    /youtu\.be\/([\w-]{11})/i,
+    /youtube\.com\/shorts\/([\w-]{11})/i,
+    /youtube\.com\/embed\/([\w-]{11})/i,
+    /youtube\.com\/live\/([\w-]{11})/i,
+  ];
+  for (const re of patterns) {
+    const match = trimmed.match(re);
+    if (match) return match[1];
+  }
+  return null;
+}
+
+
+export function parseSubtitleToText(content) {
+  if (!content || typeof content !== 'string') return '';
+  const text = content.startsWith('\uFEFF') ? content.slice(1) : content;
+  const lines = text.split(/\r?\n/);
+
+  const out = [];
+  const seen = new Set();
+  for (let i = 0; i < lines.length; i++) {
+    let l = lines[i].trim();
+    if (!l) continue;
+    if (/^WEBVTT/i.test(l) || /^(NOTE|STYLE|REGION)\b/i.test(l)) continue;
+    if (l.includes('-->')) continue;
+   
+    if (/^\d+$/.test(l)) {
+      let j = i + 1;
+      while (j < lines.length && !lines[j].trim()) j++;
+      if (j < lines.length && lines[j].includes('-->')) continue;
+    }
+    l = l.replace(/<[^>]+>/g, '');
+    l = l
+      .replace(/&amp;/g, '&')
+      .replace(/&#39;/g, "'")
+      .replace(/&apos;/g, "'")
+      .replace(/&quot;/g, '"')
+      .replace(/&gt;/g, '>')
+      .replace(/&lt;/g, '<')
+      .replace(/&nbsp;/g, ' ');
+    if (!l) continue;
+    const key = l.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(l);
+  }
+  return out.join(' ');
+}
+
+async function resolveCookieArgs() {
+  if (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES)) {
+    return ['--cookies', process.env.KIKU_YT_COOKIES];
+  }
+  if (process.env.KIKU_YT_BROWSER) {
+    return ['--cookies-from-browser', process.env.KIKU_YT_BROWSER];
+  }
+  try {
+    const status = await ensureYouTubeCookies({ silent: true });
+    if (status?.ok) {
+      if (status.cookiesPath) return ['--cookies', status.cookiesPath];
+      if (status.browser) return ['--cookies-from-browser', status.browser];
+    }
+  } catch { }
+  return [];
+}
+
+function runYtDlp(args, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn(YTDLP_CMD, args, { windowsHide: true });
+    let stdout = '';
+    let stderr = '';
+    const timer = setTimeout(() => {
+      try { proc.kill('SIGKILL'); } catch { }
+      reject(new Error(`yt-dlp timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+    proc.stdout.on('data', d => { stdout += d.toString(); });
+    proc.stderr.on('data', d => { stderr += d.toString(); });
+    proc.on('error', err => { clearTimeout(timer); reject(err); });
+    proc.on('close', code => { clearTimeout(timer); resolve({ code, stdout, stderr }); });
+  });
+}
+
+function listSubtitleFiles(dir) {
+  try {
+    return readdirSync(dir)
+      .map(name => {
+        const match = name.match(/^transcript\.(.+)\.(vtt|srt)$/i);
+        return match ? { file: join(dir, name), lang: match[1] } : null;
+      })
+      .filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+function pickBestSubtitleFile(files, prefs) {
+  const score = (entry) => {
+    const lang = entry.lang.toLowerCase();
+    let best = prefs.length > 0 ? 100 : 0;
+    prefs.forEach((p, i) => {
+      const pLower = p.toLowerCase();
+      if (lang === pLower) best = Math.min(best, i);
+      else if (lang.startsWith(pLower)) best = Math.min(best, i + 0.5);
+    });
+    if (/(^|\.)orig$/i.test(lang) || /-orig$/i.test(lang)) best += 10;
+    return best;
+  };
+  return [...files].sort((a, b) => score(a) - score(b))[0];
+}
+
+export async function fetchYoutubeTranscript({ url, videoId, lang, maxChars } = {}) {
+  const id = videoId ? String(videoId).trim() : extractYouTubeVideoId(url);
+  if (!id || !/^[\w-]{11}$/.test(id)) {
+    return {
+      status: 'error',
+      message: 'Provide a valid YouTube URL (watch/youtu.be/shorts/embed/live) or an 11-character videoId.',
+    };
+  }
+
+  const dir = mkdtempSync(join(tmpdir(), 'kiku-transcript-'));
+  try {
+    const prefs = LANG_PREFERENCES[String(lang || '').toLowerCase()] || [];
+    const subLangs = [...new Set([...prefs, ...BASE_SUB_LANGS])].join(',');
+    const baseArgs = [
+      '--skip-download', '--no-warnings', '--quiet',
+      '--write-subs', '--write-auto-subs',
+      '--sub-format', 'vtt/srt/best',
+      '--sub-langs', subLangs,
+      '-o', join(dir, 'transcript.%(ext)s'),
+    ];
+    const watchUrl = `https://www.youtube.com/watch?v=${id}`;
+
+    const cookieArgs = await resolveCookieArgs();
+    let attempt = { code: 1, stdout: '', stderr: '' };
+    try {
+      attempt = await runYtDlp([...baseArgs, ...cookieArgs, watchUrl], 60_000);
+    } catch (err) {
+      attempt = { code: 1, stdout: '', stderr: err.message };
+    }
+
+    let files = listSubtitleFiles(dir);
+    if (files.length === 0 && cookieArgs.length > 0) {
+      try {
+        attempt = await runYtDlp([...baseArgs, watchUrl], 60_000);
+      } catch (err) {
+        attempt = { code: 1, stdout: '', stderr: err.message };
+      }
+      files = listSubtitleFiles(dir);
+    }
+
+    if (files.length === 0) {
+      const blocked = /sign in|confirm you|cookies|age/i.test(attempt.stderr || '');
+      return {
+        status: 'not_found',
+        videoId: id,
+        url: watchUrl,
+        message: blocked
+          ? `YouTube refused the caption request for ${id} (sign-in/cookies or age gate). Set KIKU_YT_COOKIES or KIKU_YT_BROWSER to enable authenticated caption downloads.`
+          : `No captions are available for this video (${id}), or yt-dlp could not reach it. Verify the video exists and has captions enabled.`,
+      };
+    }
+
+    const best = pickBestSubtitleFile(files, prefs);
+    const text = parseSubtitleToText(readFileSync(best.file, 'utf8'));
+    if (!text) {
+      return {
+        status: 'not_found',
+        videoId: id,
+        url: watchUrl,
+        message: `Captions were downloaded for ${id} but contained no readable text.`,
+      };
+    }
+
+    const max = Math.max(500, Math.min(Number(maxChars) || 6000, 20000));
+    const truncated = text.length > max;
+    return {
+      status: 'success',
+      videoId: id,
+      url: watchUrl,
+      lang: best.lang,
+      charCount: text.length,
+      truncated,
+      text: truncated ? `${text.slice(0, max)}…[truncated]` : text,
+    };
+  } finally {
+    try { rmSync(dir, { recursive: true, force: true }); } catch { }
+  }
+}