kiku-agent 1.0.11 → 1.2.0

package/server/agent/version.js

@@ -4,7 +4,5 @@ const packageManifest = JSON.parse(
   readFileSync(new URL('../../package.json', import.meta.url), 'utf8')
 );
 
-export const PACKAGE_NAME = packageManifest.name;
-export const PACKAGE_VERSION = packageManifest.version;
-export const PACKAGE_ENGINES = Object.freeze({ ...(packageManifest.engines || {}) });
-export const PACKAGE_UPDATE_METADATA = Object.freeze({ ...(packageManifest.kikuUpdate || {}) });
+export const PACKAGE_NAME = packageManifest.name;
+export const PACKAGE_VERSION = packageManifest.version;

package/server/agent/webSearch.js

@@ -3,6 +3,7 @@ import { execFile } from 'node:child_process';
 import { mkdtemp, readFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { assertSafeUrlHostname, sanitizeUrl } from './sanitizer.js';
 
 let _cheerioLoad = null;
 async function loadCheerio() {
@@ -41,6 +42,73 @@ function cleanHtml(html) {
     .trim();
 }
 
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+
+export async function safeFetch(inputUrl, options = {}, maxRedirects = 5) {
+  let currentUrl = String(inputUrl || '');
+  let currentOptions = { ...options, redirect: 'manual' };
+
+  for (let redirects = 0; redirects <= maxRedirects; redirects++) {
+    const sanitized = sanitizeUrl(currentUrl);
+    if (!sanitized.safe) throw new Error(`URL rejected by SSRF guard: ${sanitized.reason}`);
+    const dnsCheck = await assertSafeUrlHostname(sanitized.hostname);
+    if (!dnsCheck.safe) throw new Error(`URL rejected by SSRF guard: ${dnsCheck.reason}`);
+
+    const response = await fetch(sanitized.url.href, currentOptions);
+    if (!REDIRECT_STATUSES.has(response.status)) return response;
+
+    const location = response.headers.get('location');
+    if (!location) return response;
+    if (redirects === maxRedirects) throw new Error('Too many redirects');
+
+    const nextUrl = new URL(location, sanitized.url).href;
+    const nextOrigin = new URL(nextUrl).origin;
+    const headers = new Headers(currentOptions.headers || {});
+    if (nextOrigin !== sanitized.url.origin) {
+      headers.delete('authorization');
+      headers.delete('cookie');
+      headers.delete('proxy-authorization');
+    }
+    const method = String(currentOptions.method || 'GET').toUpperCase();
+    if (response.status === 303 || ((response.status === 301 || response.status === 302) && method === 'POST')) {
+      headers.delete('content-type');
+      currentOptions = { ...currentOptions, method: 'GET', body: undefined, headers };
+    } else {
+      currentOptions = { ...currentOptions, headers };
+    }
+    currentUrl = nextUrl;
+  }
+  throw new Error('Too many redirects');
+}
+
+async function readResponseTextLimited(response, maxBytes) {
+  const limit = Math.max(1024, Math.min(Number(maxBytes) || 1024, 2_000_000));
+  const declared = Number(response.headers.get('content-length') || 0);
+  if (declared > limit) throw new Error(`Response exceeds ${limit} byte limit`);
+  if (!response.body?.getReader) {
+    const text = await response.text();
+    if (Buffer.byteLength(text, 'utf8') > limit) throw new Error(`Response exceeds ${limit} byte limit`);
+    return text;
+  }
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let bytes = 0;
+  let text = '';
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > limit) throw new Error(`Response exceeds ${limit} byte limit`);
+      text += decoder.decode(value, { stream: true });
+    }
+    text += decoder.decode();
+    return text;
+  } finally {
+    if (bytes > limit) await reader.cancel().catch(() => {});
+  }
+}
+
 function detectLanguage(query) {
   if (/[\u0E00-\u0E7F]/.test(query)) return 'th';
   if (/[\u4E00-\u9FA5]/.test(query)) return 'zh';
@@ -276,18 +344,28 @@ export async function fetchWebpage(url, maxLength = 8000) {
     return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
   }
 
+  let parsedUrl;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return { status: 'error', message: 'Invalid URL format.' };
+  }
+  const ssrf = await assertSafeUrlHostname(parsedUrl.hostname);
+  if (!ssrf.safe) {
+    return { status: 'error', message: `URL rejected by SSRF guard: ${ssrf.reason}` };
+  }
+
   try {
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), 15000);
 
-    const res = await fetch(url, {
+    const res = await safeFetch(url, {
       headers: {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'en-US,en;q=0.9,th;q=0.8,zh-CN;q=0.7'
       },
-      signal: controller.signal,
-      redirect: 'follow'
+      signal: controller.signal
     });
 
     clearTimeout(timeoutId);
@@ -300,7 +378,8 @@ export async function fetchWebpage(url, maxLength = 8000) {
     const finalUrl = res.url || url;
 
     if (contentType.includes('application/json')) {
-      const json = await res.json();
+      const raw = await readResponseTextLimited(res, Math.max(64_000, maxLength * 8));
+      const json = JSON.parse(raw);
       const text = JSON.stringify(json, null, 2);
       return {
         status: 'success',
@@ -311,7 +390,7 @@ export async function fetchWebpage(url, maxLength = 8000) {
       };
     }
 
-    const html = await res.text();
+    const html = await readResponseTextLimited(res, Math.max(256_000, maxLength * 16));
     const cheerioLoad = await loadCheerio();
 
     if (cheerioLoad) {
@@ -429,6 +508,17 @@ export async function fetchJson(url, options = {}) {
     return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
   }
 
+  let parsedUrl;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return { status: 'error', message: 'Invalid URL format.' };
+  }
+  const ssrf = await assertSafeUrlHostname(parsedUrl.hostname);
+  if (!ssrf.safe) {
+    return { status: 'error', message: `URL rejected by SSRF guard: ${ssrf.reason}` };
+  }
+
   const {
     method = 'GET',
     headers = {},
@@ -456,19 +546,19 @@ export async function fetchJson(url, options = {}) {
       fetchOptions.headers['Content-Type'] = 'application/json';
     }
 
-    const res = await fetch(url, fetchOptions);
+    const res = await safeFetch(url, fetchOptions);
     clearTimeout(timeoutId);
 
     const responseInfo = {
-      status: res.status,
-      statusText: res.statusText,
+      httpStatus: res.status,
+      httpStatusText: res.statusText,
       contentType: res.headers.get('content-type') || '',
       finalUrl: res.url || url
     };
 
     if (!res.ok) {
       let errorBody = '';
-      try { errorBody = (await res.text()).slice(0, 1000); } catch {}
+      try { errorBody = (await readResponseTextLimited(res, 1000)).slice(0, 1000); } catch {}
       return {
         status: 'error',
         ...responseInfo,
@@ -477,7 +567,7 @@ export async function fetchJson(url, options = {}) {
       };
     }
 
-    const rawText = await res.text();
+    const rawText = await readResponseTextLimited(res, Math.min(1_000_000, maxResponseLength + 1));
     let data;
     try {
       data = JSON.parse(rawText);
@@ -542,6 +632,17 @@ export async function screenshotWebpage(url, options = {}) {
     return { status: 'error', message: 'Valid HTTP/HTTPS URL is required.' };
   }
 
+  let parsedUrl;
+  try {
+    parsedUrl = new URL(url);
+  } catch {
+    return { status: 'error', message: 'Invalid URL format.' };
+  }
+  const ssrf = await assertSafeUrlHostname(parsedUrl.hostname);
+  if (!ssrf.safe) {
+    return { status: 'error', message: `URL rejected by SSRF guard: ${ssrf.reason}` };
+  }
+
   const { width = 1280, height = 800, fullPage = false, timeoutMs = 20000 } = options;
   const chromePath = findChromePath();
 
@@ -557,7 +658,6 @@ export async function screenshotWebpage(url, options = {}) {
     const screenshotPath = join(tmpDir, screenshotFile);
     const args = [
       '--headless=new',
-      '--no-sandbox',
       '--disable-gpu',
       '--disable-dev-shm-usage',
       '--disable-software-rasterizer',

package/server/agent/youtubeAuth.js

@@ -122,8 +122,8 @@ function loadEnv() {
     for (const line of text.split('\n')) {
       const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
       if (!m) continue;
-      const key = m[1];
-      const val = m[2].replace(/^["']|["']$/g, '');
+      const key = m[1].trim();
+      const val = m[2].trim().replace(/^["']|["']$/g, '');
       if (process.env[key] === undefined) process.env[key] = val;
     }
   } catch {}
@@ -233,12 +233,27 @@ export async function ensureYouTubeCookies(opts = {}) {
   return _inFlightPromise;
 }
 
+export function isCookieFileValid(filePath) {
+  try {
+    if (!filePath || typeof filePath !== 'string' || !fs.existsSync(filePath)) return false;
+    const stat = fs.statSync(filePath);
+    if (stat.size < 10) return false;
+    const content = fs.readFileSync(filePath, 'utf8');
+    const lines = content.split('\n').filter(l => l.trim() && !l.trim().startsWith('#'));
+    if (lines.length === 0) return false;
+    return lines.some(l => l.includes('\t') && l.split('\t').length >= 4);
+  } catch {
+    return false;
+  }
+}
+
 async function _ensureYouTubeCookiesImpl(opts = {}) {
   loadEnv();
 
- 
-  const cookiesCandidate = process.env.KIKU_YT_COOKIES || (fs.existsSync(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null);
-  if (cookiesCandidate && fs.existsSync(cookiesCandidate)) {
+  const cookiesCandidate = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
+    ? process.env.KIKU_YT_COOKIES
+    : (isCookieFileValid(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null);
+  if (cookiesCandidate && isCookieFileValid(cookiesCandidate)) {
     _ensured = { ok: true, source: 'file', cookiesPath: cookiesCandidate };
     return _ensured;
   }
@@ -367,10 +382,13 @@ export function resetCookieCheck() {
 }
 
 export function getCookieStatus() {
+  const envCookies = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
+    ? process.env.KIKU_YT_COOKIES
+    : (isCookieFileValid(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null);
   return {
     ensured: _ensured,
     envBrowser: process.env.KIKU_YT_BROWSER || null,
-    envCookies: process.env.KIKU_YT_COOKIES || (fs.existsSync(SECURE_COOKIE_PATH) ? SECURE_COOKIE_PATH : null),
+    envCookies,
     envPath: ENV_PATH,
     authConfigPath: AUTH_CONFIG_PATH
   };