kuramanime-api 1.0.0

package/src/scrapers/cheerio.js

@@ -0,0 +1,256 @@
+const cheerio = require('cheerio');
+const sources = require('../config/sources');
+const db = require('../db');
+const browserFarm = require('../services/browserFarm');
+
+/**
+ * Fetch HTML with auto-fallback across alternative URLs.
+ * Uses browser-farm for JS-rendered pages, with direct HTTP as fallback.
+ */
+async function fetchWithFallback(path, useBrowser = true) {
+  const urls = sources.getAllUrls();
+
+  // Try browser-farm first (maintains sessions, handles JS)
+  if (useBrowser) {
+    for (const baseUrl of urls) {
+      const url = `${baseUrl.replace(/\/$/, '')}${path}`;
+      try {
+        const html = await browserFarm.fetchPageWithBrowser(url, 3000);
+        if (html && (html.includes('kuramanime') || html.includes('Kuramanime'))) {
+          db.logSource(baseUrl, 'ok_browser');
+          return { html, usedUrl: baseUrl };
+        }
+        db.logSource(baseUrl, 'browser_invalid_content');
+      } catch (err) {
+        db.logSource(baseUrl, `browser_error:${err.message.slice(0, 80)}`);
+      }
+    }
+  }
+
+  // Fallback: direct HTTP
+  for (const baseUrl of urls) {
+    const url = `${baseUrl.replace(/\/$/, '')}${path}`;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 15000);
+    try {
+      const res = await fetch(url, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+          'Accept': 'text/html,application/xhtml+xml',
+        },
+        signal: controller.signal,
+      });
+      clearTimeout(timer);
+
+      if (res.status === 403) {
+        db.logSource(baseUrl, '403_forbidden');
+        continue;
+      }
+      if (!res.ok) {
+        db.logSource(baseUrl, `http_${res.status}`);
+        continue;
+      }
+      const html = await res.text();
+      db.logSource(baseUrl, 'ok_http');
+      return { html, usedUrl: baseUrl };
+    } catch (err) {
+      clearTimeout(timer);
+      db.logSource(url, `error:${err.message.slice(0, 80)}`);
+    }
+  }
+  throw new Error('All source URLs failed');
+}
+
+/**
+ * Scrape anime cards from a Cheerio-loaded page.
+ */
+function scrapeAnimeCards($) {
+  const results = [];
+  $('.product__sidebar__view__item').each((_, el) => {
+    const $el = $(el);
+    const $parent = $el.closest('a');
+    const href = $parent.attr('href') || '';
+
+    const match = href.match(/\/anime\/(\d+)\/([^/]+)/);
+    if (!match) return;
+
+    const slug = match[2];
+    const id = parseInt(match[1], 10);
+    const image = $el.attr('data-setbg') || '';
+    const title = $el.find('.sidebar-title-h5').text().trim();
+    const quality = $el.find('.view').first().text().trim();
+    const epInfo = $el.find('.ep').text().trim();
+    const status = $el.find('.d-none span').text().trim();
+
+    let episode = null;
+    let score = null;
+    if (epInfo.includes('Ep')) {
+      episode = epInfo;
+    } else if (epInfo) {
+      score = epInfo;
+    }
+
+    results.push({ id, slug, title, image, quality, episode, score, status });
+  });
+  return results;
+}
+
+/**
+ * Scrape anime detail from detail page HTML.
+ */
+function scrapeAnimeDetail($) {
+  const title = $('.anime__details__title h3').first().text().trim();
+  const altTitleText = $('.anime__details__title span').first().text().trim();
+  const synopsis = $('#synopsisField').text().trim();
+  const image = $('.anime__details__pic__mobile').attr('data-setbg') || '';
+  const scoreText = $('.anime__details__pic__mobile .ep').text().trim();
+  const score = scoreText || null;
+  const quality = $('.anime__details__pic__mobile .ep-v2').text().trim() || null;
+
+  // Parse metadata from widget
+  const metadata = {};
+  $('.anime__details__widget li').each((_, el) => {
+    const label = $(el).find('.col-3 span').text().replace(':', '').trim().toLowerCase();
+    const value = $(el).find('.col-9').text().trim();
+    if (label) metadata[label] = value;
+  });
+
+  // Parse genres
+  const genres = [];
+  $('.anime__details__widget li').each((_, el) => {
+    const label = $(el).find('.col-3 span').text().replace(':', '').trim().toLowerCase();
+    if (label === 'genre') {
+      $(el).find('a').each((__, a) => {
+        const g = $(a).text().trim().replace(',', '');
+        if (g) genres.push(g);
+      });
+    }
+  });
+
+  // Parse episodes from popover
+  const episodes = [];
+  const popoverContent = $('#episodeLists').attr('data-content') || '';
+  if (popoverContent) {
+    const popover$ = cheerio.load(popoverContent);
+    popover$('a').each((_, a) => {
+      const epHref = popover$(a).attr('href') || '';
+      const epText = popover$(a).text().trim();
+      const epMatch = epHref.match(/\/anime\/(\d+)\/([^/]+)\/episode\/(\d+)/);
+      if (epMatch) {
+        episodes.push({
+          number: parseInt(epMatch[3], 10),
+          slug: epMatch[2],
+          url: epHref,
+        });
+      }
+    });
+  }
+
+  return {
+    title,
+    altTitle: altTitleText,
+    synopsis,
+    image,
+    score,
+    quality,
+    genres,
+    metadata,
+    episodes,
+  };
+}
+
+/**
+ * Scrape episode page for server/streaming info (static parts).
+ */
+function scrapeEpisodePage($) {
+  const episodes = [];
+  $('.anime__details__episodes .filter__gallery a.ep-button').each((_, el) => {
+    const href = $(el).attr('href') || '';
+    const epMatch = href.match(/\/anime\/(\d+)\/([^/]+)\/episode\/(\d+)/);
+    if (epMatch) {
+      episodes.push({
+        number: parseInt(epMatch[3], 10),
+        slug: epMatch[2],
+        url: href,
+      });
+    }
+  });
+
+  const servers = [];
+  $('#changeServer option').each((_, el) => {
+    const value = $(el).attr('value') || '';
+    const label = $(el).text().trim();
+    servers.push({ id: value, label });
+  });
+
+  const credit = $('#episodeCredit').text().trim();
+
+  return { episodes, servers, credit };
+}
+
+/**
+ * Scrape quicksearch results (AJAX endpoint).
+ */
+function scrapeQuickSearchResults($) {
+  const results = [];
+  $('.search__result__anchor').each((_, el) => {
+    const $el = $(el);
+    const href = $el.attr('href') || '';
+
+    // Extract ID and slug from URL: /anime/{id}/{slug}
+    const match = href.match(/\/anime\/(\d+)\/([^/]+)/);
+    if (!match) return;
+
+    const slug = match[2];
+    const id = parseInt(match[1], 10);
+    const image = $el.find('.search__result__pic').attr('data-setbg') || '';
+    const title = $el.find('.title').text().trim();
+    const altTitles = $el.find('.alt__titles').text().trim();
+
+    results.push({ id, slug, title, image, altTitles });
+  });
+  return results;
+}
+
+/**
+ * Fetch from the quicksearch AJAX endpoint.
+ */
+async function fetchQuickSearch(query) {
+  const sources = require('../config/sources');
+  const urls = sources.getAllUrls();
+  const db = require('../db');
+
+  for (const baseUrl of urls) {
+    const url = `${baseUrl.replace(/\/$/, '')}/quicksearch/get?search=${encodeURIComponent(query)}`;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 10000);
+    try {
+      const res = await fetch(url, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+          'X-Requested-With': 'XMLHttpRequest',
+          'Accept': 'application/json',
+        },
+        signal: controller.signal,
+      });
+      clearTimeout(timer);
+      if (!res.ok) continue;
+      const json = await res.json();
+      db.logSource(baseUrl, 'ok_quicksearch');
+      return { json, usedUrl: baseUrl };
+    } catch (err) {
+      clearTimeout(timer);
+      db.logSource(baseUrl, `quicksearch_error:${err.message.slice(0, 80)}`);
+    }
+  }
+  throw new Error('All quicksearch URLs failed');
+}
+
+module.exports = {
+  fetchWithFallback,
+  scrapeAnimeCards,
+  scrapeAnimeDetail,
+  scrapeEpisodePage,
+  scrapeQuickSearchResults,
+  fetchQuickSearch,
+};

package/src/services/browserFarm.js

@@ -0,0 +1,106 @@
+/**
+ * Browser Farm client for headless Chromium scraping.
+ * Uses the browser-farm skill REST API.
+ */
+const config = require('../config');
+
+const BASE = config.browserFarm.baseUrl;
+const API_KEY = config.browserFarm.apiKey;
+
+const headers = {
+  'Content-Type': 'application/json',
+  'X-API-Key': API_KEY,
+};
+
+async function request(method, path, body = null) {
+  const url = `${BASE}${path}`;
+  const opts = { method, headers };
+  if (body) opts.body = JSON.stringify(body);
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), 30000);
+  try {
+    const res = await fetch(url, { ...opts, signal: controller.signal });
+    clearTimeout(timeout);
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Browser Farm ${res.status}: ${text.slice(0, 200)}`);
+    }
+    return res.json();
+  } catch (err) {
+    clearTimeout(timeout);
+    throw err;
+  }
+}
+
+async function spawn() {
+  const res = await request('POST', '/spawn', { headless: true });
+  return res.id;
+}
+
+async function navigate(browserId, url) {
+  return request('POST', `/browser/${browserId}/navigate`, { url });
+}
+
+async function evalJs(browserId, script) {
+  return request('POST', `/browser/${browserId}/eval`, { script });
+}
+
+async function getContent(browserId) {
+  return request('POST', `/browser/${browserId}/content`);
+}
+
+async function click(browserId, selector) {
+  return request('POST', `/browser/${browserId}/click`, { selector });
+}
+
+async function close(browserId) {
+  try {
+    return await request('POST', `/browser/${browserId}/close`);
+  } catch (_) { /* already closed */ }
+}
+
+/**
+ * Fetch a page using browser-farm and return HTML content.
+ * Useful for JavaScript-rendered pages.
+ */
+async function fetchPageWithBrowser(url, waitMs = 3000) {
+  let browserId;
+  try {
+    browserId = await spawn();
+    await navigate(browserId, url);
+    // Wait for JS to execute
+    await new Promise(resolve => setTimeout(resolve, waitMs));
+    const result = await getContent(browserId);
+    return result.content || result.html || '';
+  } finally {
+    if (browserId) await close(browserId);
+  }
+}
+
+/**
+ * Execute JS in page and return result.
+ */
+async function scrapeWithBrowser(url, script, waitMs = 3000) {
+  let browserId;
+  try {
+    browserId = await spawn();
+    await navigate(browserId, url);
+    await new Promise(resolve => setTimeout(resolve, waitMs));
+    const result = await evalJs(browserId, script);
+    return result;
+  } finally {
+    if (browserId) await close(browserId);
+  }
+}
+
+module.exports = {
+  spawn,
+  navigate,
+  evalJs,
+  getContent,
+  click,
+  close,
+  fetchPageWithBrowser,
+  scrapeWithBrowser,
+};

package/src/services/sourceUpdater.js

@@ -0,0 +1,125 @@
+/**
+ * Auto-update source URLs when primary gets blocked.
+ * Runs periodic health checks and updates sources.json.
+ */
+const https = require('https');
+const config = require('../config');
+const sources = require('../config/sources');
+const { logSource, cacheClear } = require('../db');
+
+const CHECK_INTERVAL_MS = (config.autoUpdateIntervalHours || 6) * 60 * 60 * 1000;
+
+const PATTERN_SITES = [
+  // Pattern: kuramainime.ing with version prefix
+  (base) => base.replace(/v\d+\./, `v${parseInt(base.match(/v(\d+)/)?.[1] || 18) + 1}.`),
+  (base) => base.replace(/v\d+\./, `v${parseInt(base.match(/v(\d+)/)?.[1] || 18) - 1}.`),
+];
+
+async function checkUrl(url, timeout = 10000) {
+  return new Promise((resolve) => {
+    const controller = new AbortController();
+    const timer = setTimeout(() => { controller.abort(); resolve(false); }, timeout);
+    const req = https.get(url, {
+      headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
+      signal: controller.signal,
+    }, (res) => {
+      clearTimeout(timer);
+      const ok = res.statusCode < 400 || res.statusCode === 403;
+      resolve(ok);
+    });
+    req.on('error', () => { clearTimeout(timer); resolve(false); });
+  });
+}
+
+async function discoverNewUrls() {
+  const primary = sources.getPrimary();
+  const alt = sources.getAll().alternatives;
+  const found = [];
+
+  // Check URL pattern variations
+  for (const pattern of PATTERN_SITES) {
+    try {
+      const candidate = pattern(primary);
+      // Validate it's a real URL
+      new URL(candidate);
+      const alive = await checkUrl(candidate);
+      if (alive && !alt.includes(candidate) && candidate !== primary) {
+        found.push(candidate);
+      }
+    } catch (_) { /* invalid URL pattern */ }
+  }
+
+  return found;
+}
+
+async function runSourceCheck() {
+  console.log('[SourceUpdater] Running health check...');
+
+  const primary = sources.getPrimary();
+  const isAlive = await checkUrl(primary, 12000);
+
+  if (!isAlive) {
+    console.log(`[SourceUpdater] ⚠️ Primary ${primary} is DOWN`);
+
+    // Try alternatives
+    const alt = sources.getAll().alternatives;
+    let switched = false;
+    for (const url of alt) {
+      if (await checkUrl(url)) {
+        console.log(`[SourceUpdater] ✅ Switching to ${url}`);
+        sources.setPrimary(url);
+        switched = true;
+        break;
+      }
+    }
+
+    // Discover new URLs
+    console.log('[SourceUpdater] Searching for new URLs...');
+    const discovered = await discoverNewUrls();
+    for (const url of discovered) {
+      if (await checkUrl(url)) {
+        console.log(`[SourceUpdater] 🆕 Discovered: ${url}`);
+        sources.addAlternative(url);
+        if (!switched) {
+          sources.setPrimary(url);
+          switched = true;
+        }
+      }
+    }
+
+    if (switched) {
+      logSource(sources.getPrimary(), 'switched');
+      cacheClear();
+    }
+
+    if (!switched) {
+      console.log('[SourceUpdater] ❌ No working source found');
+      logSource(primary, 'all_down');
+    }
+  } else {
+    console.log(`[SourceUpdater] ✅ Primary ${primary} is healthy`);
+  }
+}
+
+let intervalId = null;
+
+function initSourceUpdater() {
+  // Run immediately
+  runSourceCheck().catch(err => console.error('[SourceUpdater] Error:', err.message));
+
+  // Schedule periodic checks
+  intervalId = setInterval(() => {
+    runSourceCheck().catch(err => console.error('[SourceUpdater] Error:', err.message));
+  }, CHECK_INTERVAL_MS);
+
+  console.log(`[SourceUpdater] Scheduled every ${config.autoUpdateIntervalHours}h`);
+}
+
+function stopSourceUpdater() {
+  if (intervalId) {
+    clearInterval(intervalId);
+    intervalId = null;
+  }
+}
+
+module.exports = { initSourceUpdater, stopSourceUpdater, runSourceCheck };

package/vercel.json

@@ -0,0 +1,21 @@
+{
+  "version": 2,
+  "buildCommand": "echo 'Build: OK'",
+  "outputDirectory": ".",
+  "rewrites": [
+    { "source": "/api/(.*)", "destination": "/api/index" },
+    { "source": "/(.*)", "destination": "/api/index" }
+  ],
+  "functions": {
+    "api/index.js": {
+      "runtime": "@vercel/node@3",
+      "memory": 512,
+      "maxDuration": 30
+    }
+  },
+  "env": {
+    "BROWSER_FARM_URL": "@browser-farm-url",
+    "BROWSER_FARM_KEY": "@browser-farm-key",
+    "NODE_ENV": "production"
+  }
+}