better-ani-scraped 1.7.4 → 1.7.51

package/examples/animesama/example_usage_searchAnime.js

@@ -2,9 +2,14 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
 
 const main = async () => {
   const animesama = new AnimeScraper('animesama');
+  const browser = await animesama.createBrowser();
+  const sharedPage = await animesama.createSharedPage(browser);
 
-  const search = await animesama.searchAnime("gachiakuta", 100, [], ["Anime", "Film", "Autres", "Scans"]);
-  console.log("Search Results:", search);
+  const search = await animesama.searchAnime("gachiakuta", 100, [], ["Anime", "Film", "Autres", "Scans"], sharedPage);
+  const search2 = await animesama.searchAnime("one piece", 100, [], ["Anime", "Film", "Autres", "Scans"], sharedPage);
+  console.log("Search \"gachiakuta\" Results:", search);
+  console.log("Search \"one piece\" Results:", search2);
+  await browser.close();
 };
 
 main().catch(console.error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "better-ani-scraped",
-  "version": "1.7.4",
+  "version": "1.7.51",
   "description": "Scrape anime data from different sources (only anime-sama.org, animepahe and crunchyroll for the moment)",
   "main": "index.js",
   "scripts": {

package/scrapers/animesama.js

@@ -1,12 +1,15 @@
 import axios from "axios";
 import * as cheerio from "cheerio";
 import fs from "fs";
-import path from 'path';
-import { exec as execCallback } from 'child_process';
-import { promisify } from 'util';
-import { title } from "process";
-const execAsync = promisify(execCallback);
- 
+import path from "path";
+import util from "util";
+import { exec as execCallback } from "child_process";
+import puppeteerExtra from "puppeteer-extra";
+import StealthPlugin from "puppeteer-extra-plugin-stealth";
+const execAsync = util.promisify(execCallback);
+
+puppeteerExtra.use(StealthPlugin());
+
 
 const BASE_URL = "https://anime-sama.org";
 const CATALOGUE_URL = `${BASE_URL}/catalogue`; 
@@ -44,12 +47,41 @@ function getHeaders(referer = BASE_URL) {
   };
 }
 
+export async function createBrowser(customChromiumPath) {
+  const executablePath = await ensureChromiumInstalled(customChromiumPath);
+  const browser = await puppeteerExtra.launch({
+    headless: true,
+    executablePath,
+    args: ["--no-sandbox", "--disable-setuid-sandbox"],
+  });
+  return browser;
+}
+
+export async function createSharedPage(browser) {
+  const page = await browser.newPage();
+  await page.setExtraHTTPHeaders(getHeaders());
+  await page.setUserAgent(
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+  );
+  // block heavy resources but NOT scripts
+  await page.setRequestInterception(true);
+  page.on("request", (req) => {
+    const blocked = ["image", "stylesheet", "font", "media"]; // keep scripts
+    if (blocked.includes(req.resourceType())) req.abort();
+    else req.continue();
+  });
+  await page.evaluateOnNewDocument(() => {
+    Object.defineProperty(navigator, "webdriver", { get: () => false });
+  });
+  return page;
+}
 export async function searchAnime(
   query,
   limit = 10,
   wantedLanguages = null,
   wantedTypes = null,
-  page = null
+  page = null,
+  wantedPage = 1
 ) {
   const languages = Array.isArray(wantedLanguages)
     ? wantedLanguages
@@ -62,6 +94,20 @@ export async function searchAnime(
 
   const results = [];
 
+  const fetchHtml = async (url) => {
+    if (page) {
+      await page.goto("about:blank"); // Réinitialise la page
+      await new Promise((resolve) => setTimeout(resolve, 100));
+      const html = await page
+        .goto(url, { waitUntil: "domcontentloaded" })
+        .then(() => page.content());
+      return html;
+    } else {
+      const res = await axios.get(url, { headers: getHeaders(CATALOGUE_URL) });
+      return res.data;
+    }
+  };
+
   const fetchPage = async (pageNum) => {
     const url =
       pageNum === 1
@@ -70,9 +116,10 @@ export async function searchAnime(
             query
           )}&page=${pageNum}`;
 
-    const res = await axios.get(url, { headers: getHeaders(CATALOGUE_URL) });
-    const $ = cheerio.load(res.data);
+    const html = await fetchHtml(url);
+    const $ = cheerio.load(html);
     const containers = $("div.catalog-card > a");
+
     containers.each((_, el) => {
       if (results.length >= limit) return false;
 
@@ -81,7 +128,11 @@ export async function searchAnime(
       const title = anchor.find("h2").first().text().trim();
       const altRaw = anchor.find("p").first().text().trim();
       const cover = anchor.find("img").first().attr("src");
-      const synopsis = anchor.find("div.synopsis-content").first().text();
+      const synopsis = anchor
+        .find("div.synopsis-content")
+        .first()
+        .text()
+        .trim();
 
       const typesRaw = anchor
         .find(".info-row")
@@ -92,10 +143,12 @@ export async function searchAnime(
         .toArray()
         .map((el) => $(el).text().trim())
         .filter((t) => isWanted(t, types));
-      const filteredTypes = wantedTypes.length === 0 ? typesRaw : typesRaw;
-      const hasScans = filteredTypes.some(
-        (t) => t.toLowerCase() === "scans".toLowerCase()
-      );
+      const filteredTypes =
+        Array.isArray(wantedTypes) && wantedTypes.length === 0
+          ? typesRaw
+          : typesRaw;
+      const hasScans = filteredTypes.some((t) => t.toLowerCase() === "scans");
+
       const languagesRaw = anchor
         .find(".info-row")
         .filter(
@@ -107,9 +160,12 @@ export async function searchAnime(
         .filter((l) => isWanted(l, languages));
 
       const filteredLanguages =
-        wantedLanguages.length === 0 || hasScans
+        (Array.isArray(wantedLanguages) && wantedLanguages.length === 0) ||
+        hasScans
           ? languagesRaw
-          : languagesRaw.filter((lang) => isWanted(lang, wantedLanguages));
+          : languagesRaw.filter((lang) =>
+              isWanted(lang, wantedLanguages || languages)
+            );
 
       const altTitles = altRaw
         ? altRaw
@@ -126,8 +182,9 @@ export async function searchAnime(
             .filter(Boolean)
         : [];
 
-      const hasValidType = types.length === 0 || filteredTypes;
-      const hasValidLanguage = languages.length === 0 || filteredLanguages;
+      const hasValidType = types.length === 0 || filteredTypes.length;
+      const hasValidLanguage =
+        languages.length === 0 || filteredLanguages.length;
 
       if (title && link && hasValidType && hasValidLanguage) {
         results.push({
@@ -144,8 +201,8 @@ export async function searchAnime(
     return containers.length > 0;
   };
 
-  if (page) {
-    await fetchPage(page);
+  if (wantedPage) {
+    await fetchPage(wantedPage);
   } else {
     let currentPage = 1;
     while ((await fetchPage(currentPage++)) && results.length < limit) {

package/scrapers/scrapers.js

@@ -16,7 +16,26 @@ export class AnimeScraper {
       );
     }
   }
-
+  async createBrowser(query, ...rest) {
+    try {
+      return await this.source.createBrowser(query, ...rest);
+    } catch (error) {
+      console.error(
+        `This scraper does not have the createBrowser function implemented or an error happened -> ${error}`
+      );
+      return null;
+    }
+  }
+  async createSharedPage(query, ...rest) {
+    try {
+      return await this.source.createSharedPage(query, ...rest);
+    } catch (error) {
+      console.error(
+        `This scraper does not have the createSharedPage function implemented or an error happened -> ${error}`
+      );
+      return null;
+    }
+  }
   async searchAnime(query, ...rest) {
     try {
       return await this.source.searchAnime(query, ...rest);