better-ani-scraped 1.6.81 → 1.6.82

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "better-ani-scraped",
-  "version": "1.6.81",
+  "version": "1.6.82",
   "description": "Scrape anime data from different sources (only anime-sama.fr, animepahe and crunchyroll for the moment)",
   "main": "index.js",
   "scripts": {

package/scrapers/animesama.js

@@ -36,19 +36,13 @@ async function ensureChromiumInstalled(customPath) {
 }
 
 function getHeaders(referer = BASE_URL) {
-  const userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
   return {
-    headers: {
-      "User-Agent": userAgent,
-      "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
-      "Referer": referer,
-    },
-    userAgent
+    "User-Agent": "Mozilla/5.0",
+    "Accept-Language": "fr-FR,fr;q=0.9,en;q=0.8",
+    Referer: referer,
   };
 }
 
-
-
 export async function searchAnime(
   query,
   limit = 10,
@@ -59,7 +53,6 @@ export async function searchAnime(
   const languages = Array.isArray(wantedLanguages) ? wantedLanguages : ["vostfr", "vf", "vastfr"];
   const types = Array.isArray(wantedTypes) ? wantedTypes : ["Anime", "Film"];
 
-  const { headers } = getHeaders(CATALOGUE_URL);
   const isWanted = (text, list) =>
     list.length === 0 || list.some(item => text.toLowerCase().includes(item.toLowerCase()));
 
@@ -71,7 +64,7 @@ export async function searchAnime(
         ? `${CATALOGUE_URL}/?search=${encodeURIComponent(query)}`
         : `${CATALOGUE_URL}/?search=${encodeURIComponent(query)}&page=${pageNum}`;
 
-    const res = await axios.get(url, { headers: headers });
+    const res = await axios.get(url, { headers: getHeaders(CATALOGUE_URL) });
     const $ = cheerio.load(res.data);
 
     const containers = $("a.flex.divide-x");
@@ -132,9 +125,7 @@ export async function searchAnime(
 }
 
 export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "va", "vkr", "vcn", "vqc", "vf1", "vf2"]) {
-  const { headersCatalog } = getHeaders(CATALOGUE_URL);
-  const { headersAnime } = getHeaders(animeUrl);
-  const res = await axios.get(animeUrl, { headers: headersCatalog });
+  const res = await axios.get(animeUrl, { headers: getHeaders(CATALOGUE_URL) });
   const html = res.data;
 
   const mainAnimeOnly = html.split("Anime Version Kai")[0];
@@ -164,19 +155,19 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
         const href = match[2].split("/")[0];
         const fullUrl = `${CATALOGUE_URL}/${animeName}/${href}/${language}`;
 
-      try {
-        const check = await axios.head(fullUrl, {
-          headers: headersAnime,
-        });
-        if (check.status === 200) {
-          languageAvailable = true;
-          seasons.push({ title, url: fullUrl });
+        try {
+          const check = await axios.head(fullUrl, {
+            headers: getHeaders(animeUrl),
+          });
+          if (check.status === 200) {
+            languageAvailable = true;
+            seasons.push({ title, url: fullUrl });
+          }
+        } catch (err) {
+          // Ignore invalid URLs
         }
-      } catch (err) {
-        // Ignore missing URLs
       }
     }
-  }
 
     if (languageAvailable) {
       return { language, seasons };
@@ -188,7 +179,6 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
 
 
 export async function getEpisodeTitles(seasonUrl, customChromiumPath) {
-  const { headers, userAgent } = getHeaders(seasonUrl);
   let browser;
   try {
     const puppeteer = await import('puppeteer');
@@ -205,7 +195,7 @@ export async function getEpisodeTitles(seasonUrl, customChromiumPath) {
     });
 
     const page = await browser.newPage();
-    await page.setExtraHTTPHeaders(headers);
+    await page.setExtraHTTPHeaders(getHeaders(seasonUrl));
     await page.setRequestInterception(true);
     page.on('request', (req) => {
       const blocked = ['image', 'stylesheet', 'font', 'media'];
@@ -215,7 +205,7 @@ export async function getEpisodeTitles(seasonUrl, customChromiumPath) {
         req.continue();
       }
     });
-    await page.setUserAgent(userAgent);
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
     await page.evaluateOnNewDocument(() => {
       Object.defineProperty(navigator, 'webdriver', { get: () => false });
     });
@@ -244,10 +234,8 @@ export async function getEmbed(
   includeInfo = false, 
   customChromiumPath
 ) {
-  const { headersSplit } = getHeaders(seasonUrl.split("/").slice(0, 5).join("/"));
-  const { headers } = getHeaders(seasonUrl);
   const res = await axios.get(seasonUrl, {
-    headers: headersSplit,
+    headers: getHeaders(seasonUrl.split("/").slice(0, 5).join("/")),
   });
 
   const $ = cheerio.load(res.data);
@@ -265,7 +253,7 @@ export async function getEmbed(
     : seasonUrl + "/" + scriptTag;
 
   const episodesJs = await axios
-    .get(scriptUrl, { headers: headers })
+    .get(scriptUrl, { headers: getHeaders(seasonUrl) })
     .then((r) => r.data);
 
   const matches = [...episodesJs.matchAll(/var\s+(eps\d+)\s*=\s*(\[[^\]]+\])/g)];
@@ -322,13 +310,13 @@ export async function getEmbed(
         if (selectedUrl) break;
       }
 
-    finalEmbeds.push({
-      title: null, 
-      url: selectedUrl || null,
-      host: selectedHost || null
-    });
+      finalEmbeds.push({
+        title: null, // à remplir plus tard
+        url: selectedUrl || null,
+        host: selectedHost || null,
+      });
+    }
   }
-}
 
   const titles = await getEpisodeTitles(seasonUrl, customChromiumPath);
   finalEmbeds.forEach((embed, i) => {
@@ -351,8 +339,7 @@ export async function getEmbed(
 
 
 export async function getAnimeInfo(animeUrl) {
-  const { headers } = getHeaders(CATALOGUE_URL);
-  const res = await axios.get(animeUrl, { headers: headers });
+  const res = await axios.get(animeUrl, { headers: getHeaders(CATALOGUE_URL) });
   const $ = cheerio.load(res.data);
 
   const cover = $("#coverOeuvre").attr("src");
@@ -388,7 +375,6 @@ export async function getAvailableLanguages(
   wantedLanguages = ["vostfr", "vf", "va", "vkr", "vcn", "vqc", "vf1", "vf2"],
   numberEpisodes = false
 ) {
-  const { headers } = getHeaders(CATALOGUE_URL);
   const languageLinks = [];
 
   // Iterate over each possible language and check if the page exists
@@ -396,7 +382,7 @@ export async function getAvailableLanguages(
     const languageUrl = seasonUrl.split('/').map((s, i) => i === 6 ? language : s).join('/');
     try {
       const res = await axios.get(languageUrl, {
-        headers: headers,
+        headers: getHeaders(CATALOGUE_URL),
       });
       if (res.status === 200) {
         if (numberEpisodes){
@@ -422,7 +408,6 @@ export async function getAllAnime(
   output = "anime_list.json",
   get_seasons = false
 ) {
-  const { headers } = getHeaders(CATALOGUE_URL);
   let animeLinks = [];
 
   const isWanted = (text, list) =>
@@ -430,7 +415,7 @@ export async function getAllAnime(
 
   const fetchPage = async (pageNum) => {
     const url = pageNum === 1 ? CATALOGUE_URL : `${CATALOGUE_URL}?page=${pageNum}`;
-    const res = await axios.get(url, { headers: headers });
+    const res = await axios.get(url, { headers: getHeaders(CATALOGUE_URL) });
     const $ = cheerio.load(res.data);
   
     const containers = $("div.shrink-0.m-3.rounded.border-2");
@@ -509,8 +494,7 @@ export async function getAllAnime(
 
 export async function getLatestEpisodes(languageFilter = null) {
   try {
-    const { headers } = getHeaders(BASE_URL);
-    const res = await axios.get(BASE_URL, { headers: headers });
+    const res = await axios.get(BASE_URL, { headers: getHeaders() });
     const $ = cheerio.load(res.data);
 
     const container = $("#containerAjoutsAnimes");
@@ -522,7 +506,7 @@ export async function getLatestEpisodes(languageFilter = null) {
       const cover = $(el).find("img").attr("src");
 
       const buttons = $(el).find("button");
-      const language = $(buttons[0]).text().trim().toLowerCase(); 
+      const language = $(buttons[0]).text().trim().toLowerCase(); // Normalisation
       const episode = $(buttons[1]).text().trim();
 
       if (
@@ -556,11 +540,10 @@ export async function getRandomAnime(
   maxAttempts = null,
   attempt = 0
 ) {
-  const { headers } = getHeaders(CATALOGUE_URL);
   try {
     const res = await axios.get(
       `${CATALOGUE_URL}/?search=&random=1`,
-      { headers:headers }
+      { headers: getHeaders(CATALOGUE_URL) }
     );
 
     const $ = cheerio.load(res.data);
@@ -626,4 +609,3 @@ export async function getRandomAnime(
     return null;
   }
 }
-