better-ani-scraped 1.7.4 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +1 -1
- package/examples/animesama/example_usage_getAllTitleScans.js +1 -1
- package/examples/animesama/example_usage_getAnimeInfo.js +1 -1
- package/examples/animesama/example_usage_getAvailableLanguages.js +1 -1
- package/examples/animesama/example_usage_getEmbed.js +1 -1
- package/examples/animesama/example_usage_getEpisodeTitles.js +1 -1
- package/examples/animesama/example_usage_getImgScans.js +1 -1
- package/examples/animesama/example_usage_getSeasons.js +1 -1
- package/package.json +2 -3
- package/scrapers/animesama.js +170 -94
- package/scrapers/scrapers.js +10 -1
- package/utils/dispatcher.js +5 -8
- package/utils/extractVideoUrl.js +13 -102
package/DOCUMENTATION.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
,,,,# Better-Ani-Scraped Documentation
|
|
2
2
|
|
|
3
|
-
A set of utility functions for scraping anime data from multiple sources (only [anime-sama](https://anime-sama.
|
|
3
|
+
A set of utility functions for scraping anime data from multiple sources (only [anime-sama](https://anime-sama.fr) and [animepahe](https://animepahe.ru) available at the moment). This tool allows you to search for anime, retrieve information, get episodes, and more.
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const mangaUrl = "https://anime-sama.
|
|
5
|
+
const mangaUrl = "https://anime-sama.fr/catalogue/drcl-midnight-children/scan/vf";
|
|
6
6
|
const chapterTitles = await animesama.getAllTitleScans(mangaUrl, true);
|
|
7
7
|
console.log("Titles:", chapterTitles);
|
|
8
8
|
};
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const animeUrl = "https://anime-sama.
|
|
5
|
+
const animeUrl = "https://anime-sama.fr/catalogue/86-eighty-six/";
|
|
6
6
|
|
|
7
7
|
const animeInfo = await animesama.getAnimeInfo(animeUrl);
|
|
8
8
|
console.log(animeInfo);
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const seasonUrl = "https://anime-sama.
|
|
5
|
+
const seasonUrl = "https://anime-sama.fr/catalogue/86-eighty-six/saison1/vostfr/";
|
|
6
6
|
|
|
7
7
|
const animeLanguages = await animesama.getAvailableLanguages(seasonUrl, ["vostfr", "vf", "va", "vkr","vcn", "vqc", "vf1", "vf2"], false);
|
|
8
8
|
console.log(animeLanguages);
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const seasonUrl = "https://anime-sama.
|
|
5
|
+
const seasonUrl = "https://anime-sama.fr/catalogue/one-piece/saison11/vostfr";
|
|
6
6
|
|
|
7
7
|
const embeds = await animesama.getEmbed(seasonUrl, ["smoothpre", "movearnpre", "sibnet", "vidmoly", "sendvid"], true, true);
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const seasonUrl = "https://anime-sama.
|
|
5
|
+
const seasonUrl = "https://anime-sama.fr/catalogue/86-eighty-six/saison1/vostfr/";
|
|
6
6
|
|
|
7
7
|
const episodeTitles = await animesama.getEpisodeTitles(seasonUrl);
|
|
8
8
|
console.log("Episode Titles:", episodeTitles);
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const scansUrl = "https://anime-sama.
|
|
5
|
+
const scansUrl = "https://anime-sama.fr/catalogue/drcl-midnight-children/scan/vf";
|
|
6
6
|
const scansImgUrl = await animesama.getImgScans(scansUrl, 9);
|
|
7
7
|
console.log("Image scans:", scansImgUrl);
|
|
8
8
|
};
|
|
@@ -2,7 +2,7 @@ import { AnimeScraper } from "../../index.js"; // REPLACE BY "from 'better-ani-s
|
|
|
2
2
|
|
|
3
3
|
const main = async () => {
|
|
4
4
|
const animesama = new AnimeScraper('animesama');
|
|
5
|
-
const animeUrl = "https://anime-sama.
|
|
5
|
+
const animeUrl = "https://anime-sama.fr/catalogue/drcl-midnight-children";
|
|
6
6
|
|
|
7
7
|
const seasons = await animesama.getSeasons(animeUrl, ["vostfr", "vf"], ["Anime", "Scans"]);
|
|
8
8
|
console.log("Seasons:", seasons);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "better-ani-scraped",
|
|
3
|
-
"version": "1.7.
|
|
4
|
-
"description": "Scrape anime data from different sources (only anime-sama.
|
|
3
|
+
"version": "1.7.6",
|
|
4
|
+
"description": "Scrape anime data from different sources (only anime-sama.fr, animepahe and crunchyroll for the moment)",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"test": "echo \"Error: no test specified\" && exit 1"
|
|
@@ -22,7 +22,6 @@
|
|
|
22
22
|
"cheerio": "^1.0.0",
|
|
23
23
|
"playwright": "^1.52.0",
|
|
24
24
|
"puppeteer": "^24.8.1",
|
|
25
|
-
"puppeteer-core": "^24.29.1",
|
|
26
25
|
"puppeteer-extra": "^3.3.6",
|
|
27
26
|
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
|
28
27
|
},
|
package/scrapers/animesama.js
CHANGED
|
@@ -1,15 +1,30 @@
|
|
|
1
1
|
import axios from "axios";
|
|
2
2
|
import * as cheerio from "cheerio";
|
|
3
3
|
import fs from "fs";
|
|
4
|
-
import path from
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import
|
|
4
|
+
import path from "path";
|
|
5
|
+
import { promisify } from "util";
|
|
6
|
+
import { exec as execCallback } from "child_process";
|
|
7
|
+
import puppeteerExtra from "puppeteer-extra";
|
|
8
|
+
import StealthPlugin from "puppeteer-extra-plugin-stealth";
|
|
8
9
|
const execAsync = promisify(execCallback);
|
|
9
|
-
|
|
10
|
+
puppeteerExtra.use(StealthPlugin());
|
|
11
|
+
|
|
12
|
+
const LIST_URL = "https://anime-sama.pw";
|
|
13
|
+
|
|
14
|
+
export async function getWorkingUrl(listUrl = LIST_URL) {
|
|
15
|
+
const res = await axios.get(listUrl);
|
|
16
|
+
const $ = cheerio.load(res.data);
|
|
17
|
+
return $(".btn-primary").attr("href");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
let BASE_URL;
|
|
21
|
+
let CATALOGUE_URL;
|
|
22
|
+
|
|
23
|
+
(async () => {
|
|
24
|
+
BASE_URL = await getWorkingUrl();
|
|
25
|
+
CATALOGUE_URL = `${BASE_URL}/catalogue`;
|
|
26
|
+
})();
|
|
10
27
|
|
|
11
|
-
const BASE_URL = "https://anime-sama.org";
|
|
12
|
-
const CATALOGUE_URL = `${BASE_URL}/catalogue`;
|
|
13
28
|
|
|
14
29
|
async function ensureChromiumInstalled(customPath) {
|
|
15
30
|
if (customPath) {
|
|
@@ -22,15 +37,20 @@ async function ensureChromiumInstalled(customPath) {
|
|
|
22
37
|
}
|
|
23
38
|
const basePath = path.join(
|
|
24
39
|
process.env.HOME || process.env.USERPROFILE,
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
40
|
+
".cache",
|
|
41
|
+
"puppeteer",
|
|
42
|
+
"chrome"
|
|
43
|
+
);
|
|
44
|
+
const chromiumPath = path.join(
|
|
45
|
+
basePath,
|
|
46
|
+
"win64-135.0.7049.95",
|
|
47
|
+
"chrome-win64",
|
|
48
|
+
"chrome.exe"
|
|
28
49
|
);
|
|
29
|
-
const chromiumPath = path.join(basePath, 'win64-135.0.7049.95', 'chrome-win64', 'chrome.exe');
|
|
30
50
|
|
|
31
51
|
if (!fs.existsSync(chromiumPath)) {
|
|
32
52
|
console.log("📦 Downloading Chromium 135.0.7049.95...");
|
|
33
|
-
await execAsync(
|
|
53
|
+
await execAsync("npx puppeteer browsers install chrome@135.0.7049.95");
|
|
34
54
|
}
|
|
35
55
|
|
|
36
56
|
return chromiumPath;
|
|
@@ -44,12 +64,14 @@ function getHeaders(referer = BASE_URL) {
|
|
|
44
64
|
};
|
|
45
65
|
}
|
|
46
66
|
|
|
67
|
+
|
|
47
68
|
export async function searchAnime(
|
|
48
69
|
query,
|
|
49
70
|
limit = 10,
|
|
50
71
|
wantedLanguages = null,
|
|
51
72
|
wantedTypes = null,
|
|
52
|
-
page = null
|
|
73
|
+
page = null,
|
|
74
|
+
wantedPage = 1
|
|
53
75
|
) {
|
|
54
76
|
const languages = Array.isArray(wantedLanguages)
|
|
55
77
|
? wantedLanguages
|
|
@@ -62,6 +84,20 @@ export async function searchAnime(
|
|
|
62
84
|
|
|
63
85
|
const results = [];
|
|
64
86
|
|
|
87
|
+
const fetchHtml = async (url) => {
|
|
88
|
+
if (page) {
|
|
89
|
+
await page.goto("about:blank"); // Réinitialise la page
|
|
90
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
91
|
+
const html = await page
|
|
92
|
+
.goto(url, { waitUntil: "domcontentloaded" })
|
|
93
|
+
.then(() => page.content());
|
|
94
|
+
return html;
|
|
95
|
+
} else {
|
|
96
|
+
const res = await axios.get(url, { headers: getHeaders(CATALOGUE_URL) });
|
|
97
|
+
return res.data;
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
|
|
65
101
|
const fetchPage = async (pageNum) => {
|
|
66
102
|
const url =
|
|
67
103
|
pageNum === 1
|
|
@@ -70,9 +106,10 @@ export async function searchAnime(
|
|
|
70
106
|
query
|
|
71
107
|
)}&page=${pageNum}`;
|
|
72
108
|
|
|
73
|
-
const
|
|
74
|
-
const $ = cheerio.load(
|
|
109
|
+
const html = await fetchHtml(url);
|
|
110
|
+
const $ = cheerio.load(html);
|
|
75
111
|
const containers = $("div.catalog-card > a");
|
|
112
|
+
|
|
76
113
|
containers.each((_, el) => {
|
|
77
114
|
if (results.length >= limit) return false;
|
|
78
115
|
|
|
@@ -81,7 +118,11 @@ export async function searchAnime(
|
|
|
81
118
|
const title = anchor.find("h2").first().text().trim();
|
|
82
119
|
const altRaw = anchor.find("p").first().text().trim();
|
|
83
120
|
const cover = anchor.find("img").first().attr("src");
|
|
84
|
-
const synopsis = anchor
|
|
121
|
+
const synopsis = anchor
|
|
122
|
+
.find("div.synopsis-content")
|
|
123
|
+
.first()
|
|
124
|
+
.text()
|
|
125
|
+
.trim();
|
|
85
126
|
|
|
86
127
|
const typesRaw = anchor
|
|
87
128
|
.find(".info-row")
|
|
@@ -92,10 +133,12 @@ export async function searchAnime(
|
|
|
92
133
|
.toArray()
|
|
93
134
|
.map((el) => $(el).text().trim())
|
|
94
135
|
.filter((t) => isWanted(t, types));
|
|
95
|
-
const filteredTypes =
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
136
|
+
const filteredTypes =
|
|
137
|
+
Array.isArray(wantedTypes) && wantedTypes.length === 0
|
|
138
|
+
? typesRaw
|
|
139
|
+
: typesRaw;
|
|
140
|
+
const hasScans = filteredTypes.some((t) => t.toLowerCase() === "scans");
|
|
141
|
+
|
|
99
142
|
const languagesRaw = anchor
|
|
100
143
|
.find(".info-row")
|
|
101
144
|
.filter(
|
|
@@ -107,9 +150,12 @@ export async function searchAnime(
|
|
|
107
150
|
.filter((l) => isWanted(l, languages));
|
|
108
151
|
|
|
109
152
|
const filteredLanguages =
|
|
110
|
-
wantedLanguages.length === 0 ||
|
|
153
|
+
(Array.isArray(wantedLanguages) && wantedLanguages.length === 0) ||
|
|
154
|
+
hasScans
|
|
111
155
|
? languagesRaw
|
|
112
|
-
: languagesRaw.filter((lang) =>
|
|
156
|
+
: languagesRaw.filter((lang) =>
|
|
157
|
+
isWanted(lang, wantedLanguages || languages)
|
|
158
|
+
);
|
|
113
159
|
|
|
114
160
|
const altTitles = altRaw
|
|
115
161
|
? altRaw
|
|
@@ -126,8 +172,9 @@ export async function searchAnime(
|
|
|
126
172
|
.filter(Boolean)
|
|
127
173
|
: [];
|
|
128
174
|
|
|
129
|
-
const hasValidType = types.length === 0 || filteredTypes;
|
|
130
|
-
const hasValidLanguage =
|
|
175
|
+
const hasValidType = types.length === 0 || filteredTypes.length;
|
|
176
|
+
const hasValidLanguage =
|
|
177
|
+
languages.length === 0 || filteredLanguages.length;
|
|
131
178
|
|
|
132
179
|
if (title && link && hasValidType && hasValidLanguage) {
|
|
133
180
|
results.push({
|
|
@@ -144,8 +191,8 @@ export async function searchAnime(
|
|
|
144
191
|
return containers.length > 0;
|
|
145
192
|
};
|
|
146
193
|
|
|
147
|
-
if (
|
|
148
|
-
await fetchPage(
|
|
194
|
+
if (wantedPage) {
|
|
195
|
+
await fetchPage(wantedPage);
|
|
149
196
|
} else {
|
|
150
197
|
let currentPage = 1;
|
|
151
198
|
while ((await fetchPage(currentPage++)) && results.length < limit) {
|
|
@@ -156,30 +203,45 @@ export async function searchAnime(
|
|
|
156
203
|
return results;
|
|
157
204
|
}
|
|
158
205
|
|
|
159
|
-
export async function getSeasons(
|
|
206
|
+
export async function getSeasons(
|
|
207
|
+
animeUrl,
|
|
208
|
+
languagePriority = ["vostfr", "vf", "va", "vkr", "vcn", "vqc", "vf1", "vf2"],
|
|
209
|
+
wantedTypes = ["Anime", "Kai", "Scans"]
|
|
210
|
+
) {
|
|
160
211
|
const res = await axios.get(animeUrl, { headers: getHeaders(CATALOGUE_URL) });
|
|
161
212
|
const html = res.data;
|
|
162
213
|
let mainAnimeOnly = html;
|
|
163
214
|
if (wantedTypes.length !== 0) {
|
|
164
215
|
if (!wantedTypes.includes("Anime")) {
|
|
165
|
-
mainAnimeOnly = mainAnimeOnly.replace(
|
|
216
|
+
mainAnimeOnly = mainAnimeOnly.replace(
|
|
217
|
+
/<h2.*?>Anime<\/h2>[\s\S]*?(?=<h2|$)/g,
|
|
218
|
+
""
|
|
219
|
+
);
|
|
166
220
|
}
|
|
167
221
|
if (!wantedTypes.includes("Kai")) {
|
|
168
|
-
mainAnimeOnly = mainAnimeOnly.replace(
|
|
222
|
+
mainAnimeOnly = mainAnimeOnly.replace(
|
|
223
|
+
/<h2.*?>Anime Version Kai<\/h2>[\s\S]*?(?=<h2|$)/g,
|
|
224
|
+
""
|
|
225
|
+
);
|
|
169
226
|
}
|
|
170
227
|
if (!wantedTypes.includes("Scans")) {
|
|
171
|
-
mainAnimeOnly = mainAnimeOnly.replace(
|
|
228
|
+
mainAnimeOnly = mainAnimeOnly.replace(
|
|
229
|
+
/<h2.*?>Manga<\/h2>[\s\S]*?(?=<h2|$)/g,
|
|
230
|
+
""
|
|
231
|
+
);
|
|
172
232
|
}
|
|
173
233
|
}
|
|
174
|
-
|
|
175
|
-
const $ = cheerio.load(mainAnimeOnly);
|
|
176
|
-
const scriptTags = $("script")
|
|
177
|
-
.toArray()
|
|
178
|
-
.filter(script => ["panneauAnime", "panneauScan"].some(str => $(script).html().includes(str)));
|
|
179
234
|
|
|
235
|
+
const $ = cheerio.load(mainAnimeOnly);
|
|
236
|
+
console.log(animeUrl);
|
|
237
|
+
const scriptTags = $("script").toArray().filter((script) =>
|
|
238
|
+
["panneauAnime", "panneauScan"].some((str) =>
|
|
239
|
+
$(script).html().includes(str)
|
|
240
|
+
)
|
|
241
|
+
);
|
|
242
|
+
|
|
180
243
|
const animeName = animeUrl.split("/")[4];
|
|
181
244
|
let seasons = [];
|
|
182
|
-
|
|
183
245
|
for (const language of languagePriority) {
|
|
184
246
|
seasons = [];
|
|
185
247
|
let languageAvailable = false;
|
|
@@ -191,10 +253,13 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
|
|
|
191
253
|
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
192
254
|
.replace(/\/\/.*$/gm, "");
|
|
193
255
|
|
|
194
|
-
const matches = [
|
|
195
|
-
|
|
256
|
+
const matches = [
|
|
257
|
+
...uncommentedContent.matchAll(
|
|
258
|
+
/(panneauAnime|panneauScan)\("([^"]+)", "([^"]+)"\);/g
|
|
259
|
+
),
|
|
260
|
+
];
|
|
196
261
|
for (let match of matches) {
|
|
197
|
-
const type = match[1];
|
|
262
|
+
const type = match[1];
|
|
198
263
|
const title = match[2];
|
|
199
264
|
const href = match[3].split("/")[0];
|
|
200
265
|
|
|
@@ -203,12 +268,14 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
|
|
|
203
268
|
for (const lang of languagePriority) {
|
|
204
269
|
const fullUrl = `${CATALOGUE_URL}/${animeName}/${href}/${lang}`;
|
|
205
270
|
try {
|
|
206
|
-
const check = await axios.head(fullUrl, {
|
|
271
|
+
const check = await axios.head(fullUrl, {
|
|
272
|
+
headers: getHeaders(animeUrl),
|
|
273
|
+
});
|
|
207
274
|
if (check.status === 200) {
|
|
208
275
|
seasons.push({ title, url: fullUrl });
|
|
209
|
-
scansLanguage = lang
|
|
276
|
+
scansLanguage = lang;
|
|
210
277
|
found = true;
|
|
211
|
-
break;
|
|
278
|
+
break;
|
|
212
279
|
}
|
|
213
280
|
} catch {}
|
|
214
281
|
}
|
|
@@ -216,7 +283,9 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
|
|
|
216
283
|
} else {
|
|
217
284
|
const fullUrl = `${CATALOGUE_URL}/${animeName}/${href}/${language}`;
|
|
218
285
|
try {
|
|
219
|
-
const check = await axios.head(fullUrl, {
|
|
286
|
+
const check = await axios.head(fullUrl, {
|
|
287
|
+
headers: getHeaders(animeUrl),
|
|
288
|
+
});
|
|
220
289
|
if (check.status === 200) {
|
|
221
290
|
seasons.push({ title, url: fullUrl });
|
|
222
291
|
languageAvailable = true;
|
|
@@ -225,10 +294,13 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
|
|
|
225
294
|
}
|
|
226
295
|
}
|
|
227
296
|
}
|
|
228
|
-
if (
|
|
297
|
+
if (
|
|
298
|
+
wantedTypes.includes("Scans") &&
|
|
299
|
+
wantedTypes.length == 1 &&
|
|
300
|
+
scansLanguage
|
|
301
|
+
) {
|
|
229
302
|
return { scansLanguage, seasons };
|
|
230
|
-
}
|
|
231
|
-
else if (languageAvailable) {
|
|
303
|
+
} else if (languageAvailable) {
|
|
232
304
|
return { language, seasons };
|
|
233
305
|
}
|
|
234
306
|
}
|
|
@@ -236,50 +308,46 @@ export async function getSeasons(animeUrl, languagePriority = ["vostfr", "vf", "
|
|
|
236
308
|
return [];
|
|
237
309
|
}
|
|
238
310
|
|
|
239
|
-
|
|
240
311
|
export async function getEpisodeTitles(seasonUrl, customChromiumPath) {
|
|
241
312
|
let browser;
|
|
242
313
|
try {
|
|
243
|
-
const puppeteer = await import(
|
|
314
|
+
const puppeteer = await import("puppeteer");
|
|
244
315
|
const executablePath = await ensureChromiumInstalled(customChromiumPath);
|
|
245
316
|
|
|
246
317
|
browser = await puppeteer.launch({
|
|
247
318
|
headless: true,
|
|
248
319
|
executablePath,
|
|
249
|
-
args: [
|
|
250
|
-
'--no-sandbox',
|
|
251
|
-
'--disable-setuid-sandbox',
|
|
252
|
-
]
|
|
253
|
-
|
|
320
|
+
args: ["--no-sandbox", "--disable-setuid-sandbox"],
|
|
254
321
|
});
|
|
255
322
|
|
|
256
323
|
const page = await browser.newPage();
|
|
257
324
|
await page.setExtraHTTPHeaders(getHeaders(seasonUrl));
|
|
258
325
|
await page.setRequestInterception(true);
|
|
259
|
-
page.on(
|
|
260
|
-
const blocked = [
|
|
326
|
+
page.on("request", (req) => {
|
|
327
|
+
const blocked = ["image", "stylesheet", "font", "media"];
|
|
261
328
|
if (blocked.includes(req.resourceType())) {
|
|
262
329
|
req.abort();
|
|
263
330
|
} else {
|
|
264
331
|
req.continue();
|
|
265
332
|
}
|
|
266
333
|
});
|
|
267
|
-
await page.setUserAgent(
|
|
334
|
+
await page.setUserAgent(
|
|
335
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
|
336
|
+
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
|
337
|
+
);
|
|
268
338
|
await page.evaluateOnNewDocument(() => {
|
|
269
|
-
Object.defineProperty(navigator,
|
|
339
|
+
Object.defineProperty(navigator, "webdriver", { get: () => false });
|
|
270
340
|
});
|
|
271
|
-
await page.goto(seasonUrl, { waitUntil:
|
|
272
|
-
await page.waitForSelector(
|
|
273
|
-
|
|
341
|
+
await page.goto(seasonUrl, { waitUntil: "domcontentloaded" });
|
|
342
|
+
await page.waitForSelector("#selectEpisodes");
|
|
274
343
|
|
|
275
|
-
const titres = await page.$$eval(
|
|
276
|
-
options.map(o => o.textContent.trim())
|
|
344
|
+
const titres = await page.$$eval("#selectEpisodes option", (options) =>
|
|
345
|
+
options.map((o) => o.textContent.trim())
|
|
277
346
|
);
|
|
278
347
|
|
|
279
348
|
return titres;
|
|
280
|
-
|
|
281
349
|
} catch (error) {
|
|
282
|
-
console.error(
|
|
350
|
+
console.error("Error while retrieving titles :", error);
|
|
283
351
|
return [];
|
|
284
352
|
} finally {
|
|
285
353
|
if (browser) await browser.close();
|
|
@@ -290,7 +358,7 @@ export async function getEmbed(
|
|
|
290
358
|
seasonUrl,
|
|
291
359
|
hostPriority = ["sendvid", "sibnet", "vidmoly", "oneupload"],
|
|
292
360
|
allHost = false,
|
|
293
|
-
includeInfo = false,
|
|
361
|
+
includeInfo = false,
|
|
294
362
|
customChromiumPath
|
|
295
363
|
) {
|
|
296
364
|
const res = await axios.get(seasonUrl, {
|
|
@@ -299,10 +367,13 @@ export async function getEmbed(
|
|
|
299
367
|
|
|
300
368
|
const $ = cheerio.load(res.data);
|
|
301
369
|
|
|
302
|
-
const seasonTitle =
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
370
|
+
const seasonTitle =
|
|
371
|
+
(
|
|
372
|
+
$("script")
|
|
373
|
+
.toArray()
|
|
374
|
+
.map((s) => $(s).html())
|
|
375
|
+
.find((c) => c && c.includes("#avOeuvre")) || ""
|
|
376
|
+
).match(/#avOeuvre"\)\.html\("([^"]+)"/)?.[1] || "Saison inconnue";
|
|
306
377
|
|
|
307
378
|
const scriptTag = $('script[src*="episodes.js"]').attr("src");
|
|
308
379
|
if (!scriptTag) throw new Error("No episodes script found");
|
|
@@ -315,7 +386,9 @@ export async function getEmbed(
|
|
|
315
386
|
.get(scriptUrl, { headers: getHeaders(seasonUrl) })
|
|
316
387
|
.then((r) => r.data);
|
|
317
388
|
|
|
318
|
-
const matches = [
|
|
389
|
+
const matches = [
|
|
390
|
+
...episodesJs.toLowerCase().matchAll(/var\s+(eps\d+)\s*=\s*(\[[^\]]+\])/g),
|
|
391
|
+
];
|
|
319
392
|
if (!matches.length) throw new Error("No episode arrays found");
|
|
320
393
|
|
|
321
394
|
let episodeMatrix = [];
|
|
@@ -328,7 +401,7 @@ export async function getEmbed(
|
|
|
328
401
|
}
|
|
329
402
|
}
|
|
330
403
|
|
|
331
|
-
const maxEpisodes = Math.max(...episodeMatrix.map(arr => arr.length));
|
|
404
|
+
const maxEpisodes = Math.max(...episodeMatrix.map((arr) => arr.length));
|
|
332
405
|
const finalEmbeds = [];
|
|
333
406
|
|
|
334
407
|
for (let i = 0; i < maxEpisodes; i++) {
|
|
@@ -349,11 +422,10 @@ export async function getEmbed(
|
|
|
349
422
|
}
|
|
350
423
|
|
|
351
424
|
finalEmbeds.push({
|
|
352
|
-
title: null,
|
|
425
|
+
title: null,
|
|
353
426
|
url: urls.length ? urls : null,
|
|
354
427
|
host: hosts.length ? hosts : null,
|
|
355
428
|
});
|
|
356
|
-
|
|
357
429
|
} else {
|
|
358
430
|
let selectedUrl = null;
|
|
359
431
|
let selectedHost = null;
|
|
@@ -370,7 +442,7 @@ export async function getEmbed(
|
|
|
370
442
|
}
|
|
371
443
|
|
|
372
444
|
finalEmbeds.push({
|
|
373
|
-
title: null,
|
|
445
|
+
title: null,
|
|
374
446
|
url: selectedUrl || null,
|
|
375
447
|
host: selectedHost || null,
|
|
376
448
|
});
|
|
@@ -387,16 +459,14 @@ export async function getEmbed(
|
|
|
387
459
|
episodes: finalEmbeds,
|
|
388
460
|
animeInfo: {
|
|
389
461
|
seasonTitle,
|
|
390
|
-
episodeCount: maxEpisodes
|
|
391
|
-
}
|
|
462
|
+
episodeCount: maxEpisodes,
|
|
463
|
+
},
|
|
392
464
|
};
|
|
393
465
|
} else {
|
|
394
466
|
return finalEmbeds;
|
|
395
467
|
}
|
|
396
468
|
}
|
|
397
469
|
|
|
398
|
-
|
|
399
|
-
|
|
400
470
|
export async function getAnimeInfo(animeUrl) {
|
|
401
471
|
const res = await axios.get(animeUrl, { headers: getHeaders(CATALOGUE_URL) });
|
|
402
472
|
const $ = cheerio.load(res.data);
|
|
@@ -438,19 +508,24 @@ export async function getAvailableLanguages(
|
|
|
438
508
|
|
|
439
509
|
// Iterate over each possible language and check if the page exists
|
|
440
510
|
for (let language of wantedLanguages) {
|
|
441
|
-
const languageUrl = seasonUrl
|
|
511
|
+
const languageUrl = seasonUrl
|
|
512
|
+
.split("/")
|
|
513
|
+
.map((s, i) => (i === 6 ? language : s))
|
|
514
|
+
.join("/");
|
|
442
515
|
try {
|
|
443
516
|
const res = await axios.get(languageUrl, {
|
|
444
517
|
headers: getHeaders(CATALOGUE_URL),
|
|
445
518
|
});
|
|
446
519
|
if (res.status === 200) {
|
|
447
|
-
if (numberEpisodes){
|
|
520
|
+
if (numberEpisodes) {
|
|
448
521
|
const episodeCount = (await getEmbed(languageUrl)).length;
|
|
449
|
-
languageLinks.push({
|
|
522
|
+
languageLinks.push({
|
|
523
|
+
language: language.toUpperCase(),
|
|
524
|
+
episodeCount: episodeCount,
|
|
525
|
+
});
|
|
450
526
|
} else {
|
|
451
527
|
languageLinks.push(language.toUpperCase());
|
|
452
528
|
}
|
|
453
|
-
|
|
454
529
|
}
|
|
455
530
|
} catch (error) {
|
|
456
531
|
// If an error occurs (like a 404), we skip that language
|
|
@@ -600,12 +675,12 @@ export async function getLatestEpisodes(languageFilter = null) {
|
|
|
600
675
|
try {
|
|
601
676
|
const res = await axios.get(BASE_URL, { headers: getHeaders() });
|
|
602
677
|
const $ = cheerio.load(res.data);
|
|
603
|
-
|
|
678
|
+
|
|
604
679
|
const container = $("#containerAjoutsAnimes");
|
|
605
680
|
const episodes = [];
|
|
606
681
|
|
|
607
682
|
container.find("a").each((_, el) => {
|
|
608
|
-
const link = $(el).attr("href");
|
|
683
|
+
const link = BASE_URL + $(el).attr("href");
|
|
609
684
|
const title = $(el).find("h2").text().trim();
|
|
610
685
|
const cover = $(el).find("img").attr("src");
|
|
611
686
|
const language = link.split("/").slice(6, 7).join("");
|
|
@@ -769,31 +844,32 @@ export async function getRandomAnime(
|
|
|
769
844
|
}
|
|
770
845
|
}
|
|
771
846
|
|
|
772
|
-
|
|
773
847
|
export async function getAllTitleScans(mangaUrl, numberImg = false) {
|
|
774
848
|
const res = await axios.get(mangaUrl, { headers: getHeaders() });
|
|
775
849
|
const $ = cheerio.load(res.data);
|
|
776
850
|
|
|
777
851
|
const title = encodeURIComponent($("#titreOeuvre").text().trim());
|
|
778
|
-
const urlInfo = `https://anime-sama.org/s2/scans/get_nb_chap_et_img.php?oeuvre=${title}
|
|
852
|
+
const urlInfo = `https://anime-sama.org/s2/scans/get_nb_chap_et_img.php?oeuvre=${title}`;
|
|
779
853
|
const infoPage = await axios.get(urlInfo, { headers: getHeaders() });
|
|
780
|
-
const titleChapter = Object.keys(infoPage.data)
|
|
854
|
+
const titleChapter = Object.keys(infoPage.data);
|
|
781
855
|
|
|
782
856
|
if (numberImg) {
|
|
783
|
-
return {scans
|
|
857
|
+
return { scans: infoPage.data, mangaTitle: title };
|
|
784
858
|
} else {
|
|
785
|
-
return titleChapter
|
|
859
|
+
return titleChapter;
|
|
786
860
|
}
|
|
787
861
|
}
|
|
788
862
|
|
|
789
863
|
export async function getImgScans(mangaUrl, wantedChapter) {
|
|
790
|
-
const infoScan = await getAllTitleScans(mangaUrl, true)
|
|
864
|
+
const infoScan = await getAllTitleScans(mangaUrl, true);
|
|
791
865
|
const numberImg = infoScan.scans[wantedChapter.toString()];
|
|
792
|
-
const mangaTitle = infoScan.mangaTitle
|
|
866
|
+
const mangaTitle = infoScan.mangaTitle;
|
|
793
867
|
const imgUrls = [];
|
|
794
868
|
for (let i = 1; i <= numberImg; i++) {
|
|
795
|
-
imgUrls.push(
|
|
869
|
+
imgUrls.push(
|
|
870
|
+
`https://anime-sama.org/s2/scans/${mangaTitle}/${wantedChapter}/${i}.jpg`
|
|
871
|
+
);
|
|
796
872
|
}
|
|
797
873
|
|
|
798
|
-
return imgUrls
|
|
799
|
-
}
|
|
874
|
+
return imgUrls;
|
|
875
|
+
}
|
package/scrapers/scrapers.js
CHANGED
|
@@ -16,7 +16,16 @@ export class AnimeScraper {
|
|
|
16
16
|
);
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
async getWorkingUrl(listUrl) {
|
|
20
|
+
try {
|
|
21
|
+
return await this.source.getWorkingUrl(listUrl);
|
|
22
|
+
} catch (error) {
|
|
23
|
+
console.error(
|
|
24
|
+
`This scraper does not have the getWorkingUrl function implemented or an error happened -> ${error}`
|
|
25
|
+
);
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
20
29
|
async searchAnime(query, ...rest) {
|
|
21
30
|
try {
|
|
22
31
|
return await this.source.searchAnime(query, ...rest);
|
package/utils/dispatcher.js
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
import * as extractor from "./extractVideoUrl.js";
|
|
2
2
|
|
|
3
|
-
export async function getVideoUrlFromEmbed(source, embedUrl
|
|
3
|
+
export async function getVideoUrlFromEmbed(source, embedUrl) {
|
|
4
4
|
if (source === "sibnet") {
|
|
5
5
|
return await extractor.getSibnetVideo(embedUrl);
|
|
6
6
|
}
|
|
7
7
|
if (source === "sendvid") {
|
|
8
8
|
return await extractor.getSendvidVideo(embedUrl);
|
|
9
9
|
}
|
|
10
|
-
if (source === "vidmoly") {
|
|
11
|
-
return await extractor.
|
|
10
|
+
if (source === "vidmoly" || source === "oneupload" ) {
|
|
11
|
+
return await extractor.getVidmolyOrOneuploadVideo(embedUrl);
|
|
12
12
|
}
|
|
13
|
-
if (source === "
|
|
14
|
-
return await extractor.
|
|
15
|
-
}
|
|
16
|
-
if (source === "movearnpre" || source === "smoothpre") {
|
|
17
|
-
return await extractor.getMovearnpreOrSmoothpreVideo(embedUrl, ...rest);
|
|
13
|
+
if (source === "movearnpre" || source === "smoothpre" ) {
|
|
14
|
+
return await extractor.getMovearnpreOrSmoothpreVideo(embedUrl);
|
|
18
15
|
}
|
|
19
16
|
|
|
20
17
|
throw new Error(`Unsupported embed source: ${source}`);
|
package/utils/extractVideoUrl.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import axios from "axios";
|
|
2
2
|
import * as cheerio from "cheerio";
|
|
3
|
-
|
|
4
|
-
import path from "path";
|
|
5
|
-
import fs from "fs";
|
|
3
|
+
|
|
6
4
|
|
|
7
5
|
const getHeaders = (referer) => ({
|
|
8
6
|
Accept: "*/*",
|
|
@@ -10,39 +8,6 @@ const getHeaders = (referer) => ({
|
|
|
10
8
|
"User-Agent":
|
|
11
9
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
12
10
|
});
|
|
13
|
-
async function ensureChromiumInstalled(customPath) {
|
|
14
|
-
if (customPath) {
|
|
15
|
-
if (fs.existsSync(customPath)) {
|
|
16
|
-
console.log("customPath:", customPath);
|
|
17
|
-
return customPath;
|
|
18
|
-
} else {
|
|
19
|
-
console.log(`The custom path to Chromium is invalid : ${customPath}`);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
const basePath = path.join(
|
|
23
|
-
process.env.HOME || process.env.USERPROFILE,
|
|
24
|
-
'.cache',
|
|
25
|
-
'puppeteer',
|
|
26
|
-
'chrome'
|
|
27
|
-
);
|
|
28
|
-
const chromiumPath = path.join(basePath, 'win64-135.0.7049.95', 'chrome-win64', 'chrome.exe');
|
|
29
|
-
|
|
30
|
-
if (!fs.existsSync(chromiumPath)) {
|
|
31
|
-
console.log("📦 Downloading Chromium 135.0.7049.95...");
|
|
32
|
-
await execAsync('npx puppeteer browsers install chrome@135.0.7049.95');
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
return chromiumPath;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async function launchBrowser (customChromiumPath) {
|
|
39
|
-
const executablePath = await ensureChromiumInstalled(customChromiumPath);
|
|
40
|
-
return puppeteer.launch({
|
|
41
|
-
executablePath: executablePath,
|
|
42
|
-
headless: true,
|
|
43
|
-
args: ["--no-sandbox", "--disable-setuid-sandbox"],
|
|
44
|
-
});
|
|
45
|
-
};
|
|
46
11
|
export async function getSibnetVideo(embedUrl) {
|
|
47
12
|
try {
|
|
48
13
|
const { data } = await axios.get(embedUrl, {
|
|
@@ -85,69 +50,13 @@ export async function getSendvidVideo(embedUrl) {
|
|
|
85
50
|
return null;
|
|
86
51
|
}
|
|
87
52
|
}
|
|
88
|
-
export async function getVidmolyVideo(embedUrl, customChromiumPath) {
|
|
89
|
-
if (embedUrl.includes("vidmoly.to/"))
|
|
90
|
-
embedUrl = embedUrl.replace("vidmoly.to/", "vidmoly.net/");
|
|
91
|
-
|
|
92
|
-
const browser = await launchBrowser(customChromiumPath);
|
|
93
|
-
|
|
94
|
-
const page = await browser.newPage();
|
|
95
|
-
await page.setUserAgent(
|
|
96
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
|
97
|
-
);
|
|
98
|
-
|
|
99
|
-
await page.goto(embedUrl, { waitUntil: "domcontentloaded" });
|
|
100
|
-
|
|
101
|
-
const start = Date.now();
|
|
102
|
-
const maxWait = 10000;
|
|
103
|
-
const maxReloads = 50;
|
|
104
|
-
let reloadCount = 0;
|
|
105
|
-
let videoUrl = null;
|
|
106
|
-
|
|
107
|
-
while (Date.now() - start < maxWait) {
|
|
108
|
-
videoUrl = await page.evaluate(() => {
|
|
109
|
-
const scripts = Array.from(document.querySelectorAll("script")).map(
|
|
110
|
-
(s) => s.textContent || ""
|
|
111
|
-
);
|
|
112
|
-
for (const s of scripts) {
|
|
113
|
-
const match = s.match(/file\s*:\s*"(https?:\/\/[^"]+\.m3u8[^"]*)"/);
|
|
114
|
-
if (match && match[1]) return match[1];
|
|
115
|
-
}
|
|
116
|
-
return null;
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
if (videoUrl) break;
|
|
120
|
-
|
|
121
|
-
const pleaseWait = await page.evaluate(() => {
|
|
122
|
-
const t = (document.title || "").toLowerCase();
|
|
123
|
-
const b = document.body?.innerText?.toLowerCase() || "";
|
|
124
|
-
return t.includes("please wait") || b.includes("please wait");
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
if (pleaseWait && reloadCount < maxReloads) {
|
|
128
|
-
reloadCount++;
|
|
129
|
-
console.log(
|
|
130
|
-
`[getVidmolyVideo] detected 'Please Wait' — reload #${reloadCount}`
|
|
131
|
-
);
|
|
132
|
-
await page.reload({ waitUntil: "domcontentloaded" }).catch(() => null);
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
await new Promise((r) => setTimeout(r, 300));
|
|
136
|
-
}
|
|
137
|
-
const openingTime = await page.evaluate(() => {
|
|
138
|
-
const st = typeof skipTime !== "undefined" ? skipTime : null;
|
|
139
|
-
const rs = typeof result !== "undefined" ? result : null;
|
|
140
|
-
return [rs, st];
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
await browser.close();
|
|
144
|
-
|
|
145
|
-
if (!videoUrl) console.warn("[getVidmolyVideo] timeout: video not found");
|
|
146
|
-
return { videoUrl, openingTime };
|
|
147
|
-
}
|
|
148
53
|
|
|
149
|
-
export async function
|
|
54
|
+
export async function getVidmolyOrOneuploadVideo(embedUrl) {
|
|
150
55
|
try {
|
|
56
|
+
if (embedUrl.includes("vidmoly.to/")) {
|
|
57
|
+
embedUrl = embedUrl.replace("vidmoly.to/", "vidmoly.net/");
|
|
58
|
+
}
|
|
59
|
+
console.log(embedUrl)
|
|
151
60
|
const { data } = await axios.get(embedUrl, {
|
|
152
61
|
headers: getHeaders(embedUrl),
|
|
153
62
|
});
|
|
@@ -163,20 +72,22 @@ export async function getOneuploadVideo(embedUrl) {
|
|
|
163
72
|
}
|
|
164
73
|
return null;
|
|
165
74
|
} catch (err) {
|
|
166
|
-
console.error("Erreur
|
|
75
|
+
console.error("Erreur getVidmolyVideo:", err.message);
|
|
167
76
|
return null;
|
|
168
77
|
}
|
|
169
78
|
}
|
|
170
79
|
|
|
171
|
-
|
|
172
|
-
|
|
80
|
+
import puppeteer from 'puppeteer';
|
|
81
|
+
|
|
82
|
+
export async function getMovearnpreOrSmoothpreVideo(embedUrl) {
|
|
83
|
+
const browser = await puppeteer.launch({ headless: true });
|
|
173
84
|
const page = await browser.newPage();
|
|
174
85
|
|
|
175
86
|
await page.setUserAgent(
|
|
176
87
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
|
177
88
|
);
|
|
178
89
|
|
|
179
|
-
await page.goto(embedUrl, { waitUntil:
|
|
90
|
+
await page.goto(embedUrl, { waitUntil: 'networkidle2' });
|
|
180
91
|
|
|
181
92
|
await page.waitForFunction('typeof jwplayer !== "undefined"');
|
|
182
93
|
|
|
@@ -187,6 +98,6 @@ export async function getMovearnpreOrSmoothpreVideo(embedUrl, customChromiumPath
|
|
|
187
98
|
});
|
|
188
99
|
|
|
189
100
|
await browser.close();
|
|
190
|
-
const finalUrl = embedUrl.split("/").slice(0, 3).join("/") + videoUrl
|
|
101
|
+
const finalUrl = embedUrl.split("/").slice(0, 3).join("/") + videoUrl
|
|
191
102
|
return finalUrl;
|
|
192
103
|
}
|