coomer-downloader 3.4.2 → 3.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +154 -151
- package/package.json +1 -1
- package/src/api/providers/gofile.ts +11 -9
- package/src/api/providers/reddit.ts +48 -42
- package/src/cli/{args-handler.ts → parse-args.ts} +1 -1
- package/src/core/downloader.ts +6 -1
- package/src/index.ts +6 -50
- package/src/main.ts +42 -0
- package/src/utils/error.ts +30 -0
- package/src/utils/requests.ts +5 -1
package/dist/index.js
CHANGED
|
@@ -364,19 +364,18 @@ async function getToken() {
|
|
|
364
364
|
throw new Error("Token Not Found");
|
|
365
365
|
}
|
|
366
366
|
async function getWebsiteToken() {
|
|
367
|
-
const
|
|
368
|
-
const
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
return match[1];
|
|
372
|
-
}
|
|
373
|
-
throw new Error("cannot get wt");
|
|
367
|
+
const config = await fetch3("https://gofile.io/dist/js/config.js").then((r) => r.text());
|
|
368
|
+
const wt = config.match(/appdata\.wt = "([^"]+)"/)?.[1];
|
|
369
|
+
if (wt) return wt;
|
|
370
|
+
throw new Error("Token Not Found");
|
|
374
371
|
}
|
|
375
372
|
async function getFolderFiles(id, token, websiteToken) {
|
|
376
373
|
const url = `https://api.gofile.io/contents/${id}?wt=${websiteToken}&cache=true}`;
|
|
377
374
|
const response = await fetch3(url, {
|
|
378
375
|
headers: {
|
|
379
|
-
Authorization: `Bearer ${token}
|
|
376
|
+
Authorization: `Bearer ${token}`,
|
|
377
|
+
referrer: "https://gofile.io",
|
|
378
|
+
"x-website-token": websiteToken
|
|
380
379
|
}
|
|
381
380
|
});
|
|
382
381
|
if (!response.ok) {
|
|
@@ -399,9 +398,9 @@ var GofileAPI = class {
|
|
|
399
398
|
const id = url.match(/gofile.io\/d\/(\w+)/)?.[1];
|
|
400
399
|
const token = await getToken();
|
|
401
400
|
const websiteToken = await getWebsiteToken();
|
|
401
|
+
setGlobalHeaders({ Cookie: `accountToken=${token}` });
|
|
402
402
|
const filelist = await getFolderFiles(id, token, websiteToken);
|
|
403
403
|
filelist.dirName = `gofile-${id}`;
|
|
404
|
-
setGlobalHeaders({ Cookie: `accountToken=${token}` });
|
|
405
404
|
return filelist;
|
|
406
405
|
}
|
|
407
406
|
};
|
|
@@ -421,142 +420,37 @@ var PlainFileAPI = class {
|
|
|
421
420
|
};
|
|
422
421
|
|
|
423
422
|
// src/api/providers/reddit.ts
|
|
424
|
-
import * as cheerio2 from "cheerio";
|
|
425
423
|
import { fetch as fetch4 } from "undici";
|
|
426
424
|
|
|
427
|
-
// src/utils/logger.ts
|
|
428
|
-
import pino from "pino";
|
|
429
|
-
var logger = pino(
|
|
430
|
-
{
|
|
431
|
-
level: "debug"
|
|
432
|
-
},
|
|
433
|
-
pino.destination({
|
|
434
|
-
dest: "./debug.log",
|
|
435
|
-
append: false,
|
|
436
|
-
sync: true
|
|
437
|
-
})
|
|
438
|
-
);
|
|
439
|
-
var logger_default = logger;
|
|
440
|
-
|
|
441
|
-
// src/api/providers/reddit.ts
|
|
442
|
-
async function getUserPage(user, offset) {
|
|
443
|
-
const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
|
|
444
|
-
return fetch4(url).then((r) => r.text());
|
|
445
|
-
}
|
|
446
|
-
async function getUserPosts(user) {
|
|
447
|
-
const posts = [];
|
|
448
|
-
for (let i = 1; i < 1e5; i++) {
|
|
449
|
-
const page = await getUserPage(user, i);
|
|
450
|
-
if (page.length < 1) break;
|
|
451
|
-
const $ = cheerio2.load(page);
|
|
452
|
-
const newPosts = $("a").map((_, a) => $(a).attr("href")).get().filter((href) => href?.startsWith("https://nsfw.xxx/post"));
|
|
453
|
-
logger_default.debug({ count: posts.length });
|
|
454
|
-
posts.push(...newPosts);
|
|
455
|
-
}
|
|
456
|
-
return posts;
|
|
457
|
-
}
|
|
458
|
-
async function getPostsData(posts) {
|
|
459
|
-
const filelist = new CoomerFileList();
|
|
460
|
-
for (const post of posts) {
|
|
461
|
-
const page = await fetch4(post).then((r) => r.text());
|
|
462
|
-
const $ = cheerio2.load(page);
|
|
463
|
-
const src = $(".sh-section .sh-section__image img").attr("src") || $(".sh-section .sh-section__image video source").attr("src") || null;
|
|
464
|
-
if (!src) continue;
|
|
465
|
-
const slug = post.split("post/")[1].split("?")[0];
|
|
466
|
-
const date = $(".sh-section .sh-section__passed").first().text().replace(/ /g, "-") || "";
|
|
467
|
-
const ext = src.split(".").pop();
|
|
468
|
-
const name = `${slug}-${date}.${ext}`;
|
|
469
|
-
logger_default.debug({ hehe: filelist.files.length, src });
|
|
470
|
-
filelist.files.push(CoomerFile.from({ name, url: src }));
|
|
471
|
-
}
|
|
472
|
-
return filelist;
|
|
473
|
-
}
|
|
474
|
-
var RedditAPI = class {
|
|
475
|
-
testURL(url) {
|
|
476
|
-
return /^\/user\/[\w-]+$/.test(url.pathname);
|
|
477
|
-
}
|
|
478
|
-
async getData(url) {
|
|
479
|
-
const user = url.match(/^\/user\/([\w-]+)/)?.[1];
|
|
480
|
-
const posts = await getUserPosts(user);
|
|
481
|
-
const filelist = await getPostsData(posts);
|
|
482
|
-
filelist.dirName = `${user}-reddit`;
|
|
483
|
-
return filelist;
|
|
484
|
-
}
|
|
485
|
-
};
|
|
486
|
-
|
|
487
|
-
// src/api/resolver.ts
|
|
488
|
-
var providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
|
|
489
|
-
async function resolveAPI(url_) {
|
|
490
|
-
const url = new URL(url_);
|
|
491
|
-
for (const p of providers) {
|
|
492
|
-
const provider = new p();
|
|
493
|
-
if (provider.testURL(url)) {
|
|
494
|
-
const filelist = await provider.getData(url.toString());
|
|
495
|
-
filelist.provider = provider;
|
|
496
|
-
return filelist;
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
throw Error("Invalid URL");
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
// src/cli/args-handler.ts
|
|
503
|
-
import yargs from "yargs";
|
|
504
|
-
import { hideBin } from "yargs/helpers";
|
|
505
|
-
function argumentHander() {
|
|
506
|
-
return yargs(hideBin(process.argv)).option("url", {
|
|
507
|
-
alias: "u",
|
|
508
|
-
type: "string",
|
|
509
|
-
description: "A URL from Coomer/Kemono/Bunkr/GoFile, a Reddit user (u/<username>), or a direct file link",
|
|
510
|
-
demandOption: true
|
|
511
|
-
}).option("dir", {
|
|
512
|
-
type: "string",
|
|
513
|
-
description: "The directory where files will be downloaded",
|
|
514
|
-
default: "./"
|
|
515
|
-
}).option("media", {
|
|
516
|
-
type: "string",
|
|
517
|
-
choices: ["video", "image"],
|
|
518
|
-
description: "The type of media to download: 'video', 'image', or 'all'. 'all' is the default."
|
|
519
|
-
}).option("include", {
|
|
520
|
-
type: "string",
|
|
521
|
-
default: "",
|
|
522
|
-
description: "Filter file names by a comma-separated list of keywords to include"
|
|
523
|
-
}).option("exclude", {
|
|
524
|
-
type: "string",
|
|
525
|
-
default: "",
|
|
526
|
-
description: "Filter file names by a comma-separated list of keywords to exclude"
|
|
527
|
-
}).option("min-size", {
|
|
528
|
-
type: "string",
|
|
529
|
-
default: "",
|
|
530
|
-
description: 'Minimum file size to download. Example: "1mb" or "500kb"'
|
|
531
|
-
}).option("max-size", {
|
|
532
|
-
type: "string",
|
|
533
|
-
default: "",
|
|
534
|
-
description: 'Maximum file size to download. Example: "1mb" or "500kb"'
|
|
535
|
-
}).option("skip", {
|
|
536
|
-
type: "number",
|
|
537
|
-
default: 0,
|
|
538
|
-
description: "Skips the first N files in the download queue"
|
|
539
|
-
}).option("remove-dupilicates", {
|
|
540
|
-
type: "boolean",
|
|
541
|
-
default: true,
|
|
542
|
-
description: "removes duplicates by url and file hash"
|
|
543
|
-
}).help().alias("help", "h").parseSync();
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
// src/cli/ui/index.tsx
|
|
547
|
-
import { render } from "ink";
|
|
548
|
-
import React10 from "react";
|
|
549
|
-
|
|
550
|
-
// src/cli/ui/app.tsx
|
|
551
|
-
import { Box as Box7 } from "ink";
|
|
552
|
-
import React9 from "react";
|
|
553
|
-
|
|
554
425
|
// src/core/downloader.ts
|
|
555
426
|
import fs2 from "node:fs";
|
|
556
427
|
import { Readable, Transform } from "node:stream";
|
|
557
428
|
import { pipeline as pipeline2 } from "node:stream/promises";
|
|
558
429
|
import { Subject } from "rxjs";
|
|
559
430
|
|
|
431
|
+
// src/utils/error.ts
|
|
432
|
+
function printError(err, options = {}) {
|
|
433
|
+
const e = err;
|
|
434
|
+
const status = Number(
|
|
435
|
+
e?.response?.status || e?.status || e?.message?.match(/\d{3}/)?.[0] || 500
|
|
436
|
+
);
|
|
437
|
+
const type = e?.code || e?.name || "Error";
|
|
438
|
+
const message = e?.message || "No details";
|
|
439
|
+
const quietList = options.quiet ?? [403, 404];
|
|
440
|
+
const isQuiet = quietList.includes(status);
|
|
441
|
+
console.error(
|
|
442
|
+
`\x1B[31m[ERROR]\x1B[0m \x1B[33m${status}\x1B[0m | \x1B[36m${type}\x1B[0m: ${message}`
|
|
443
|
+
);
|
|
444
|
+
if (options.context) {
|
|
445
|
+
console.error("\x1B[90mContext:\x1B[0m", options.context);
|
|
446
|
+
}
|
|
447
|
+
if (!isQuiet && e?.stack) {
|
|
448
|
+
console.error(`
|
|
449
|
+
\x1B[90mStack Trace:
|
|
450
|
+
${e.stack}\x1B[0m`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
560
454
|
// src/utils/promise.ts
|
|
561
455
|
async function sleep(time) {
|
|
562
456
|
return new Promise((resolve) => setTimeout(resolve, time));
|
|
@@ -708,7 +602,11 @@ var Downloader = class {
|
|
|
708
602
|
for (const file of this.filelist.files) {
|
|
709
603
|
file.active = true;
|
|
710
604
|
this.subject.next({ type: "FILE_DOWNLOADING_START" });
|
|
711
|
-
|
|
605
|
+
try {
|
|
606
|
+
await this.downloadFile(file);
|
|
607
|
+
} catch (e) {
|
|
608
|
+
printError(e, { quiet: [403], context: file.url });
|
|
609
|
+
}
|
|
712
610
|
file.active = false;
|
|
713
611
|
this.subject.next({ type: "FILE_DOWNLOADING_END" });
|
|
714
612
|
}
|
|
@@ -716,6 +614,114 @@ var Downloader = class {
|
|
|
716
614
|
}
|
|
717
615
|
};
|
|
718
616
|
|
|
617
|
+
// src/api/providers/reddit.ts
|
|
618
|
+
async function getUserPage(user, offset) {
|
|
619
|
+
const url = `https://nsfw.xxx/api/v1/user/${user}/newest?page=${offset}&types[]=image&types[]=video&types[]=gallery&nsfw[]=0&nsfw[]=1&nsfw[]=2&nsfw[]=3&nsfw[]=4`;
|
|
620
|
+
const res = await fetch4(url).then((r) => r.json());
|
|
621
|
+
return res;
|
|
622
|
+
}
|
|
623
|
+
async function getUserPostsData(user) {
|
|
624
|
+
const filelist = new CoomerFileList();
|
|
625
|
+
for (let i = 1; i < 1e4; i++) {
|
|
626
|
+
const { data } = await getUserPage(user, i);
|
|
627
|
+
if (data.posts.length < 1) break;
|
|
628
|
+
data.posts.forEach((post) => {
|
|
629
|
+
const date = post.publishedAt;
|
|
630
|
+
const title = post.content.title;
|
|
631
|
+
const name = `${date} ${title}`;
|
|
632
|
+
const preview = post.data.url;
|
|
633
|
+
const files = (post.data.videos_v2 || []).filter((f) => !f.url.includes("imgur"));
|
|
634
|
+
if (files?.length === 0 && preview) {
|
|
635
|
+
files.push({ format: "jpg", url: preview });
|
|
636
|
+
}
|
|
637
|
+
files.forEach(({ format, url }, i2) => {
|
|
638
|
+
const index = i2 > 0 ? ` ${i2}` : "";
|
|
639
|
+
const _name = `${name}${index}.${format}`;
|
|
640
|
+
filelist.files.push(CoomerFile.from({ name: _name, url }));
|
|
641
|
+
});
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
return filelist;
|
|
645
|
+
}
|
|
646
|
+
var RedditAPI = class {
|
|
647
|
+
testURL(url) {
|
|
648
|
+
return /^\/user\/[\w-]+$/.test(url.pathname);
|
|
649
|
+
}
|
|
650
|
+
async getData(url) {
|
|
651
|
+
const user = url.match(/\/user\/([\w-]+)/)?.[1];
|
|
652
|
+
const filelist = await getUserPostsData(user);
|
|
653
|
+
filelist.dirName = `${user}-reddit`;
|
|
654
|
+
return filelist;
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
|
|
658
|
+
// src/api/resolver.ts
|
|
659
|
+
var providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
|
|
660
|
+
async function resolveAPI(url_) {
|
|
661
|
+
const url = new URL(url_);
|
|
662
|
+
for (const p of providers) {
|
|
663
|
+
const provider = new p();
|
|
664
|
+
if (provider.testURL(url)) {
|
|
665
|
+
const filelist = await provider.getData(url.toString());
|
|
666
|
+
filelist.provider = provider;
|
|
667
|
+
return filelist;
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
throw Error("Invalid URL");
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// src/cli/parse-args.ts
|
|
674
|
+
import yargs from "yargs";
|
|
675
|
+
import { hideBin } from "yargs/helpers";
|
|
676
|
+
function parseArgs() {
|
|
677
|
+
return yargs(hideBin(process.argv)).option("url", {
|
|
678
|
+
alias: "u",
|
|
679
|
+
type: "string",
|
|
680
|
+
description: "A URL from Coomer/Kemono/Bunkr/GoFile, a Reddit user (u/<username>), or a direct file link",
|
|
681
|
+
demandOption: true
|
|
682
|
+
}).option("dir", {
|
|
683
|
+
type: "string",
|
|
684
|
+
description: "The directory where files will be downloaded",
|
|
685
|
+
default: "./"
|
|
686
|
+
}).option("media", {
|
|
687
|
+
type: "string",
|
|
688
|
+
choices: ["video", "image"],
|
|
689
|
+
description: "The type of media to download: 'video', 'image', or 'all'. 'all' is the default."
|
|
690
|
+
}).option("include", {
|
|
691
|
+
type: "string",
|
|
692
|
+
default: "",
|
|
693
|
+
description: "Filter file names by a comma-separated list of keywords to include"
|
|
694
|
+
}).option("exclude", {
|
|
695
|
+
type: "string",
|
|
696
|
+
default: "",
|
|
697
|
+
description: "Filter file names by a comma-separated list of keywords to exclude"
|
|
698
|
+
}).option("min-size", {
|
|
699
|
+
type: "string",
|
|
700
|
+
default: "",
|
|
701
|
+
description: 'Minimum file size to download. Example: "1mb" or "500kb"'
|
|
702
|
+
}).option("max-size", {
|
|
703
|
+
type: "string",
|
|
704
|
+
default: "",
|
|
705
|
+
description: 'Maximum file size to download. Example: "1mb" or "500kb"'
|
|
706
|
+
}).option("skip", {
|
|
707
|
+
type: "number",
|
|
708
|
+
default: 0,
|
|
709
|
+
description: "Skips the first N files in the download queue"
|
|
710
|
+
}).option("remove-dupilicates", {
|
|
711
|
+
type: "boolean",
|
|
712
|
+
default: true,
|
|
713
|
+
description: "removes duplicates by url and file hash"
|
|
714
|
+
}).help().alias("help", "h").parseSync();
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// src/cli/ui/index.tsx
|
|
718
|
+
import { render } from "ink";
|
|
719
|
+
import React10 from "react";
|
|
720
|
+
|
|
721
|
+
// src/cli/ui/app.tsx
|
|
722
|
+
import { Box as Box7 } from "ink";
|
|
723
|
+
import React9 from "react";
|
|
724
|
+
|
|
719
725
|
// src/cli/ui/components/file.tsx
|
|
720
726
|
import { Box as Box2, Spacer, Text as Text2 } from "ink";
|
|
721
727
|
import React3 from "react";
|
|
@@ -881,7 +887,7 @@ import { Box as Box6, Spacer as Spacer2, Text as Text6 } from "ink";
|
|
|
881
887
|
import React8 from "react";
|
|
882
888
|
|
|
883
889
|
// package.json
|
|
884
|
-
var version = "3.4.
|
|
890
|
+
var version = "3.4.3";
|
|
885
891
|
|
|
886
892
|
// src/cli/ui/components/titlebar.tsx
|
|
887
893
|
function TitleBar() {
|
|
@@ -915,10 +921,10 @@ function createReactInk() {
|
|
|
915
921
|
return render(/* @__PURE__ */ React10.createElement(App, null));
|
|
916
922
|
}
|
|
917
923
|
|
|
918
|
-
// src/
|
|
919
|
-
async function
|
|
924
|
+
// src/main.ts
|
|
925
|
+
async function main() {
|
|
920
926
|
createReactInk();
|
|
921
|
-
const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
|
|
927
|
+
const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } = parseArgs();
|
|
922
928
|
const filelist = await resolveAPI(url);
|
|
923
929
|
filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
|
|
924
930
|
if (removeDupilicates) {
|
|
@@ -935,12 +941,9 @@ async function run() {
|
|
|
935
941
|
await filelist.removeDuplicatesByHash();
|
|
936
942
|
}
|
|
937
943
|
}
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
process2.exit(1);
|
|
945
|
-
}
|
|
946
|
-
})();
|
|
944
|
+
|
|
945
|
+
// src/index.ts
|
|
946
|
+
main().then(() => process2.exit(0)).catch((err) => {
|
|
947
|
+
console.error(err);
|
|
948
|
+
process2.exit(1);
|
|
949
|
+
});
|
package/package.json
CHANGED
|
@@ -11,6 +11,7 @@ async function getToken(): Promise<string> {
|
|
|
11
11
|
const response = await fetch('https://api.gofile.io/accounts', {
|
|
12
12
|
method: 'POST',
|
|
13
13
|
});
|
|
14
|
+
|
|
14
15
|
const data = (await response.json()) as GoFileAPIToken;
|
|
15
16
|
if (data.status === 'ok') {
|
|
16
17
|
return data.data.token;
|
|
@@ -20,13 +21,12 @@ async function getToken(): Promise<string> {
|
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
async function getWebsiteToken() {
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
if (
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
throw new Error('cannot get wt');
|
|
24
|
+
const config = await fetch('https://gofile.io/dist/js/config.js').then((r) => r.text());
|
|
25
|
+
|
|
26
|
+
const wt = config.match(/appdata\.wt = "([^"]+)"/)?.[1];
|
|
27
|
+
if (wt) return wt;
|
|
28
|
+
|
|
29
|
+
throw new Error('Token Not Found');
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
async function getFolderFiles(
|
|
@@ -38,6 +38,8 @@ async function getFolderFiles(
|
|
|
38
38
|
const response = await fetch(url, {
|
|
39
39
|
headers: {
|
|
40
40
|
Authorization: `Bearer ${token}`,
|
|
41
|
+
referrer: 'https://gofile.io',
|
|
42
|
+
'x-website-token': websiteToken,
|
|
41
43
|
},
|
|
42
44
|
});
|
|
43
45
|
|
|
@@ -67,11 +69,11 @@ export class GofileAPI implements ProviderAPI {
|
|
|
67
69
|
const token = await getToken();
|
|
68
70
|
const websiteToken = await getWebsiteToken();
|
|
69
71
|
|
|
72
|
+
setGlobalHeaders({ Cookie: `accountToken=${token}` });
|
|
73
|
+
|
|
70
74
|
const filelist = await getFolderFiles(id, token, websiteToken);
|
|
71
75
|
filelist.dirName = `gofile-${id}`;
|
|
72
76
|
|
|
73
|
-
setGlobalHeaders({ Cookie: `accountToken=${token}` });
|
|
74
|
-
|
|
75
77
|
return filelist;
|
|
76
78
|
}
|
|
77
79
|
}
|
|
@@ -1,55 +1,62 @@
|
|
|
1
|
-
import * as cheerio from 'cheerio';
|
|
2
1
|
import { fetch } from 'undici';
|
|
3
|
-
import { CoomerFile } from '../../core
|
|
2
|
+
import { CoomerFile } from '../../core';
|
|
4
3
|
import { CoomerFileList } from '../../core/filelist';
|
|
5
|
-
import logger from '../../utils/logger';
|
|
6
4
|
import type { ProviderAPI } from '../provider';
|
|
7
5
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
6
|
+
type RedditAPIPosts = {
|
|
7
|
+
data: {
|
|
8
|
+
posts: Array<{
|
|
9
|
+
id: number;
|
|
10
|
+
content: {
|
|
11
|
+
title: string;
|
|
12
|
+
description: string;
|
|
13
|
+
};
|
|
14
|
+
data: {
|
|
15
|
+
url: string;
|
|
16
|
+
videos: {
|
|
17
|
+
mp4: string;
|
|
18
|
+
};
|
|
19
|
+
videos_v2: Array<{
|
|
20
|
+
format: string;
|
|
21
|
+
url: string;
|
|
22
|
+
}>;
|
|
23
|
+
};
|
|
24
|
+
publishedAt: string;
|
|
25
|
+
}>;
|
|
26
|
+
};
|
|
27
|
+
};
|
|
18
28
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
.filter((href) => href?.startsWith('https://nsfw.xxx/post'));
|
|
24
|
-
|
|
25
|
-
logger.debug({ count: posts.length });
|
|
26
|
-
posts.push(...newPosts);
|
|
27
|
-
}
|
|
28
|
-
return posts;
|
|
29
|
+
async function getUserPage(user: string, offset: number): Promise<RedditAPIPosts> {
|
|
30
|
+
const url = `https://nsfw.xxx/api/v1/user/${user}/newest?page=${offset}&types[]=image&types[]=video&types[]=gallery&nsfw[]=0&nsfw[]=1&nsfw[]=2&nsfw[]=3&nsfw[]=4`;
|
|
31
|
+
const res = await fetch(url).then((r) => r.json());
|
|
32
|
+
return res as RedditAPIPosts;
|
|
29
33
|
}
|
|
30
34
|
|
|
31
|
-
async function
|
|
35
|
+
async function getUserPostsData(user: string): Promise<CoomerFileList> {
|
|
32
36
|
const filelist = new CoomerFileList();
|
|
33
|
-
for (const post of posts) {
|
|
34
|
-
const page = await fetch(post).then((r) => r.text());
|
|
35
|
-
const $ = cheerio.load(page);
|
|
36
37
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
null;
|
|
38
|
+
for (let i = 1; i < 10_000; i++) {
|
|
39
|
+
const { data } = await getUserPage(user, i);
|
|
40
|
+
if (data.posts.length < 1) break;
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
data.posts.forEach((post) => {
|
|
43
|
+
const date = post.publishedAt;
|
|
44
|
+
const title = post.content.title;
|
|
45
|
+
const name = `${date} ${title}`;
|
|
43
46
|
|
|
44
|
-
|
|
45
|
-
const date =
|
|
46
|
-
$('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
|
|
47
|
+
const preview = post.data.url;
|
|
47
48
|
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
const files = (post.data.videos_v2 || []).filter((f) => !f.url.includes('imgur'));
|
|
50
|
+
if (files?.length === 0 && preview) {
|
|
51
|
+
files.push({ format: 'jpg', url: preview });
|
|
52
|
+
}
|
|
50
53
|
|
|
51
|
-
|
|
52
|
-
|
|
54
|
+
files.forEach(({ format, url }, i) => {
|
|
55
|
+
const index = i > 0 ? ` ${i}` : '';
|
|
56
|
+
const _name = `${name}${index}.${format}`;
|
|
57
|
+
filelist.files.push(CoomerFile.from({ name: _name, url }));
|
|
58
|
+
});
|
|
59
|
+
});
|
|
53
60
|
}
|
|
54
61
|
|
|
55
62
|
return filelist;
|
|
@@ -61,9 +68,8 @@ export class RedditAPI implements ProviderAPI {
|
|
|
61
68
|
}
|
|
62
69
|
|
|
63
70
|
public async getData(url: string): Promise<CoomerFileList> {
|
|
64
|
-
const user = url.match(
|
|
65
|
-
const
|
|
66
|
-
const filelist = await getPostsData(posts);
|
|
71
|
+
const user = url.match(/\/user\/([\w-]+)/)?.[1] as string;
|
|
72
|
+
const filelist = await getUserPostsData(user);
|
|
67
73
|
filelist.dirName = `${user}-reddit`;
|
|
68
74
|
return filelist;
|
|
69
75
|
}
|
package/src/core/downloader.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { Readable, Transform } from 'node:stream';
|
|
|
3
3
|
import { pipeline } from 'node:stream/promises';
|
|
4
4
|
import { Subject } from 'rxjs';
|
|
5
5
|
import type { AbortControllerSubject, DownloaderSubject } from '../types';
|
|
6
|
+
import { printError } from '../utils/error';
|
|
6
7
|
import { deleteFile, getFileSize, mkdir } from '../utils/io';
|
|
7
8
|
import { sleep } from '../utils/promise';
|
|
8
9
|
import { fetchByteRange } from '../utils/requests';
|
|
@@ -152,7 +153,11 @@ export class Downloader {
|
|
|
152
153
|
|
|
153
154
|
this.subject.next({ type: 'FILE_DOWNLOADING_START' });
|
|
154
155
|
|
|
155
|
-
|
|
156
|
+
try {
|
|
157
|
+
await this.downloadFile(file);
|
|
158
|
+
} catch (e) {
|
|
159
|
+
printError(e, { quiet: [403], context: file.url });
|
|
160
|
+
}
|
|
156
161
|
|
|
157
162
|
file.active = false;
|
|
158
163
|
|
package/src/index.ts
CHANGED
|
@@ -1,55 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env -S node --no-warnings=ExperimentalWarning
|
|
2
2
|
|
|
3
3
|
import process from 'node:process';
|
|
4
|
-
import {
|
|
5
|
-
import { argumentHander } from './cli/args-handler';
|
|
6
|
-
import { createReactInk } from './cli/ui';
|
|
7
|
-
import { useInkStore } from './cli/ui/store';
|
|
8
|
-
import { Downloader } from './core';
|
|
9
|
-
import { parseSizeValue } from './utils/filters';
|
|
10
|
-
import { setGlobalHeaders } from './utils/requests';
|
|
4
|
+
import { main } from './main';
|
|
11
5
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
argumentHander();
|
|
17
|
-
|
|
18
|
-
const filelist = await resolveAPI(url);
|
|
19
|
-
|
|
20
|
-
filelist
|
|
21
|
-
.setDirPath(dir)
|
|
22
|
-
.skip(skip)
|
|
23
|
-
.filterByText(include, exclude)
|
|
24
|
-
.filterByMediaType(media);
|
|
25
|
-
|
|
26
|
-
if (removeDupilicates) {
|
|
27
|
-
filelist.removeURLDuplicates();
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
|
|
31
|
-
const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
|
|
32
|
-
|
|
33
|
-
await filelist.calculateFileSizes();
|
|
34
|
-
|
|
35
|
-
setGlobalHeaders({ Referer: url });
|
|
36
|
-
|
|
37
|
-
const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
|
|
38
|
-
useInkStore.getState().setDownloader(downloader);
|
|
39
|
-
|
|
40
|
-
await downloader.downloadFiles();
|
|
41
|
-
|
|
42
|
-
if (removeDupilicates) {
|
|
43
|
-
await filelist.removeDuplicatesByHash();
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
(async () => {
|
|
48
|
-
try {
|
|
49
|
-
await run();
|
|
50
|
-
process.exit(0);
|
|
51
|
-
} catch (err) {
|
|
52
|
-
console.error('Fatal error:', err);
|
|
6
|
+
main()
|
|
7
|
+
.then(() => process.exit(0))
|
|
8
|
+
.catch((err) => {
|
|
9
|
+
console.error(err);
|
|
53
10
|
process.exit(1);
|
|
54
|
-
}
|
|
55
|
-
})();
|
|
11
|
+
});
|
package/src/main.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { resolveAPI } from './api';
|
|
2
|
+
import { parseArgs } from './cli/parse-args';
|
|
3
|
+
import { createReactInk } from './cli/ui';
|
|
4
|
+
import { useInkStore } from './cli/ui/store';
|
|
5
|
+
import { Downloader } from './core';
|
|
6
|
+
import { parseSizeValue } from './utils/filters';
|
|
7
|
+
import { setGlobalHeaders } from './utils/requests';
|
|
8
|
+
|
|
9
|
+
export async function main() {
|
|
10
|
+
createReactInk();
|
|
11
|
+
|
|
12
|
+
const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
|
|
13
|
+
parseArgs();
|
|
14
|
+
|
|
15
|
+
const filelist = await resolveAPI(url);
|
|
16
|
+
|
|
17
|
+
filelist
|
|
18
|
+
.setDirPath(dir)
|
|
19
|
+
.skip(skip)
|
|
20
|
+
.filterByText(include, exclude)
|
|
21
|
+
.filterByMediaType(media);
|
|
22
|
+
|
|
23
|
+
if (removeDupilicates) {
|
|
24
|
+
filelist.removeURLDuplicates();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
|
|
28
|
+
const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
|
|
29
|
+
|
|
30
|
+
await filelist.calculateFileSizes();
|
|
31
|
+
|
|
32
|
+
setGlobalHeaders({ Referer: url });
|
|
33
|
+
|
|
34
|
+
const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
|
|
35
|
+
useInkStore.getState().setDownloader(downloader);
|
|
36
|
+
|
|
37
|
+
await downloader.downloadFiles();
|
|
38
|
+
|
|
39
|
+
if (removeDupilicates) {
|
|
40
|
+
await filelist.removeDuplicatesByHash();
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
interface PrintOptions {
|
|
2
|
+
quiet?: number[];
|
|
3
|
+
context?: any;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export function printError(err: unknown, options: PrintOptions = {}): void {
|
|
7
|
+
const e = err as any;
|
|
8
|
+
|
|
9
|
+
const status = Number(
|
|
10
|
+
e?.response?.status || e?.status || e?.message?.match(/\d{3}/)?.[0] || 500,
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
const type = e?.code || e?.name || 'Error';
|
|
14
|
+
const message = e?.message || 'No details';
|
|
15
|
+
|
|
16
|
+
const quietList = options.quiet ?? [403, 404];
|
|
17
|
+
const isQuiet = quietList.includes(status);
|
|
18
|
+
|
|
19
|
+
console.error(
|
|
20
|
+
`\x1b[31m[ERROR]\x1b[0m \x1b[33m${status}\x1b[0m | \x1b[36m${type}\x1b[0m: ${message}`,
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
if (options.context) {
|
|
24
|
+
console.error('\x1b[90mContext:\x1b[0m', options.context);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (!isQuiet && e?.stack) {
|
|
28
|
+
console.error(`\n\x1b[90mStack Trace:\n${e.stack}\x1b[0m`);
|
|
29
|
+
}
|
|
30
|
+
}
|
package/src/utils/requests.ts
CHANGED
|
@@ -29,7 +29,11 @@ export function fetchWithGlobalHeader(url: string) {
|
|
|
29
29
|
return fetch(url, { headers: requestHeaders });
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
export function fetchByteRange(
|
|
32
|
+
export function fetchByteRange(
|
|
33
|
+
url: string,
|
|
34
|
+
downloadedSize: number,
|
|
35
|
+
signal?: AbortSignal,
|
|
36
|
+
) {
|
|
33
37
|
const requestHeaders = new Headers(HeadersDefault);
|
|
34
38
|
requestHeaders.set('Range', `bytes=${downloadedSize}-`);
|
|
35
39
|
return fetch(url, { headers: requestHeaders, signal });
|