coomer-downloader 3.4.2 → 3.4.3

package/dist/index.js

@@ -421,142 +421,37 @@ var PlainFileAPI = class {
 };
 
 // src/api/providers/reddit.ts
-import * as cheerio2 from "cheerio";
 import { fetch as fetch4 } from "undici";
 
-// src/utils/logger.ts
-import pino from "pino";
-var logger = pino(
-  {
-    level: "debug"
-  },
-  pino.destination({
-    dest: "./debug.log",
-    append: false,
-    sync: true
-  })
-);
-var logger_default = logger;
-
-// src/api/providers/reddit.ts
-async function getUserPage(user, offset) {
-  const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
-  return fetch4(url).then((r) => r.text());
-}
-async function getUserPosts(user) {
-  const posts = [];
-  for (let i = 1; i < 1e5; i++) {
-    const page = await getUserPage(user, i);
-    if (page.length < 1) break;
-    const $ = cheerio2.load(page);
-    const newPosts = $("a").map((_, a) => $(a).attr("href")).get().filter((href) => href?.startsWith("https://nsfw.xxx/post"));
-    logger_default.debug({ count: posts.length });
-    posts.push(...newPosts);
-  }
-  return posts;
-}
-async function getPostsData(posts) {
-  const filelist = new CoomerFileList();
-  for (const post of posts) {
-    const page = await fetch4(post).then((r) => r.text());
-    const $ = cheerio2.load(page);
-    const src = $(".sh-section .sh-section__image img").attr("src") || $(".sh-section .sh-section__image video source").attr("src") || null;
-    if (!src) continue;
-    const slug = post.split("post/")[1].split("?")[0];
-    const date = $(".sh-section .sh-section__passed").first().text().replace(/ /g, "-") || "";
-    const ext = src.split(".").pop();
-    const name = `${slug}-${date}.${ext}`;
-    logger_default.debug({ hehe: filelist.files.length, src });
-    filelist.files.push(CoomerFile.from({ name, url: src }));
-  }
-  return filelist;
-}
-var RedditAPI = class {
-  testURL(url) {
-    return /^\/user\/[\w-]+$/.test(url.pathname);
-  }
-  async getData(url) {
-    const user = url.match(/^\/user\/([\w-]+)/)?.[1];
-    const posts = await getUserPosts(user);
-    const filelist = await getPostsData(posts);
-    filelist.dirName = `${user}-reddit`;
-    return filelist;
-  }
-};
-
-// src/api/resolver.ts
-var providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
-async function resolveAPI(url_) {
-  const url = new URL(url_);
-  for (const p of providers) {
-    const provider = new p();
-    if (provider.testURL(url)) {
-      const filelist = await provider.getData(url.toString());
-      filelist.provider = provider;
-      return filelist;
-    }
-  }
-  throw Error("Invalid URL");
-}
-
-// src/cli/args-handler.ts
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-function argumentHander() {
-  return yargs(hideBin(process.argv)).option("url", {
-    alias: "u",
-    type: "string",
-    description: "A URL from Coomer/Kemono/Bunkr/GoFile, a Reddit user (u/<username>), or a direct file link",
-    demandOption: true
-  }).option("dir", {
-    type: "string",
-    description: "The directory where files will be downloaded",
-    default: "./"
-  }).option("media", {
-    type: "string",
-    choices: ["video", "image"],
-    description: "The type of media to download: 'video', 'image', or 'all'. 'all' is the default."
-  }).option("include", {
-    type: "string",
-    default: "",
-    description: "Filter file names by a comma-separated list of keywords to include"
-  }).option("exclude", {
-    type: "string",
-    default: "",
-    description: "Filter file names by a comma-separated list of keywords to exclude"
-  }).option("min-size", {
-    type: "string",
-    default: "",
-    description: 'Minimum file size to download. Example: "1mb" or "500kb"'
-  }).option("max-size", {
-    type: "string",
-    default: "",
-    description: 'Maximum file size to download. Example: "1mb" or "500kb"'
-  }).option("skip", {
-    type: "number",
-    default: 0,
-    description: "Skips the first N files in the download queue"
-  }).option("remove-dupilicates", {
-    type: "boolean",
-    default: true,
-    description: "removes duplicates by url and file hash"
-  }).help().alias("help", "h").parseSync();
-}
-
-// src/cli/ui/index.tsx
-import { render } from "ink";
-import React10 from "react";
-
-// src/cli/ui/app.tsx
-import { Box as Box7 } from "ink";
-import React9 from "react";
-
 // src/core/downloader.ts
 import fs2 from "node:fs";
 import { Readable, Transform } from "node:stream";
 import { pipeline as pipeline2 } from "node:stream/promises";
 import { Subject } from "rxjs";
 
+// src/utils/error.ts
+function printError(err, options = {}) {
+  const e = err;
+  const status = Number(
+    e?.response?.status || e?.status || e?.message?.match(/\d{3}/)?.[0] || 500
+  );
+  const type = e?.code || e?.name || "Error";
+  const message = e?.message || "No details";
+  const quietList = options.quiet ?? [403, 404];
+  const isQuiet = quietList.includes(status);
+  console.error(
+    `\x1B[31m[ERROR]\x1B[0m \x1B[33m${status}\x1B[0m | \x1B[36m${type}\x1B[0m: ${message}`
+  );
+  if (options.context) {
+    console.error("\x1B[90mContext:\x1B[0m", options.context);
+  }
+  if (!isQuiet && e?.stack) {
+    console.error(`
+\x1B[90mStack Trace:
+${e.stack}\x1B[0m`);
+  }
+}
+
 // src/utils/promise.ts
 async function sleep(time) {
   return new Promise((resolve) => setTimeout(resolve, time));
@@ -708,7 +603,11 @@ var Downloader = class {
     for (const file of this.filelist.files) {
       file.active = true;
       this.subject.next({ type: "FILE_DOWNLOADING_START" });
-      await this.downloadFile(file);
+      try {
+        await this.downloadFile(file);
+      } catch (e) {
+        printError(e, { quiet: [403], context: file.url });
+      }
       file.active = false;
       this.subject.next({ type: "FILE_DOWNLOADING_END" });
     }
@@ -716,6 +615,114 @@ var Downloader = class {
   }
 };
 
+// src/api/providers/reddit.ts
+async function getUserPage(user, offset) {
+  const url = `https://nsfw.xxx/api/v1/user/${user}/newest?page=${offset}&types[]=image&types[]=video&types[]=gallery&nsfw[]=0&nsfw[]=1&nsfw[]=2&nsfw[]=3&nsfw[]=4`;
+  const res = await fetch4(url).then((r) => r.json());
+  return res;
+}
+async function getUserPostsData(user) {
+  const filelist = new CoomerFileList();
+  for (let i = 1; i < 1e4; i++) {
+    const { data } = await getUserPage(user, i);
+    if (data.posts.length < 1) break;
+    data.posts.forEach((post) => {
+      const date = post.publishedAt;
+      const title = post.content.title;
+      const name = `${date} ${title}`;
+      const preview = post.data.url;
+      const files = (post.data.videos_v2 || []).filter((f) => !f.url.includes("imgur"));
+      if (files?.length === 0 && preview) {
+        files.push({ format: "jpg", url: preview });
+      }
+      files.forEach(({ format, url }, i2) => {
+        const index = i2 > 0 ? ` ${i2}` : "";
+        const _name = `${name}${index}.${format}`;
+        filelist.files.push(CoomerFile.from({ name: _name, url }));
+      });
+    });
+  }
+  return filelist;
+}
+var RedditAPI = class {
+  testURL(url) {
+    return /^\/user\/[\w-]+$/.test(url.pathname);
+  }
+  async getData(url) {
+    const user = url.match(/\/user\/([\w-]+)/)?.[1];
+    const filelist = await getUserPostsData(user);
+    filelist.dirName = `${user}-reddit`;
+    return filelist;
+  }
+};
+
+// src/api/resolver.ts
+var providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
+async function resolveAPI(url_) {
+  const url = new URL(url_);
+  for (const p of providers) {
+    const provider = new p();
+    if (provider.testURL(url)) {
+      const filelist = await provider.getData(url.toString());
+      filelist.provider = provider;
+      return filelist;
+    }
+  }
+  throw Error("Invalid URL");
+}
+
+// src/cli/parse-args.ts
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+function parseArgs() {
+  return yargs(hideBin(process.argv)).option("url", {
+    alias: "u",
+    type: "string",
+    description: "A URL from Coomer/Kemono/Bunkr/GoFile, a Reddit user (u/<username>), or a direct file link",
+    demandOption: true
+  }).option("dir", {
+    type: "string",
+    description: "The directory where files will be downloaded",
+    default: "./"
+  }).option("media", {
+    type: "string",
+    choices: ["video", "image"],
+    description: "The type of media to download: 'video', 'image', or 'all'. 'all' is the default."
+  }).option("include", {
+    type: "string",
+    default: "",
+    description: "Filter file names by a comma-separated list of keywords to include"
+  }).option("exclude", {
+    type: "string",
+    default: "",
+    description: "Filter file names by a comma-separated list of keywords to exclude"
+  }).option("min-size", {
+    type: "string",
+    default: "",
+    description: 'Minimum file size to download. Example: "1mb" or "500kb"'
+  }).option("max-size", {
+    type: "string",
+    default: "",
+    description: 'Maximum file size to download. Example: "1mb" or "500kb"'
+  }).option("skip", {
+    type: "number",
+    default: 0,
+    description: "Skips the first N files in the download queue"
+  }).option("remove-dupilicates", {
+    type: "boolean",
+    default: true,
+    description: "removes duplicates by url and file hash"
+  }).help().alias("help", "h").parseSync();
+}
+
+// src/cli/ui/index.tsx
+import { render } from "ink";
+import React10 from "react";
+
+// src/cli/ui/app.tsx
+import { Box as Box7 } from "ink";
+import React9 from "react";
+
 // src/cli/ui/components/file.tsx
 import { Box as Box2, Spacer, Text as Text2 } from "ink";
 import React3 from "react";
@@ -881,7 +888,7 @@ import { Box as Box6, Spacer as Spacer2, Text as Text6 } from "ink";
 import React8 from "react";
 
 // package.json
-var version = "3.4.2";
+var version = "3.4.3";
 
 // src/cli/ui/components/titlebar.tsx
 function TitleBar() {
@@ -915,10 +922,10 @@ function createReactInk() {
   return render(/* @__PURE__ */ React10.createElement(App, null));
 }
 
-// src/index.ts
-async function run() {
+// src/main.ts
+async function main() {
   createReactInk();
-  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } = argumentHander();
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } = parseArgs();
   const filelist = await resolveAPI(url);
   filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
   if (removeDupilicates) {
@@ -935,12 +942,9 @@ async function run() {
     await filelist.removeDuplicatesByHash();
   }
 }
-(async () => {
-  try {
-    await run();
-    process2.exit(0);
-  } catch (err) {
-    console.error("Fatal error:", err);
-    process2.exit(1);
-  }
-})();
+
+// src/index.ts
+main().then(() => process2.exit(0)).catch((err) => {
+  console.error(err);
+  process2.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.4.2",
+  "version": "3.4.3",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",

package/src/api/providers/reddit.ts

@@ -1,55 +1,62 @@
-import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile } from '../../core/file';
+import { CoomerFile } from '../../core';
 import { CoomerFileList } from '../../core/filelist';
-import logger from '../../utils/logger';
 import type { ProviderAPI } from '../provider';
 
-async function getUserPage(user: string, offset: number) {
-  const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
-  return fetch(url).then((r) => r.text());
-}
-
-async function getUserPosts(user: string): Promise<string[]> {
-  const posts = [];
-  for (let i = 1; i < 100000; i++) {
-    const page = await getUserPage(user, i);
-    if (page.length < 1) break;
+type RedditAPIPosts = {
+  data: {
+    posts: Array<{
+      id: number;
+      content: {
+        title: string;
+        description: string;
+      };
+      data: {
+        url: string;
+        videos: {
+          mp4: string;
+        };
+        videos_v2: Array<{
+          format: string;
+          url: string;
+        }>;
+      };
+      publishedAt: string;
+    }>;
+  };
+};
 
-    const $ = cheerio.load(page);
-    const newPosts = $('a')
-      .map((_, a) => $(a).attr('href'))
-      .get()
-      .filter((href) => href?.startsWith('https://nsfw.xxx/post'));
-
-    logger.debug({ count: posts.length });
-    posts.push(...newPosts);
-  }
-  return posts;
+async function getUserPage(user: string, offset: number): Promise<RedditAPIPosts> {
+  const url = `https://nsfw.xxx/api/v1/user/${user}/newest?page=${offset}&types[]=image&types[]=video&types[]=gallery&nsfw[]=0&nsfw[]=1&nsfw[]=2&nsfw[]=3&nsfw[]=4`;
+  const res = await fetch(url).then((r) => r.json());
+  return res as RedditAPIPosts;
 }
 
-async function getPostsData(posts: string[]): Promise<CoomerFileList> {
+async function getUserPostsData(user: string): Promise<CoomerFileList> {
   const filelist = new CoomerFileList();
-  for (const post of posts) {
-    const page = await fetch(post).then((r) => r.text());
-    const $ = cheerio.load(page);
 
-    const src =
-      $('.sh-section .sh-section__image img').attr('src') ||
-      $('.sh-section .sh-section__image video source').attr('src') ||
-      null;
+  for (let i = 1; i < 10_000; i++) {
+    const { data } = await getUserPage(user, i);
+    if (data.posts.length < 1) break;
 
-    if (!src) continue;
+    data.posts.forEach((post) => {
+      const date = post.publishedAt;
+      const title = post.content.title;
+      const name = `${date} ${title}`;
 
-    const slug = post.split('post/')[1].split('?')[0];
-    const date =
-      $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
+      const preview = post.data.url;
 
-    const ext = src.split('.').pop();
-    const name = `${slug}-${date}.${ext}`;
+      const files = (post.data.videos_v2 || []).filter((f) => !f.url.includes('imgur'));
+      if (files?.length === 0 && preview) {
+        files.push({ format: 'jpg', url: preview });
+      }
 
-    logger.debug({ hehe: filelist.files.length, src });
-    filelist.files.push(CoomerFile.from({ name, url: src }));
+      files.forEach(({ format, url }, i) => {
+        const index = i > 0 ? ` ${i}` : '';
+        const _name = `${name}${index}.${format}`;
+        filelist.files.push(CoomerFile.from({ name: _name, url }));
+      });
+    });
   }
 
   return filelist;
@@ -61,9 +68,8 @@ export class RedditAPI implements ProviderAPI {
   }
 
   public async getData(url: string): Promise<CoomerFileList> {
-    const user = url.match(/^\/user\/([\w-]+)/)?.[1] as string;
-    const posts = await getUserPosts(user);
-    const filelist = await getPostsData(posts);
+    const user = url.match(/\/user\/([\w-]+)/)?.[1] as string;
+    const filelist = await getUserPostsData(user);
     filelist.dirName = `${user}-reddit`;
     return filelist;
   }

package/src/cli/{args-handler.ts → parse-args.ts}

@@ -1,7 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-export function argumentHander() {
+export function parseArgs() {
   return yargs(hideBin(process.argv))
     .option('url', {
       alias: 'u',

package/src/core/downloader.ts

@@ -3,6 +3,7 @@ import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
 import type { AbortControllerSubject, DownloaderSubject } from '../types';
+import { printError } from '../utils/error';
 import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
 import { fetchByteRange } from '../utils/requests';
@@ -152,7 +153,11 @@ export class Downloader {
 
       this.subject.next({ type: 'FILE_DOWNLOADING_START' });
 
-      await this.downloadFile(file);
+      try {
+        await this.downloadFile(file);
+      } catch (e) {
+        printError(e, { quiet: [403], context: file.url });
+      }
 
       file.active = false;
 

package/src/index.ts

@@ -1,55 +1,11 @@
 #!/usr/bin/env -S node --no-warnings=ExperimentalWarning
 
 import process from 'node:process';
-import { resolveAPI } from './api';
-import { argumentHander } from './cli/args-handler';
-import { createReactInk } from './cli/ui';
-import { useInkStore } from './cli/ui/store';
-import { Downloader } from './core';
-import { parseSizeValue } from './utils/filters';
-import { setGlobalHeaders } from './utils/requests';
+import { main } from './main';
 
-async function run() {
-  createReactInk();
-
-  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
-    argumentHander();
-
-  const filelist = await resolveAPI(url);
-
-  filelist
-    .setDirPath(dir)
-    .skip(skip)
-    .filterByText(include, exclude)
-    .filterByMediaType(media);
-
-  if (removeDupilicates) {
-    filelist.removeURLDuplicates();
-  }
-
-  const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
-  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
-
-  await filelist.calculateFileSizes();
-
-  setGlobalHeaders({ Referer: url });
-
-  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
-  useInkStore.getState().setDownloader(downloader);
-
-  await downloader.downloadFiles();
-
-  if (removeDupilicates) {
-    await filelist.removeDuplicatesByHash();
-  }
-}
-
-(async () => {
-  try {
-    await run();
-    process.exit(0);
-  } catch (err) {
-    console.error('Fatal error:', err);
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(err);
     process.exit(1);
-  }
-})();
+  });

package/src/main.ts

@@ -0,0 +1,42 @@
+import { resolveAPI } from './api';
+import { parseArgs } from './cli/parse-args';
+import { createReactInk } from './cli/ui';
+import { useInkStore } from './cli/ui/store';
+import { Downloader } from './core';
+import { parseSizeValue } from './utils/filters';
+import { setGlobalHeaders } from './utils/requests';
+
+export async function main() {
+  createReactInk();
+
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
+    parseArgs();
+
+  const filelist = await resolveAPI(url);
+
+  filelist
+    .setDirPath(dir)
+    .skip(skip)
+    .filterByText(include, exclude)
+    .filterByMediaType(media);
+
+  if (removeDupilicates) {
+    filelist.removeURLDuplicates();
+  }
+
+  const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
+  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
+
+  await filelist.calculateFileSizes();
+
+  setGlobalHeaders({ Referer: url });
+
+  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
+  useInkStore.getState().setDownloader(downloader);
+
+  await downloader.downloadFiles();
+
+  if (removeDupilicates) {
+    await filelist.removeDuplicatesByHash();
+  }
+}

package/src/utils/error.ts

@@ -0,0 +1,30 @@
+interface PrintOptions {
+  quiet?: number[];
+  context?: any;
+}
+
+export function printError(err: unknown, options: PrintOptions = {}): void {
+  const e = err as any;
+
+  const status = Number(
+    e?.response?.status || e?.status || e?.message?.match(/\d{3}/)?.[0] || 500,
+  );
+
+  const type = e?.code || e?.name || 'Error';
+  const message = e?.message || 'No details';
+
+  const quietList = options.quiet ?? [403, 404];
+  const isQuiet = quietList.includes(status);
+
+  console.error(
+    `\x1b[31m[ERROR]\x1b[0m \x1b[33m${status}\x1b[0m | \x1b[36m${type}\x1b[0m: ${message}`,
+  );
+
+  if (options.context) {
+    console.error('\x1b[90mContext:\x1b[0m', options.context);
+  }
+
+  if (!isQuiet && e?.stack) {
+    console.error(`\n\x1b[90mStack Trace:\n${e.stack}\x1b[0m`);
+  }
+}