coomer-downloader 2.6.5 → 3.1.0

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "coomer-downloader",
-  "version": "2.6.5",
+  "version": "3.1.0",
   "author": "smartacephal",
   "license": "MIT",
-  "description": "Downloads images/videos from Coomer/Kemono/Reddit-NSFW user posts",
+  "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",
   "keywords": [
     "downloader",
     "cli",
@@ -15,30 +15,41 @@
     "kemono",
     "reddit",
     "nsfw.xxx",
-    "bunkr"
+    "bunkr",
+    "gofile"
   ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/smartacephale/coomer-downloader.git"
   },
-  "main": "index.js",
+  "main": "./src/index.ts",
   "type": "module",
   "scripts": {
-    "start": "node ./index.js",
-    "test": "echo \"Error: no test specified\""
+    "start": "tsx ./src/index.ts",
+    "build": "node build.js",
+    "start-build": "npm run build && node ./dist/index.js"
   },
   "bin": {
-    "coomer-downloader": "index.js"
+    "coomer-downloader": "./dist/index.js"
   },
   "bugs": {
     "url": "https://github.com/smartacephale/coomer-downloader",
     "email": "atm.mormon@protonmail.com"
   },
   "dependencies": {
-    "cheerio": "^1.0.0",
+    "cheerio": "1.1.0",
     "cli-progress": "^3.12.0",
-    "fetch-cookie": "^3.1.0",
-    "node-fetch": "^3.3.2",
+    "http-cookie-agent": "^6.0.0",
+    "rxjs": "^7.8.2",
+    "tough-cookie": "5.1.2",
+    "undici": "^6.22.0",
     "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.6",
+    "@types/node": "^24.10.0",
+    "@types/yargs": "^17.0.34",
+    "esbuild": "^0.27.0",
+    "tsx": "^4.20.6"
   }
 }

package/src/api/{bunkr.js → bunkr.ts}

@@ -1,34 +1,36 @@
 import * as cheerio from 'cheerio';
-import { testMediaType, fetch } from './../utils/index.js';
+import { fetch } from 'undici';
+import type { ApiResult, MediaType } from '../types/index.js';
+import { testMediaType } from '../utils/index.js';
 
-async function getEncryptionData(slug) {
+type EncData = { url: string; timestamp: number };
+
+async function getEncryptionData(slug: string): Promise<EncData> {
   const response = await fetch('https://bunkr.cr/api/vs', {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ slug: slug }),
   });
-  return await response.json();
+  return (await response.json()) as EncData;
 }
 
-function decryptEncryptedUrl(encryptionData) {
+function decryptEncryptedUrl(encryptionData: EncData) {
   const secretKey = `SECRET_KEY_${Math.floor(encryptionData.timestamp / 3600)}`;
   const encryptedUrlBuffer = Buffer.from(encryptionData.url, 'base64');
   const secretKeyBuffer = Buffer.from(secretKey, 'utf-8');
   return Array.from(encryptedUrlBuffer)
-    .map((byte, i) =>
-      String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]),
-    )
+    .map((byte, i) => String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]))
     .join('');
 }
 
-async function getFileData(url, name) {
-  const slug = url.split('/').pop();
+async function getFileData(url: string, name: string) {
+  const slug = url.split('/').pop() as string;
   const encryptionData = await getEncryptionData(slug);
   const src = decryptEncryptedUrl(encryptionData);
-  return { name, src };
+  return { name, url: src };
 }
 
-async function getGalleryFiles(url, mediaType) {
+async function getGalleryFiles(url: string, mediaType: MediaType) {
   const data = [];
   const page = await fetch(url).then((r) => r.text());
   const $ = cheerio.load(page);
@@ -42,26 +44,24 @@ async function getGalleryFiles(url, mediaType) {
     return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
   }
 
-  const fileNames = Array.from(
-    $('div[title]').map((_, e) => $(e).attr('title')),
-  );
+  const fileNames = Array.from($('div[title]').map((_, e) => $(e).attr('title')));
 
   const files = Array.from($('a').map((_, e) => $(e).attr('href')))
     .filter((a) => /\/f\/\w+/.test(a))
     .map((a, i) => ({
-      src: `${url_.origin}${a}`,
-      name: fileNames[i] || src.split('/').pop(),
+      url: `${url_.origin}${a}`,
+      name: fileNames[i] || (url.split('/').pop() as string),
     }));
 
-  for (const { name, src } of files) {
-    const res = await getFileData(src, name);
+  for (const { name, url } of files) {
+    const res = await getFileData(url, name);
     data.push(res);
   }
 
   return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
 }
 
-export async function getBunkrData(url, mediaType) {
+export async function getBunkrData(url: string, mediaType: MediaType): Promise<ApiResult> {
   const { files, title } = await getGalleryFiles(url, mediaType);
   const dirName = `${title.split('|')[0].trim()}-bunkr`;
   return { dirName, files };

package/src/api/coomer-api.ts

@@ -0,0 +1,93 @@
+import type { ApiResult, File, MediaType } from '../types/index.js';
+import { fetchWithGlobalHeader, isImage, setGlobalHeaders, testMediaType } from '../utils/index.js';
+
+type CoomerUser = { domain: string; service: string; id: string; name?: string };
+type CoomerUserApi = { name: string };
+type CoomerFile = { path: string; name: string };
+type CoomerPost = {
+  title: string;
+  content: string;
+  published: string;
+  attachments: CoomerFile[];
+  file: CoomerFile;
+};
+
+const SERVERS = ['n1', 'n2', 'n3', 'n4'];
+
+export function tryFixCoomerUrl(url: string, attempts: number) {
+  if (attempts < 2 && isImage(url)) {
+    return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
+  }
+  const server = url.match(/n\d\./)?.[0].slice(0, 2) as string;
+  const i = SERVERS.indexOf(server);
+  if (i !== -1) {
+    const newServer = SERVERS[(i + 1) % SERVERS.length];
+    return url.replace(/n\d./, `${newServer}.`);
+  }
+  return url;
+}
+
+async function getUserProfileAPI(user: CoomerUser): Promise<CoomerUserApi> {
+  const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
+  const result = await fetchWithGlobalHeader(url).then((r) => r.json());
+  return result as CoomerUserApi;
+}
+
+async function getUserPostsAPI(user: CoomerUser, offset: number): Promise<CoomerPost[]> {
+  const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
+  const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
+  return posts as CoomerPost[];
+}
+
+export async function getUserFiles(user: CoomerUser, mediaType: MediaType): Promise<File[]> {
+  const userPosts = [];
+
+  const offset = 50;
+  for (let i = 0; i < 1000; i++) {
+    const posts = await getUserPostsAPI(user, i * offset);
+    userPosts.push(...posts);
+    if (posts.length < 50) break;
+  }
+
+  const files: File[] = [];
+
+  for (const p of userPosts) {
+    const title = p.title.match(/\w+/g)?.join(' ') || '';
+    const content = p.content;
+    const date = p.published.replace(/T/, ' ');
+    const datentitle = `${date} ${title}`.trim();
+
+    const postFiles = [...p.attachments, p.file]
+      .filter((f) => f.path)
+      .filter((f) => testMediaType(f.name, mediaType))
+      .map((f, i) => {
+        const ext = f.name.split('.').pop();
+        const name = `${datentitle} ${i + 1}.${ext}`;
+        const url = `${user.domain}/${f.path}`;
+        return { name, url, content };
+      });
+
+    files.push(...postFiles);
+  }
+
+  return files;
+}
+
+async function parseUser(url: string): Promise<CoomerUser> {
+  const [_, domain, service, id] = url.match(
+    /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/,
+  ) as RegExpMatchArray;
+  if (!domain || !service || !id) console.error('Invalid URL', url);
+
+  const { name } = await getUserProfileAPI({ domain, service, id });
+
+  return { domain, service, id, name };
+}
+
+export async function getCoomerData(url: string, mediaType: MediaType): Promise<ApiResult> {
+  setGlobalHeaders({ accept: 'text/css' });
+  const user = await parseUser(url);
+  const dirName = `${user.name}-${user.service}`;
+  const files = await getUserFiles(user, mediaType);
+  return { dirName, files };
+}

package/src/api/{gofile.js → gofile.ts}

@@ -1,10 +1,15 @@
-import { fetch, testMediaType } from './../utils/index.js';
+import { fetch } from 'undici';
+import type { ApiResult, File, MediaType } from '../types/index.js';
+import { setGlobalHeaders, testMediaType } from '../utils/index.js';
 
-async function getToken() {
+type GoFileAPIToken = { status: string; data: { token: string } };
+type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };
+
+async function getToken(): Promise<string> {
   const response = await fetch('https://api.gofile.io/accounts', {
     method: 'POST',
   });
-  const data = await response.json();
+  const data = (await response.json()) as GoFileAPIToken;
   if (data.status === 'ok') {
     return data.data.token;
   }
@@ -21,7 +26,7 @@ async function getWebsiteToken() {
   throw new Error('cannot get wt');
 }
 
-async function getFolderFiles(id, token, websiteToken) {
+async function getFolderFiles(id: string, token: string, websiteToken: string): Promise<File[]> {
   const url = `https://api.gofile.io/contents/${id}?wt=${websiteToken}&cache=true}`;
   const response = await fetch(url, {
     headers: {
@@ -33,27 +38,27 @@ async function getFolderFiles(id, token, websiteToken) {
     throw new Error(`HTTP error! status: ${response.status}`);
   }
 
-  const data = await response.json();
+  const data = (await response.json()) as GoFileAPIFilelist;
   const files = Object.values(data.data.children).map((f) => ({
-    src: f.link,
+    url: f.link,
     name: f.name,
   }));
 
   return files;
 }
 
-export async function getGofileData(url, mediaType) {
-  const id = url.match(/gofile.io\/d\/(\w+)/)[1];
+export async function getGofileData(url: string, mediaType: MediaType): Promise<ApiResult> {
+  const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
   const dirName = `gofile-${id}`;
 
   const token = await getToken();
   const websiteToken = await getWebsiteToken();
 
-  const files = await getFolderFiles(id, token, websiteToken);
+  const files = (await getFolderFiles(id, token, websiteToken)).filter((f) =>
+    testMediaType(f.name, mediaType),
+  );
+
+  setGlobalHeaders({ Cookie: `accountToken=${token}` });
 
-  return {
-    dirName,
-    files: files.filter((f) => testMediaType(f.name, mediaType)),
-    headerData: { Cookie: `accountToken=${token}` },
-  };
+  return { dirName, files };
 }

package/src/api/index.ts

@@ -0,0 +1,28 @@
+import type { ApiResult, MediaType } from '../types';
+import { getBunkrData } from './bunkr';
+import { getCoomerData } from './coomer-api';
+import { getGofileData } from './gofile';
+import { getRedditData } from './nsfw.xxx';
+import { getPlainFileData } from './plain-curl';
+
+export async function apiHandler(
+  url: string,
+  mediaType: MediaType,
+): Promise<ApiResult | undefined> {
+  if (/^u\/\w+$/.test(url.trim())) {
+    return getRedditData(url, mediaType);
+  }
+  if (/coomer|kemono/.test(url)) {
+    return getCoomerData(url, mediaType);
+  }
+  if (/bunkr/.test(url)) {
+    return getBunkrData(url, mediaType);
+  }
+  if (/gofile\.io/.test(url)) {
+    return getGofileData(url, mediaType);
+  }
+  if (/\.\w+/.test(url.split('/').pop() as string)) {
+    return getPlainFileData(url);
+  }
+  console.error('Wrong URL.');
+}

package/src/api/{nsfw.xxx.js → nsfw.xxx.ts}

@@ -1,15 +1,18 @@
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
-import { testMediaType } from './../utils/index.js';
+import { fetch } from 'undici';
+import type { ApiResult, File, MediaType } from '../types/index.js';
+import { testMediaType } from '../utils/index.js';
 
-const userPostsURL = (n, user) =>
-  `https://nsfw.xxx/page/${n}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
+async function getUserPage(user: string, offset: number) {
+  const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
+  return fetch(url).then((r) => r.text());
+}
 
-async function getUserPosts(user) {
+async function getUserPosts(user: string): Promise<string[]> {
   console.log('Fetching user posts...');
   const posts = [];
   for (let i = 1; i < 100000; i++) {
-    const page = await fetch(userPostsURL(i, user)).then((r) => r.text());
+    const page = await getUserPage(user, i);
     if (page.length < 1) break;
 
     const $ = cheerio.load(page);
@@ -23,7 +26,7 @@ async function getUserPosts(user) {
   return posts;
 }
 
-async function getPostsData(posts, mediaType) {
+async function getPostsData(posts: string[], mediaType: MediaType): Promise<File[]> {
   console.log('Fetching posts data...');
   const data = [];
   for (const post of posts) {
@@ -38,21 +41,19 @@ async function getPostsData(posts, mediaType) {
     if (!src) continue;
 
     const slug = post.split('post/')[1].split('?')[0];
-    const date =
-      $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') ||
-      '';
+    const date = $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
 
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;
 
-    data.push({ name, src });
+    data.push({ name, url: src });
   }
 
   return data.filter((f) => testMediaType(f.name, mediaType));
 }
 
-export async function getRedditData(url, mediaType) {
-  const user = url.match(/u\/(\w+)/)[1];
+export async function getRedditData(url: string, mediaType: MediaType): Promise<ApiResult> {
+  const user = url.match(/u\/(\w+)/)?.[1] as string;
   const posts = await getUserPosts(user);
   const files = await getPostsData(posts, mediaType);
   const dirName = `${user}-reddit`;

package/src/api/plain-curl.ts

@@ -0,0 +1,13 @@
+import type { ApiResult } from '../types';
+
+export async function getPlainFileData(url: string): Promise<ApiResult> {
+  return {
+    dirName: '',
+    files: [
+      {
+        name: url.split('/').pop() as string,
+        url,
+      },
+    ],
+  };
+}

package/src/args-handler.ts

@@ -0,0 +1,55 @@
+import yargs from 'yargs';
+import { hideBin } from 'yargs/helpers';
+
+export type ArgumentHandlerResult = {
+  [x: string]: unknown;
+  url: string;
+  dir: string;
+  media: string;
+  include: string;
+  exclude: string;
+  skip: number;
+  _: (string | number)[];
+  $0: string;
+};
+
+export function argumentHander(): ArgumentHandlerResult {
+  return yargs(hideBin(process.argv))
+    .option('url', {
+      alias: 'u',
+      type: 'string',
+      description:
+        'A URL from Coomer/Kemono/Bunkr/GoFile, a Reddit user (u/<username>), or a direct file link',
+      demandOption: true,
+    })
+    .option('dir', {
+      type: 'string',
+      description: 'The directory where files will be downloaded',
+      default: './',
+    })
+    .option('media', {
+      type: 'string',
+      choices: ['video', 'image', 'all'],
+      default: 'all',
+      description:
+        "The type of media to download: 'video', 'image', or 'all'. 'all' is the default.",
+    })
+    .option('include', {
+      type: 'string',
+      default: '',
+      description: 'Filter file names by a comma-separated list of keywords to include',
+    })
+    .option('exclude', {
+      type: 'string',
+      default: '',
+      description: 'Filter file names by a comma-separated list of keywords to exclude',
+    })
+    .option('skip', {
+      type: 'number',
+      default: 0,
+      description: 'Skips the first N files in the download queue',
+    })
+    .help()
+    .alias('help', 'h')
+    .parseSync();
+}

package/src/index.ts

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+import { apiHandler } from './api';
+import { argumentHander } from './args-handler';
+import type { ApiResult, MediaType } from './types';
+import { createMultibar, downloadFiles, filterKeywords, setGlobalHeaders } from './utils';
+
+async function run() {
+  const { url, dir, media, include, exclude, skip } = argumentHander();
+
+  const { dirName, files } = (await apiHandler(url, media as MediaType)) as ApiResult;
+
+  const downloadDir =
+    dir === './' ? path.resolve(dir, dirName) : path.join(os.homedir(), path.join(dir, dirName));
+
+  const filteredFiles = filterKeywords(files.slice(skip), include, exclude);
+
+  console.table([
+    {
+      found: files.length,
+      skip,
+      filtered: files.length - filteredFiles.length - skip,
+      folder: downloadDir,
+    },
+  ]);
+
+  setGlobalHeaders({ Referer: url });
+
+  createMultibar();
+
+  await downloadFiles(filteredFiles, downloadDir);
+
+  process.kill(process.pid, 'SIGINT');
+}
+
+run();

package/src/types/index.ts

@@ -0,0 +1,23 @@
+export type File = {
+  name: string;
+  url: string;
+  filepath?: string;
+  size?: number;
+  downloaded?: number;
+  content?: string;
+};
+
+export type DownloaderSubject = {
+  type: string;
+  file?: File;
+  index?: number;
+  filesCount?: number;
+};
+
+export type ApiResult = {
+  files: File[];
+  // should merge into filepaths?
+  dirName: string;
+};
+
+export type MediaType = 'video' | 'image' | 'all';

package/src/utils/downloader.ts

@@ -0,0 +1,102 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { Subject } from 'rxjs';
+import { tryFixCoomerUrl } from '../api/coomer-api';
+import type { DownloaderSubject, File } from '../types';
+import { getFileSize, mkdir } from './files';
+import { PromiseRetry } from './promise';
+import { fetchByteRange } from './requests';
+import { Timer } from './timer';
+
+export const subject = new Subject<DownloaderSubject>();
+
+const CHUNK_TIMEOUT = 30_000;
+const CHUNK_FETCH_RETRIES = 5;
+const FETCH_RETRIES = 7;
+
+async function fetchStream(file: File, stream: Readable): Promise<void> {
+  const { timer, signal } = Timer.withSignal(CHUNK_TIMEOUT, 'CHUNK_TIMEOUT');
+
+  const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
+
+  const progressStream = new Transform({
+    transform(chunk, _encoding, callback) {
+      this.push(chunk);
+      file.downloaded += chunk.length;
+      timer.reset();
+      subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE', file });
+      callback();
+    },
+  });
+
+  try {
+    subject.next({ type: 'CHUNK_DOWNLOADING_START', file });
+    await pipeline(stream, progressStream, fileStream, { signal });
+  } catch (error) {
+    console.error((error as Error).name === 'AbortError' ? signal.reason : error);
+  } finally {
+    subject.next({ type: 'CHUNK_DOWNLOADING_END', file });
+  }
+}
+
+async function downloadFile(file: File): Promise<void> {
+  file.downloaded = await getFileSize(file.filepath as string);
+
+  const response = await fetchByteRange(file.url, file.downloaded);
+
+  if (!response?.ok && response?.status !== 416) {
+    throw new Error(`HTTP error! status: ${response?.status}`);
+  }
+
+  const contentLength = response.headers.get('Content-Length') as string;
+
+  if (!contentLength && file.downloaded > 0) {
+    return;
+  }
+
+  const restFileSize = parseInt(contentLength);
+  file.size = restFileSize + file.downloaded;
+
+  if (file.size > file.downloaded && response.body) {
+    const stream = Readable.fromWeb(response.body);
+    const sizeOld = file.downloaded;
+
+    await PromiseRetry.create({
+      retries: CHUNK_FETCH_RETRIES,
+      callback: () => {
+        if (sizeOld !== file.downloaded) {
+          return { newRetries: 5 };
+        }
+      },
+    }).execute(async () => await fetchStream(file, stream));
+  }
+
+  subject.next({ type: 'FILE_DOWNLOADING_END' });
+}
+
+export async function downloadFiles(data: File[], downloadDir: string): Promise<void> {
+  mkdir(downloadDir);
+
+  subject.next({ type: 'FILES_DOWNLOADING_START', filesCount: data.length });
+
+  for (const [_, file] of data.entries()) {
+    file.filepath = path.join(downloadDir, file.name);
+
+    subject.next({ type: 'FILE_DOWNLOADING_START' });
+
+    await PromiseRetry.create({
+      retries: FETCH_RETRIES,
+      callback: (retries) => {
+        if (/coomer|kemono/.test(file.url)) {
+          file.url = tryFixCoomerUrl(file.url, retries);
+        }
+      },
+    }).execute(async () => await downloadFile(file));
+
+    subject.next({ type: 'FILE_DOWNLOADING_END' });
+  }
+
+  subject.next({ type: 'FILES_DOWNLOADING_END' });
+}

package/src/utils/files.ts

@@ -0,0 +1,15 @@
+import fs from 'node:fs';
+
+export async function getFileSize(filepath: string) {
+  let size = 0;
+  if (fs.existsSync(filepath)) {
+    size = (await fs.promises.stat(filepath)).size || 0;
+  }
+  return size;
+}
+
+export function mkdir(filepath: string) {
+  if (!fs.existsSync(filepath)) {
+    fs.mkdirSync(filepath, { recursive: true });
+  }
+}

package/src/utils/filters.ts

@@ -0,0 +1,37 @@
+import type { File, MediaType } from '../types';
+
+export const isImage = (name: string) => /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+
+export const isVideo = (name: string) =>
+  /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+
+export const testMediaType = (name: string, type: MediaType) =>
+  type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
+
+function includesAllWords(str: string, words: string[]) {
+  if (!words.length) return true;
+  return words.every((w) => str.includes(w));
+}
+
+function includesNoWords(str: string, words: string[]) {
+  if (!words.length) return true;
+  return words.every((w) => !str.includes(w));
+}
+
+function parseQuery(query: string) {
+  return query
+    .split(',')
+    .map((x) => x.toLowerCase().trim())
+    .filter((_) => _);
+}
+
+function filterString(text: string, include: string, exclude: string): boolean {
+  return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
+}
+
+export function filterKeywords(files: File[], include: string, exclude: string) {
+  return files.filter((f) => {
+    const text = `${f.name || ''} ${f.content || ''}`.toLowerCase();
+    return filterString(text, include, exclude);
+  });
+}