coomer-downloader 2.6.5 → 3.1.0

package/src/utils/index.ts

@@ -0,0 +1,11 @@
+export { downloadFiles } from './downloader';
+export { getFileSize, mkdir } from './files';
+export { filterKeywords, isImage, isVideo, testMediaType } from './filters';
+export { createMultibar } from './multibar';
+export {
+  fetchByteRange,
+  fetchWithGlobalHeader,
+  HeadersDefault,
+  setGlobalHeaders,
+} from './requests';
+export { b2mb } from './strings';

package/src/utils/multibar.ts

@@ -0,0 +1,62 @@
+import { MultiBar, type Options, type SingleBar } from 'cli-progress';
+import { subject } from './downloader';
+import { b2mb, formatNameStdout } from './strings';
+
+const config: Options = {
+  clearOnComplete: true,
+  gracefulExit: true,
+  autopadding: true,
+  hideCursor: true,
+  format: '{percentage}% | {filename} | {value}/{total}{size}',
+};
+
+export function createMultibar() {
+  const multibar = new MultiBar(config);
+  let bar: SingleBar;
+  let minibar: SingleBar;
+  let filename: string;
+  let index = 0;
+
+  subject.subscribe({
+    next: ({ type, filesCount, file }) => {
+      switch (type) {
+        case 'FILES_DOWNLOADING_START':
+          bar?.stop();
+          bar = multibar.create(filesCount as number, 0);
+          break;
+
+        case 'FILES_DOWNLOADING_END':
+          bar?.stop();
+          break;
+
+        case 'FILE_DOWNLOADING_START':
+          bar?.update(++index, { filename: 'Downloaded files', size: '' });
+          break;
+
+        case 'FILE_DOWNLOADING_END':
+          multibar.remove(minibar);
+          break;
+
+        case 'CHUNK_DOWNLOADING_START':
+          multibar?.remove(minibar);
+          filename = formatNameStdout(file?.filepath as string);
+          minibar = multibar.create(b2mb(file?.size as number), b2mb(file?.downloaded as number));
+          break;
+
+        case 'CHUNK_DOWNLOADING_UPDATE':
+          minibar?.update(b2mb(file?.downloaded as number), {
+            filename: filename as string,
+            size: 'mb',
+          });
+          break;
+
+        case 'CHUNK_DOWNLOADING_END':
+          multibar?.remove(minibar);
+          break;
+
+        default:
+          break;
+      }
+    },
+  });
+}

package/src/utils/promise.ts

@@ -0,0 +1,53 @@
+export async function sleep(time: number) {
+  return new Promise((resolve) => setTimeout(resolve, time));
+}
+
+type PromiseRetryCallback = (retries: number, error: Error) => void | { newRetries?: number };
+
+interface PromiseRetryOptions {
+  retries?: number;
+  callback?: PromiseRetryCallback;
+  delay?: number;
+}
+
+export class PromiseRetry {
+  private retries: number;
+  private delay: number;
+  private callback?: PromiseRetryCallback;
+
+  constructor(options: PromiseRetryOptions) {
+    this.retries = options.retries || 3;
+    this.delay = options.delay || 1000;
+    this.callback = options.callback;
+  }
+
+  async execute(fn: () => Promise<void>) {
+    let retries = this.retries;
+
+    while (true) {
+      try {
+        return await fn();
+      } catch (error) {
+        if (retries <= 0) {
+          throw error;
+        }
+
+        if (this.callback) {
+          const res = this.callback(retries, error as Error);
+          if (res) {
+            const { newRetries } = res;
+            if (newRetries === 0) throw error;
+            this.retries = newRetries || retries;
+          }
+        }
+
+        await sleep(this.delay);
+        retries--;
+      }
+    }
+  }
+
+  static create(options: PromiseRetryOptions) {
+    return new PromiseRetry(options);
+  }
+}

package/src/utils/requests.ts

@@ -0,0 +1,36 @@
+import { CookieAgent } from 'http-cookie-agent/undici';
+import { CookieJar } from 'tough-cookie';
+import { fetch, interceptors, setGlobalDispatcher } from 'undici';
+
+function setCookieJarDispatcher() {
+  const jar = new CookieJar();
+  const agent = new CookieAgent({ cookies: { jar } })
+    .compose(interceptors.retry())
+    .compose(interceptors.redirect({ maxRedirections: 3 }));
+  setGlobalDispatcher(agent);
+}
+
+setCookieJarDispatcher();
+
+export const HeadersDefault = new Headers({
+  accept: 'application/json, text/css',
+  'User-Agent':
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+});
+
+export function setGlobalHeaders(headers: Record<string, string>) {
+  Object.keys(headers).forEach((k) => {
+    HeadersDefault.set(k, headers[k]);
+  });
+}
+
+export function fetchWithGlobalHeader(url: string) {
+  const requestHeaders = new Headers(HeadersDefault);
+  return fetch(url, { headers: requestHeaders });
+}
+
+export function fetchByteRange(url: string, downloadedSize: number) {
+  const requestHeaders = new Headers(HeadersDefault);
+  requestHeaders.set('Range', `bytes=${downloadedSize}-`);
+  return fetch(url, { headers: requestHeaders });
+}

package/src/utils/strings.ts

@@ -0,0 +1,21 @@
+export function b2mb(bytes: number) {
+  return Number.parseFloat((bytes / 1048576).toFixed(2));
+}
+
+export function sanitizeString(str: string) {
+  return (
+    str
+      .match(/(\w| |-)/g)
+      ?.join('')
+      .replace(/ +/g, ' ') || ''
+  );
+}
+
+export function formatNameStdout(pathname: string) {
+  const name = pathname.split('/').pop() || '';
+  const consoleWidth = process.stdout.columns;
+  const width = Math.max((consoleWidth / 2) | 0, 40);
+  if (name.length < width) return name.trim();
+  const result = `${name.slice(0, width - 15)} ... ${name.slice(-10)}`.replace(/ +/g, ' ');
+  return result;
+}

package/src/utils/timer.ts

@@ -0,0 +1,47 @@
+export class Timer {
+  private timer: NodeJS.Timeout | undefined = undefined;
+
+  constructor(
+    private timeout = 10_000,
+    private timeoutCallback: () => void,
+  ) {
+    this.timeout = timeout;
+  }
+
+  start() {
+    this.timer = setTimeout(() => {
+      this.stop();
+      this.timeoutCallback();
+    }, this.timeout);
+    return this;
+  }
+
+  stop() {
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = undefined;
+    }
+    return this;
+  }
+
+  reset() {
+    this.stop();
+    this.start();
+    return this;
+  }
+
+  static withSignal(timeout?: number, message?: string) {
+    const controller = new AbortController();
+
+    const callback = () => {
+      controller.abort(message);
+    };
+
+    const timer = new Timer(timeout, callback).start();
+
+    return {
+      timer,
+      signal: controller.signal,
+    };
+  }
+}

package/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

package/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "module": "esnext",
+    "lib": ["ESNext", "DOM", "DOM.Iterable"],
+    "skipLibCheck": true,
+
+    "moduleResolution": "bundler",
+    "isolatedModules": true,
+    "moduleDetection": "force",
+    
+    "declaration": true,
+    "typeRoots": ["./dist/index.d.ts", "./src/types", "./node_modules/@types"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src", "temp"]
+}

package/index.js

@@ -1,34 +0,0 @@
-#!/usr/bin/env node
-import os from 'node:os';
-import path from 'node:path';
-import process from 'node:process';
-import { downloadFiles } from './src/downloader.js';
-import { apiHandler } from './src/api/index.js';
-import { argumentHander } from './src/args-handler.js';
-import { filterKeywords } from './src/utils/index.js';
-
-async function run() {
-  const { url, dir, media, include, exclude, skip = 0 } = argumentHander();
-
-  const { dirName, files, headerData = {} } = await apiHandler(url, media);
-
-  console.log(' ', files.length, 'files found');
-
-  const downloadDir =
-    dir === './'
-      ? path.resolve(dir, dirName)
-      : path.join(os.homedir(), path.join(dir, dirName));
-
-  const filteredFiles = filterKeywords(files, include, exclude).slice(skip);
-
-  if (skip) console.log(skip, 'files skipped');
-
-  await downloadFiles(filteredFiles, downloadDir, {
-    Referer: url,
-    ...headerData,
-  });
-
-  process.kill(process.pid, 'SIGINT');
-}
-
-run();

package/src/api/coomer-api.js

@@ -1,78 +0,0 @@
-import { fetch_, isImage, testMediaType } from '../utils/index.js';
-
-const SERVERS = ['n1', 'n2', 'n3', 'n4'];
-
-export function tryFixCoomerUrl(url, attempts) {
-  if (attempts < 2 && isImage(url)) {
-    return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
-  }
-  const server = url.match(/n\d\./)[0].slice(0, 2);
-  const i = SERVERS.indexOf(server);
-  if (i !== -1) {
-    const newServer = SERVERS[(i + 1) % SERVERS.length];
-    return url.replace(/n\d./, `${newServer}.`);
-  }
-  return url;
-}
-
-const userProfileAPI = ({ domain, service, id }) =>
-  `${domain}/api/v1/${service}/user/${id}/profile`;
-
-const userPostsAPI = (user, offset) =>
-  `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
-
-export async function getUserFiles(user, typeFilter) {
-  const userPosts = [];
-
-  const offset = 50;
-  for (let i = 0; i < 1000; i++) {
-    const posts = await fetch_(userPostsAPI(user, offset * i)).then((r) =>
-      r.json(),
-    );
-    userPosts.push(...posts);
-    if (posts.length < 50) break;
-  }
-
-  const files = [];
-
-  for (const p of userPosts) {
-    const title = p.title.match(/\w+/g)?.join(' ') || '';
-    const content = p.content;
-    const date = p.published.replace(/T/, ' ');
-    const datentitle = `${date} ${title}`.trim();
-
-    const postFiles = [...p.attachments, p.file]
-      .filter((f) => f.path)
-      .filter((f) => testMediaType(f.name, typeFilter))
-      .map((f, i) => {
-        const ext = f.name.split('.').pop();
-        const name = `${datentitle} ${i + 1}.${ext}`;
-        const src = `${user.domain}/${f.path}`;
-        return { name, src, content };
-      });
-
-    files.push(...postFiles);
-  }
-
-  return files;
-}
-
-async function parseUser(url) {
-  const [_, domain, service, id] = url.match(
-    /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|\.|-]+)/,
-  );
-  if (!domain || !service || !id) console.error('Invalid URL', url);
-
-  const { name } = await fetch_(userProfileAPI({ domain, service, id })).then(
-    (r) => r.json(),
-  );
-
-  return { domain, service, id, name };
-}
-
-export async function getCoomerData(url, mediaType) {
-  const user = await parseUser(url);
-  const dirName = `${user.name}-${user.service}`;
-  const files = await getUserFiles(user, mediaType);
-  return { dirName, files };
-}

package/src/api/index.js

@@ -1,24 +0,0 @@
-import { getBunkrData } from './bunkr.js';
-import { getCoomerData } from './coomer-api.js';
-import { getGofileData } from './gofile.js';
-import { getRedditData } from './nsfw.xxx.js';
-import { getPlainFileData } from './plain-curl.js';
-
-export async function apiHandler(url, mediaType) {
-  if (/^u\/\w+$/.test(url.trim())) {
-    return getRedditData(url, mediaType);
-  }
-  if (/coomer|kemono/.test(url)) {
-    return getCoomerData(url, mediaType);
-  }
-  if (/bunkr/.test(url)) {
-    return getBunkrData(url, mediaType);
-  }
-  if (/gofile\.io/.test(url)) {
-    return getGofileData(url, mediaType);
-  }
-  if (/\.\w+/.test(url.split('/').pop())) {
-    return getPlainFileData(url);
-  }
-  console.error('Wrong URL.');
-}

package/src/api/plain-curl.js

@@ -1,11 +0,0 @@
-export async function getPlainFileData(url) {
-  return {
-    dirName: '',
-    files: [
-      {
-        name: url.split('/').pop(),
-        src: url,
-      },
-    ],
-  };
-}

package/src/args-handler.js

@@ -1,42 +0,0 @@
-import yargs from 'yargs';
-import { hideBin } from 'yargs/helpers';
-
-export function argumentHander() {
-  return yargs(hideBin(process.argv))
-    .option('url', {
-      alias: 'u',
-      type: 'string',
-      description:
-        'Coomer/Kemono/Bunkr/GoFile, u/<reddit-username> or plain file url',
-      demandOption: true,
-    })
-    .option('dir', {
-      type: 'string',
-      description: 'Directory to download files to',
-      default: './',
-    })
-    .option('media', {
-      type: 'string',
-      choices: ['video', 'image', 'all'],
-      default: 'all',
-      description:
-        "Download media type: 'video', 'image', or 'all', 'all' is default",
-    })
-    .option('include', {
-      type: 'string',
-      default: '',
-      description: 'filter files with names which includes keywords',
-    })
-    .option('exclude', {
-      type: 'string',
-      default: '',
-      description: 'filter files with names which excludes keywords',
-    })
-    .option('skip', {
-      type: 'string',
-      default: '',
-      description: 'skips N files in download queue',
-    })
-    .help()
-    .alias('help', 'h').argv;
-}

package/src/downloader.js

@@ -1,91 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import { Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { b2mb, fetchByteRange, DEFAULT_HEADERS, getFileSize } from './utils/index.js';
-import { MultiBar } from 'cli-progress';
-import { tryFixCoomerUrl } from './api/coomer-api.js';
-
-const multibar = new MultiBar({
-  clearOnComplete: true,
-  gracefulExit: true,
-  autopadding: true,
-  hideCursor: true,
-  format: '{percentage}% | {filename} | {value}/{total}{size}',
-});
-
-async function downloadFile(url, outputFile, attempts = 7) {
-  let response;
-
-  try {
-    let existingFileSize = await getFileSize(outputFile);
-
-    response = await fetchByteRange(url, existingFileSize);
-
-    if (!response.ok && response.status !== 416) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-
-    const contentLength = parseInt(response.headers.get('Content-Length'));
-
-    if (!response.headers.get('Content-Range') && existingFileSize > 0) {
-      return;
-    }
-
-    const fileStream = fs.createWriteStream(outputFile, { flags: 'a' });
-
-    const restFileSize = contentLength;
-    const totalFileSize = restFileSize + existingFileSize;
-
-    if (totalFileSize > existingFileSize) {
-      const bar = multibar.create(b2mb(totalFileSize), b2mb(existingFileSize));
-      const filename = outputFile.slice(-40);
-
-      const progressStream = new Transform({
-        transform(chunk, _encoding, callback) {
-          this.push(chunk);
-          existingFileSize += chunk.length;
-          bar.update(b2mb(existingFileSize), { filename, size: 'mb' });
-          callback();
-        },
-      });
-
-      await pipeline(response.body, progressStream, fileStream);
-      multibar.remove(bar);
-    }
-  } catch (error) {
-    if (attempts < 1) {
-      console.error(url);
-      console.error(error);
-    } else {
-      let newUrl = url;
-      if (/coomer|kemono/.test(response.url)) {
-        newUrl = tryFixCoomerUrl(response.url, attempts);
-      }
-      await downloadFile(newUrl, outputFile, attempts - 1);
-    }
-  }
-}
-
-export async function downloadFiles(data, downloadDir, headers) {
-  if (!fs.existsSync(downloadDir)) {
-    fs.mkdirSync(downloadDir, { recursive: true });
-  }
-
-  Object.keys(headers).forEach((k) => DEFAULT_HEADERS.set(k, headers[k]));
-
-  const bar = multibar.create(data.length, 0);
-
-  for (const [index, { name, src }] of data.entries()) {
-    const filePath = path.join(downloadDir, name);
-    try {
-      bar.update(index + 1, { filename: 'Downloaded files', size: '' });
-      await downloadFile(src.replace(/[\/]+/g, '/'), filePath);
-    } catch (error) {
-      console.error(`\nError downloading ${name}:`, error.message);
-      console.error(src);
-    }
-  }
-
-  bar.stop();
-}

package/src/utils/index.js

@@ -1,62 +0,0 @@
-import fs from 'node:fs';
-import nodeFetch from 'node-fetch';
-import makeFetchCookie from 'fetch-cookie';
-export { TransformWithTimeout } from './streams.js';
-
-export const fetch = makeFetchCookie(nodeFetch);
-
-export const DEFAULT_HEADERS = new Headers({
-  'accept': 'application/json', 
-  'accept': 'text/css',
-  'User-Agent':
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-});
-
-export function fetch_(url) {
-  const requestHeaders = new Headers(DEFAULT_HEADERS);
-  const headers = Object.fromEntries(requestHeaders.entries());
-  return fetch(url, { headers });
-}
-
-export function fetchByteRange(url, existingFileSize) {
-  const requestHeaders = new Headers(DEFAULT_HEADERS);
-
-  requestHeaders.set('Range', `bytes=${existingFileSize}-`);
-
-  const headers = Object.fromEntries(requestHeaders.entries());
-
-  return fetch(url, { headers });
-}
-
-export const isImage = (name) =>
-  /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
-
-export const isVideo = (name) =>
-  /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
-
-export const testMediaType = (name, type) =>
-  type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
-
-export const b2mb = (b) => Number.parseFloat((b / 1048576).toFixed(2));
-
-export function filterKeywords(files, include, exclude) {
-  const incl = include.split(',').map((x) => x.toLowerCase().trim());
-  const excl = exclude.split(',').map((x) => x.toLowerCase().trim());
-
-  const isValid = text => incl.some(e => text.includes(e)) &&
-    (!exclude.trim().length || excl.every((e) => !text.includes(e)));
-
-  return files
-    .filter(f => {
-      const text = `${f.name || ""} ${f.content || ""}`.toLowerCase();
-      return isValid(text);
-    });
-}
-
-export async function getFileSize(file) {
-  let size = 0;
-  if (fs.existsSync(file)) {
-    size = (await fs.promises.stat(file)).size || 0;
-  }
-  return size;
-}

package/src/utils/streams.js

@@ -1,40 +0,0 @@
-import { Transform } from 'node:stream';
-import { setInterval, clearInterval } from 'node:timers';
-
-export class TransformWithTimeout extends Transform {
-  lastChunkReceived;
-  timeoutMilliseconds;
-  timeoutInterval;
-
-  constructor(timeoutSeconds, options) {
-    super(options);
-    if (timeoutSeconds === undefined) {
-      throw new Error(
-        'new Timeout(timeoutSeconds): timeoutSeconds is a required parameter.',
-      );
-    }
-    this.timeoutMilliseconds = timeoutSeconds * 1000;
-    this.timeoutInterval = setInterval(
-      () => this._check(),
-      this.timeoutMilliseconds,
-    );
-  }
-
-  _check() {
-    const millisecondsSinceLast = new Date().getTime() - this.lastChunkReceived;
-    if (millisecondsSinceLast > this.timeoutMilliseconds) {
-      this.emit('error', new Error(`Timed out: ${millisecondsSinceLast}ms`));
-    }
-  }
-
-  _transform(chunk, encoding, callback) {
-    this.lastChunkReceived = new Date().getTime();
-    this.push(chunk);
-    callback();
-  }
-
-  _flush(callback) {
-    clearInterval(this.timeoutInterval);
-    callback();
-  }
-}