coomer-downloader 3.2.0 → 3.4.0

package/src/index.ts

@@ -1,36 +1,55 @@
-#!/usr/bin/env node
+#!/usr/bin/env -S node --no-warnings=ExperimentalWarning
+
 import process from 'node:process';
 import { apiHandler } from './api';
-import { argumentHander } from './args-handler';
-import { createMultibar, Downloader, setGlobalHeaders } from './utils';
+import { argumentHander } from './cli/args-handler';
+import { createReactInk } from './cli/ui';
+import { useInkStore } from './cli/ui/store';
+import { Downloader } from './services/downloader';
+import { parseSizeValue } from './utils/filters';
+import { setGlobalHeaders } from './utils/requests';
 
 async function run() {
-  const { url, dir, media, include, exclude, skip } = argumentHander();
+  createReactInk();
+
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
+    argumentHander();
 
   const filelist = await apiHandler(url);
 
-  const found = filelist.files.length;
-  filelist.setDirPath(dir);
-  filelist.skip(skip);
-  filelist.filterByText(include, exclude);
-  filelist.filterByMediaType(media);
-
-  console.table([
-    {
-      found,
-      skip,
-      filtered: found - filelist.files.length,
-      folder: filelist.dirPath,
-    },
-  ]);
+  filelist
+    .setDirPath(dir)
+    .skip(skip)
+    .filterByText(include, exclude)
+    .filterByMediaType(media);
+
+  if (removeDupilicates) {
+    filelist.removeURLDuplicates();
+  }
+
+  const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
+  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
+
+  await filelist.calculateFileSizes();
 
   setGlobalHeaders({ Referer: url });
 
-  const downloader = new Downloader();
-  createMultibar(downloader);
-  await downloader.downloadFiles(filelist);
+  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
+  useInkStore.getState().setDownloader(downloader);
+
+  await downloader.downloadFiles();
 
-  process.kill(process.pid, 'SIGINT');
+  if (removeDupilicates) {
+    await filelist.removeDuplicatesByHash();
+  }
 }
 
-run();
+(async () => {
+  try {
+    await run();
+    process.exit(0);
+  } catch (err) {
+    console.error('Fatal error:', err);
+    process.exit(1);
+  }
+})();

package/src/logger/index.ts

@@ -0,0 +1,15 @@
+import pino from 'pino';
+
+// tail -f debug.log | jq -C .
+const logger = pino(
+  {
+    level: 'debug',
+  },
+  pino.destination({
+    dest: './debug.log',
+    append: false,
+    sync: true,
+  }),
+);
+
+export default logger;

package/src/services/downloader.ts

@@ -0,0 +1,161 @@
+import fs from 'node:fs';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { Subject } from 'rxjs';
+import { tryFixCoomerUrl } from '../api/coomer-api';
+import type { AbortControllerSubject, DownloaderSubject } from '../types';
+import { deleteFile, getFileSize, mkdir } from '../utils/io';
+import { sleep } from '../utils/promise';
+import { fetchByteRange } from '../utils/requests';
+import { Timer } from '../utils/timer';
+import type { CoomerFile, CoomerFileList } from './file';
+
+export class Downloader {
+  public subject = new Subject<DownloaderSubject>();
+
+  private abortController = new AbortController();
+  public abortControllerSubject = new Subject<AbortControllerSubject>();
+
+  setAbortControllerListener() {
+    this.abortControllerSubject.subscribe((type) => {
+      this.abortController.abort(type);
+      this.abortController = new AbortController();
+    });
+  }
+
+  constructor(
+    public filelist: CoomerFileList,
+    public minSize?: number,
+    public maxSize?: number,
+    public chunkTimeout = 30_000,
+    public chunkFetchRetries = 5,
+    public fetchRetries = 7,
+  ) {
+    this.setAbortControllerListener();
+  }
+
+  async fetchStream(
+    file: CoomerFile,
+    stream: Readable,
+    sizeOld = 0,
+    retries = this.chunkFetchRetries,
+  ): Promise<void> {
+    const signal = this.abortController.signal;
+    const subject = this.subject;
+    const { timer } = Timer.withAbortController(
+      this.chunkTimeout,
+      this.abortControllerSubject,
+    );
+
+    try {
+      const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
+
+      const progressStream = new Transform({
+        transform(chunk, _encoding, callback) {
+          this.push(chunk);
+          file.downloaded += chunk.length;
+          timer.reset();
+          subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE' });
+          callback();
+        },
+      });
+
+      subject.next({ type: 'CHUNK_DOWNLOADING_START' });
+      await pipeline(stream, progressStream, fileStream, { signal });
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+        if (signal.reason === 'TIMEOUT') {
+          if (retries === 0 && sizeOld < file.downloaded) {
+            retries += this.chunkFetchRetries;
+            sizeOld = file.downloaded;
+          }
+          if (retries === 0) return;
+          return await this.fetchStream(file, stream, sizeOld, retries - 1);
+        }
+      }
+      throw error;
+    } finally {
+      subject.next({ type: 'CHUNK_DOWNLOADING_END' });
+      timer.stop();
+    }
+  }
+
+  public skip() {
+    this.abortControllerSubject.next('FILE_SKIP');
+  }
+
+  private filterFileSize(file: CoomerFile) {
+    if (!file.size) return;
+    if (
+      (this.minSize && file.size < this.minSize) ||
+      (this.maxSize && file.size > this.maxSize)
+    ) {
+      try {
+        deleteFile(file.filepath);
+      } catch {}
+      this.skip();
+      return;
+    }
+  }
+
+  async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
+    const signal = this.abortController.signal;
+    try {
+      file.downloaded = await getFileSize(file.filepath as string);
+
+      const response = await fetchByteRange(file.url, file.downloaded, signal);
+
+      if (!response?.ok && response?.status !== 416) {
+        throw new Error(`HTTP error! status: ${response?.status}`);
+      }
+
+      const contentLength = response.headers.get('Content-Length') as string;
+
+      if (!contentLength && file.downloaded > 0) return;
+
+      const restFileSize = parseInt(contentLength);
+      file.size = restFileSize + file.downloaded;
+
+      this.filterFileSize(file);
+
+      if (file.size > file.downloaded && response.body) {
+        const stream = Readable.fromWeb(response.body);
+        stream.setMaxListeners(20);
+        await this.fetchStream(file, stream, file.downloaded);
+      }
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+      }
+      if (retries > 0) {
+        if (/coomer|kemono/.test(file.url)) {
+          file.url = tryFixCoomerUrl(file.url, retries);
+        }
+        await sleep(1000);
+        return await this.downloadFile(file, retries - 1);
+      }
+      throw error;
+    }
+  }
+
+  async downloadFiles(): Promise<void> {
+    mkdir(this.filelist.dirPath as string);
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_START' });
+
+    for (const file of this.filelist.files) {
+      file.active = true;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_START' });
+
+      await this.downloadFile(file);
+
+      file.active = false;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_END' });
+    }
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_END' });
+  }
+}

package/src/services/file.ts

@@ -0,0 +1,113 @@
+import os from 'node:os';
+import path from 'node:path';
+import logger from '../logger';
+import type { MediaType } from '../types';
+import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
+import { filterString } from '../utils/filters';
+import { deleteFile, getFileHash, getFileSize, sanitizeFilename } from '../utils/io';
+import { testMediaType } from '../utils/mediatypes';
+
+export class CoomerFile {
+  public active = false;
+  public hash?: string;
+
+  constructor(
+    public name: string,
+    public url: string,
+    public filepath = '',
+    public size?: number,
+    public downloaded = 0,
+    public content?: string,
+  ) {}
+
+  public async getDownloadedSize() {
+    this.downloaded = await getFileSize(this.filepath as string);
+    return this;
+  }
+
+  public get textContent() {
+    const text = `${this.name || ''} ${this.content || ''}`.toLowerCase();
+    return text;
+  }
+
+  public static from(f: Pick<CoomerFile, 'name' | 'url'> & Partial<CoomerFile>) {
+    return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
+  }
+}
+
+export class CoomerFileList {
+  public dirPath?: string;
+  public dirName?: string;
+
+  constructor(public files: CoomerFile[] = []) {}
+
+  public setDirPath(dir: string, dirName?: string) {
+    dirName = dirName || (this.dirName as string);
+
+    if (dir === './') {
+      this.dirPath = path.resolve(dir, dirName);
+    } else {
+      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
+    }
+
+    this.files.forEach((file) => {
+      const safeName = sanitizeFilename(file.name) || file.name;
+      file.filepath = path.join(this.dirPath as string, safeName);
+    });
+
+    return this;
+  }
+
+  public filterByText(include: string, exclude: string) {
+    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
+    return this;
+  }
+
+  public filterByMediaType(media?: string) {
+    if (media) {
+      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
+    }
+    return this;
+  }
+
+  public skip(n: number) {
+    this.files = this.files.slice(n);
+    return this;
+  }
+
+  public async calculateFileSizes() {
+    for (const file of this.files) {
+      await file.getDownloadedSize();
+    }
+    return this;
+  }
+
+  public getActiveFiles() {
+    return this.files.filter((f) => f.active);
+  }
+
+  public getDownloaded() {
+    return this.files.filter((f) => f.size && f.size <= f.downloaded);
+  }
+
+  public async removeDuplicatesByHash() {
+    for (const file of this.files) {
+      file.hash = await getFileHash(file.filepath);
+    }
+
+    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
+
+    console.log({ duplicates });
+
+    logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+
+    duplicates.forEach((f) => {
+      deleteFile(f.filepath);
+    });
+  }
+
+  public removeURLDuplicates() {
+    this.files = removeDuplicatesBy(this.files, 'url');
+    return this;
+  }
+}

package/src/types/index.ts

@@ -1 +1,16 @@
-export type MediaType = 'video' | 'image' | 'all';
+export type MediaType = 'video' | 'image';
+
+export type DownloaderSubjectSignal =
+  | 'FILES_DOWNLOADING_START'
+  | 'FILES_DOWNLOADING_END'
+  | 'FILE_DOWNLOADING_START'
+  | 'FILE_DOWNLOADING_END'
+  | 'CHUNK_DOWNLOADING_START'
+  | 'CHUNK_DOWNLOADING_UPDATE'
+  | 'CHUNK_DOWNLOADING_END';
+
+export type AbortControllerSubject = 'FILE_SKIP' | 'TIMEOUT';
+
+export type DownloaderSubject = {
+  type: DownloaderSubjectSignal;
+};

package/src/utils/duplicates.ts

@@ -0,0 +1,23 @@
+export function collectUniquesAndDuplicatesBy<T extends {}, K extends keyof T>(
+  xs: T[],
+  k: K,
+): { uniques: T[]; duplicates: T[] } {
+  const seen = new Set<T[K]>();
+
+  return xs.reduce(
+    (acc, item) => {
+      if (seen.has(item[k])) {
+        acc.duplicates.push(item);
+      } else {
+        seen.add(item[k]);
+        acc.uniques.push(item);
+      }
+      return acc;
+    },
+    { uniques: [] as T[], duplicates: [] as T[] },
+  );
+}
+
+export function removeDuplicatesBy<T extends {}, K extends keyof T>(xs: T[], k: K) {
+  return [...new Map(xs.map((x) => [x[k], x])).values()];
+}

package/src/utils/filters.ts

@@ -1,17 +1,3 @@
-import type { MediaType } from '../types';
-
-export function isImage(name: string) {
-  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
-}
-
-export function isVideo(name: string) {
-  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
-}
-
-export function testMediaType(name: string, type: MediaType) {
-  return type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
-}
-
 function includesAllWords(str: string, words: string[]) {
   if (!words.length) return true;
   return words.every((w) => str.includes(w));
@@ -30,5 +16,19 @@ function parseQuery(query: string) {
 }
 
 export function filterString(text: string, include: string, exclude: string): boolean {
-  return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
+  return (
+    includesAllWords(text, parseQuery(include)) &&
+    includesNoWords(text, parseQuery(exclude))
+  );
+}
+
+export function parseSizeValue(s: string) {
+  if (!s) return NaN;
+  const m = s.match(/^([0-9]+(?:\.[0-9]+)?)(b|kb|mb|gb)?$/i);
+  if (!m) return NaN;
+  const val = parseFloat(m[1]);
+  const unit = (m[2] || 'b').toLowerCase();
+  const mult =
+    unit === 'kb' ? 1024 : unit === 'mb' ? 1024 ** 2 : unit === 'gb' ? 1024 ** 3 : 1;
+  return Math.floor(val * mult);
 }

package/src/utils/io.ts

@@ -1,4 +1,7 @@
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
+import { access, constants, unlink } from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
 
 export async function getFileSize(filepath: string) {
   let size = 0;
@@ -8,8 +11,30 @@ export async function getFileSize(filepath: string) {
   return size;
 }
 
+export async function getFileHash(filepath: string) {
+  const hash = createHash('sha256');
+  const filestream = fs.createReadStream(filepath);
+  await pipeline(filestream, hash);
+  return hash.digest('hex');
+}
+
 export function mkdir(filepath: string) {
   if (!fs.existsSync(filepath)) {
     fs.mkdirSync(filepath, { recursive: true });
   }
 }
+
+export async function deleteFile(path: string) {
+  await access(path, constants.F_OK);
+  await unlink(path);
+}
+
+export function sanitizeFilename(name: string) {
+  if (!name) return name;
+
+  return name
+    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '-') // Newlines (\r \n) are caught here
+    .replace(/\s+/g, ' ') // Turn tabs/multiple spaces into one space
+    .trim()
+    .replace(/[.]+$/, ''); // Remove trailing dots
+}

package/src/utils/mediatypes.ts

@@ -0,0 +1,13 @@
+import type { MediaType } from '../types';
+
+export function isImage(name: string) {
+  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+}
+
+export function isVideo(name: string) {
+  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+}
+
+export function testMediaType(name: string, type: MediaType) {
+  return type === 'image' ? isImage(name) : isVideo(name);
+}

package/src/utils/promise.ts

@@ -1,53 +1,3 @@
 export async function sleep(time: number) {
   return new Promise((resolve) => setTimeout(resolve, time));
 }
-
-type PromiseRetryCallback = (retries: number, error: Error) => void | { newRetries?: number };
-
-interface PromiseRetryOptions {
-  retries?: number;
-  callback?: PromiseRetryCallback;
-  delay?: number;
-}
-
-export class PromiseRetry {
-  private retries: number;
-  private delay: number;
-  private callback?: PromiseRetryCallback;
-
-  constructor(options: PromiseRetryOptions) {
-    this.retries = options.retries || 3;
-    this.delay = options.delay || 1000;
-    this.callback = options.callback;
-  }
-
-  async execute(fn: () => Promise<void>) {
-    let retries = this.retries;
-
-    while (true) {
-      try {
-        return await fn();
-      } catch (error) {
-        if (retries <= 0) {
-          throw error;
-        }
-
-        if (this.callback) {
-          const res = this.callback(retries, error as Error);
-          if (res) {
-            const { newRetries } = res;
-            if (newRetries === 0) throw error;
-            this.retries = newRetries || retries;
-          }
-        }
-
-        await sleep(this.delay);
-        retries--;
-      }
-    }
-  }
-
-  static create(options: PromiseRetryOptions) {
-    return new PromiseRetry(options);
-  }
-}

package/src/utils/requests.ts

@@ -29,8 +29,8 @@ export function fetchWithGlobalHeader(url: string) {
   return fetch(url, { headers: requestHeaders });
 }
 
-export function fetchByteRange(url: string, downloadedSize: number) {
+export function fetchByteRange(url: string, downloadedSize: number, signal?: AbortSignal) {
   const requestHeaders = new Headers(HeadersDefault);
   requestHeaders.set('Range', `bytes=${downloadedSize}-`);
-  return fetch(url, { headers: requestHeaders });
+  return fetch(url, { headers: requestHeaders, signal });
 }

package/src/utils/strings.ts

@@ -1,12 +1,3 @@
 export function b2mb(bytes: number) {
-  return Number.parseFloat((bytes / 1048576).toFixed(2));
-}
-
-export function formatNameStdout(pathname: string) {
-  const name = pathname.split('/').pop() || '';
-  const consoleWidth = process.stdout.columns;
-  const width = Math.max((consoleWidth / 2) | 0, 40);
-  if (name.length < width) return name.trim();
-  const result = `${name.slice(0, width - 15)} ... ${name.slice(-10)}`.replace(/ +/g, ' ');
-  return result;
+  return (bytes / 1048576).toFixed(2);
 }

package/src/utils/timer.ts

@@ -1,5 +1,8 @@
+import type { Subject } from 'rxjs';
+import type { AbortControllerSubject } from '../types';
+
 export class Timer {
-  private timer: NodeJS.Timeout | undefined = undefined;
+  private timer: NodeJS.Timeout | undefined;
 
   constructor(
     private timeout = 10_000,
@@ -30,18 +33,17 @@ export class Timer {
     return this;
   }
 
-  static withSignal(timeout?: number, message?: string) {
-    const controller = new AbortController();
-
+  static withAbortController(
+    timeout: number,
+    abortControllerSubject: Subject<AbortControllerSubject>,
+    message: AbortControllerSubject = 'TIMEOUT',
+  ) {
     const callback = () => {
-      controller.abort(message);
+      abortControllerSubject.next(message);
     };
 
     const timer = new Timer(timeout, callback).start();
 
-    return {
-      timer,
-      signal: controller.signal,
-    };
+    return { timer };
   }
 }

package/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "jsx": "react",
     "target": "ESNext",
     "useDefineForClassFields": true,
     "module": "esnext",
@@ -9,7 +10,7 @@
     "moduleResolution": "bundler",
     "isolatedModules": true,
     "moduleDetection": "force",
-    
+
     "declaration": true,
     "typeRoots": ["./dist/index.d.ts", "./src/types", "./node_modules/@types"],
     "outDir": "./dist",