coomer-downloader 3.2.0 → 3.4.0

package/src/utils/downloader.ts

@@ -1,108 +0,0 @@
-import fs from 'node:fs';
-import { Readable, Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { Subject } from 'rxjs';
-import { tryFixCoomerUrl } from '../api/coomer-api';
-import type { CoomerFile, CoomerFileList } from './file';
-import { getFileSize, mkdir } from './io';
-import { PromiseRetry } from './promise';
-import { fetchByteRange } from './requests';
-import { Timer } from './timer';
-
-type DownloaderSubject = {
-  type: string;
-  file?: CoomerFile;
-  filesCount?: number;
-};
-
-export class Downloader {
-  public subject = new Subject<DownloaderSubject>();
-
-  constructor(
-    public chunkTimeout = 30_000,
-    public chunkFetchRetries = 5,
-    public fetchRetries = 7,
-  ) {}
-
-  async fetchStream(file: CoomerFile, stream: Readable): Promise<void> {
-    const { subject, chunkTimeout } = this;
-    const { timer, signal } = Timer.withSignal(chunkTimeout, 'chunkTimeout');
-
-    const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
-
-    const progressStream = new Transform({
-      transform(chunk, _encoding, callback) {
-        this.push(chunk);
-        file.downloaded += chunk.length;
-        timer.reset();
-        subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE', file });
-        callback();
-      },
-    });
-
-    try {
-      subject.next({ type: 'CHUNK_DOWNLOADING_START', file });
-      await pipeline(stream, progressStream, fileStream, { signal });
-    } catch (error) {
-      console.error((error as Error).name === 'AbortError' ? signal.reason : error);
-    } finally {
-      subject.next({ type: 'CHUNK_DOWNLOADING_END', file });
-    }
-  }
-
-  async downloadFile(file: CoomerFile): Promise<void> {
-    file.downloaded = await getFileSize(file.filepath as string);
-
-    const response = await fetchByteRange(file.url, file.downloaded);
-
-    if (!response?.ok && response?.status !== 416) {
-      throw new Error(`HTTP error! status: ${response?.status}`);
-    }
-
-    const contentLength = response.headers.get('Content-Length') as string;
-
-    if (!contentLength && file.downloaded > 0) return;
-
-    const restFileSize = parseInt(contentLength);
-    file.size = restFileSize + file.downloaded;
-
-    if (file.size > file.downloaded && response.body) {
-      const stream = Readable.fromWeb(response.body);
-      const sizeOld = file.downloaded;
-
-      await PromiseRetry.create({
-        retries: this.chunkFetchRetries,
-        callback: () => {
-          if (sizeOld !== file.downloaded) {
-            return { newRetries: 5 };
-          }
-        },
-      }).execute(async () => await this.fetchStream(file, stream));
-    }
-
-    this.subject.next({ type: 'FILE_DOWNLOADING_END' });
-  }
-
-  async downloadFiles(filelist: CoomerFileList): Promise<void> {
-    mkdir(filelist.dirPath as string);
-
-    this.subject.next({ type: 'FILES_DOWNLOADING_START', filesCount: filelist.files.length });
-
-    for (const file of filelist.files) {
-      this.subject.next({ type: 'FILE_DOWNLOADING_START' });
-
-      await PromiseRetry.create({
-        retries: this.fetchRetries,
-        callback: (retries) => {
-          if (/coomer|kemono/.test(file.url)) {
-            file.url = tryFixCoomerUrl(file.url, retries);
-          }
-        },
-      }).execute(async () => await this.downloadFile(file));
-
-      this.subject.next({ type: 'FILE_DOWNLOADING_END' });
-    }
-
-    this.subject.next({ type: 'FILES_DOWNLOADING_END' });
-  }
-}

package/src/utils/file.ts

@@ -1,75 +0,0 @@
-import os from 'node:os';
-import path from 'node:path';
-import type { MediaType } from '../types';
-import { filterString, testMediaType } from './filters';
-
-interface ICoomerFile {
-  name: string;
-  url: string;
-  filepath?: string;
-  size?: number;
-  downloaded?: number;
-  content?: string;
-}
-
-export class CoomerFile {
-  public state: 'downloading' | 'pause' = 'pause';
-
-  constructor(
-    public name: string,
-    public url: string,
-    public filepath?: string,
-    public size?: number,
-    public downloaded?: number,
-    public content?: string,
-  ) {}
-
-  get textContent() {
-    const text = `${this.name || ''} ${this.content || ''}`.toLowerCase();
-    return text;
-  }
-
-  public static from(f: ICoomerFile) {
-    return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
-  }
-}
-
-export class CoomerFileList {
-  public dirPath?: string;
-  public dirName?: string;
-
-  constructor(public files: CoomerFile[] = []) {}
-
-  public setDirPath(dir: string, dirName?: string) {
-    dirName = dirName || (this.dirName as string);
-
-    if (dir === './') {
-      this.dirPath = path.resolve(dir, dirName);
-    } else {
-      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
-    }
-
-    this.files.forEach((file) => {
-      file.filepath = path.join(this.dirPath as string, file.name);
-    });
-
-    return this;
-  }
-
-  public filterByText(include: string, exclude: string) {
-    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
-    return this;
-  }
-
-  public filterByMediaType(media?: string) {
-    if (media) {
-      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
-    }
-    return this;
-  }
-
-  public skip(n: number) {
-    this.files = this.files.slice(n);
-    return this;
-  }
-}

package/src/utils/index.ts

@@ -1,11 +0,0 @@
-export { Downloader } from './downloader';
-export { isImage, isVideo, testMediaType } from './filters';
-export { getFileSize, mkdir } from './io';
-export { createMultibar } from './multibar';
-export {
-  fetchByteRange,
-  fetchWithGlobalHeader,
-  HeadersDefault,
-  setGlobalHeaders,
-} from './requests';
-export { b2mb } from './strings';

package/src/utils/multibar.ts

@@ -1,62 +0,0 @@
-import { MultiBar, type Options, type SingleBar } from 'cli-progress';
-import type { Downloader } from './downloader';
-import { b2mb, formatNameStdout } from './strings';
-
-const config: Options = {
-  clearOnComplete: true,
-  gracefulExit: true,
-  autopadding: true,
-  hideCursor: true,
-  format: '{percentage}% | {filename} | {value}/{total}{size}',
-};
-
-export function createMultibar(downloader: Downloader) {
-  const multibar = new MultiBar(config);
-  let bar: SingleBar;
-  let minibar: SingleBar;
-  let filename: string;
-  let index = 0;
-
-  downloader.subject.subscribe({
-    next: ({ type, filesCount, file }) => {
-      switch (type) {
-        case 'FILES_DOWNLOADING_START':
-          bar?.stop();
-          bar = multibar.create(filesCount as number, 0);
-          break;
-
-        case 'FILES_DOWNLOADING_END':
-          bar?.stop();
-          break;
-
-        case 'FILE_DOWNLOADING_START':
-          bar?.update(++index, { filename: 'Downloaded files', size: '' });
-          break;
-
-        case 'FILE_DOWNLOADING_END':
-          multibar.remove(minibar);
-          break;
-
-        case 'CHUNK_DOWNLOADING_START':
-          multibar?.remove(minibar);
-          filename = formatNameStdout(file?.filepath as string);
-          minibar = multibar.create(b2mb(file?.size as number), b2mb(file?.downloaded as number));
-          break;
-
-        case 'CHUNK_DOWNLOADING_UPDATE':
-          minibar?.update(b2mb(file?.downloaded as number), {
-            filename: filename as string,
-            size: 'mb',
-          });
-          break;
-
-        case 'CHUNK_DOWNLOADING_END':
-          multibar?.remove(minibar);
-          break;
-
-        default:
-          break;
-      }
-    },
-  });
-}