coomer-downloader 3.4.0 → 3.4.2

package/src/{services → core}/downloader.ts

@@ -2,13 +2,13 @@ import fs from 'node:fs';
 import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
-import { tryFixCoomerUrl } from '../api/coomer-api';
 import type { AbortControllerSubject, DownloaderSubject } from '../types';
 import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
 import { fetchByteRange } from '../utils/requests';
 import { Timer } from '../utils/timer';
-import type { CoomerFile, CoomerFileList } from './file';
+import type { CoomerFile } from './file';
+import type { CoomerFileList } from './filelist';
 
 export class Downloader {
   public subject = new Subject<DownloaderSubject>();
@@ -34,7 +34,7 @@ export class Downloader {
     this.setAbortControllerListener();
   }
 
-  async fetchStream(
+  private async fetchStream(
     file: CoomerFile,
     stream: Readable,
     sizeOld = 0,
@@ -99,7 +99,10 @@ export class Downloader {
     }
   }
 
-  async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
+  public async downloadFile(
+    file: CoomerFile,
+    retries = this.fetchRetries,
+  ): Promise<void> {
     const signal = this.abortController.signal;
     try {
       file.downloaded = await getFileSize(file.filepath as string);
@@ -129,8 +132,8 @@ export class Downloader {
         if (signal.reason === 'FILE_SKIP') return;
       }
       if (retries > 0) {
-        if (/coomer|kemono/.test(file.url)) {
-          file.url = tryFixCoomerUrl(file.url, retries);
+        if (this.filelist.provider?.fixURL) {
+          file.url = this.filelist.provider.fixURL(file.url, retries);
         }
         await sleep(1000);
         return await this.downloadFile(file, retries - 1);
@@ -139,7 +142,7 @@ export class Downloader {
     }
   }
 
-  async downloadFiles(): Promise<void> {
+  public async downloadFiles(): Promise<void> {
     mkdir(this.filelist.dirPath as string);
 
     this.subject.next({ type: 'FILES_DOWNLOADING_START' });

package/src/core/file.ts

@@ -0,0 +1,29 @@
+import { getFileSize } from '../utils/io';
+
+export class CoomerFile {
+  public active = false;
+  public hash?: string;
+
+  constructor(
+    public name: string,
+    public url: string,
+    public filepath = '',
+    public size?: number,
+    public downloaded = 0,
+    public content?: string,
+  ) {}
+
+  public async calcDownloadedSize() {
+    this.downloaded = await getFileSize(this.filepath as string);
+    return this;
+  }
+
+  public get textContent() {
+    const text = `${this.name || ''} ${this.content || ''}`.toLowerCase();
+    return text;
+  }
+
+  public static from(f: Pick<CoomerFile, 'name' | 'url'> & Partial<CoomerFile>) {
+    return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
+  }
+}

package/src/{services/file.ts → core/filelist.ts}

@@ -1,43 +1,17 @@
 import os from 'node:os';
 import path from 'node:path';
-import logger from '../logger';
+import type { ProviderAPI } from '../api/provider';
 import type { MediaType } from '../types';
 import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
 import { filterString } from '../utils/filters';
-import { deleteFile, getFileHash, getFileSize, sanitizeFilename } from '../utils/io';
+import { deleteFile, getFileHash, sanitizeFilename } from '../utils/io';
 import { testMediaType } from '../utils/mediatypes';
-
-export class CoomerFile {
-  public active = false;
-  public hash?: string;
-
-  constructor(
-    public name: string,
-    public url: string,
-    public filepath = '',
-    public size?: number,
-    public downloaded = 0,
-    public content?: string,
-  ) {}
-
-  public async getDownloadedSize() {
-    this.downloaded = await getFileSize(this.filepath as string);
-    return this;
-  }
-
-  public get textContent() {
-    const text = `${this.name || ''} ${this.content || ''}`.toLowerCase();
-    return text;
-  }
-
-  public static from(f: Pick<CoomerFile, 'name' | 'url'> & Partial<CoomerFile>) {
-    return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
-  }
-}
+import type { CoomerFile } from './file';
 
 export class CoomerFileList {
   public dirPath?: string;
   public dirName?: string;
+  public provider?: ProviderAPI;
 
   constructor(public files: CoomerFile[] = []) {}
 
@@ -77,7 +51,7 @@ export class CoomerFileList {
 
   public async calculateFileSizes() {
     for (const file of this.files) {
-      await file.getDownloadedSize();
+      await file.calcDownloadedSize();
     }
     return this;
   }
@@ -97,9 +71,9 @@ export class CoomerFileList {
 
     const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
 
-    console.log({ duplicates });
+    // console.log({ duplicates });
 
-    logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+    // logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
 
     duplicates.forEach((f) => {
       deleteFile(f.filepath);

package/src/core/index.ts

@@ -0,0 +1,3 @@
+export { Downloader } from './downloader';
+export { CoomerFile } from './file';
+export { CoomerFileList } from './filelist';

package/src/index.ts

@@ -1,11 +1,11 @@
 #!/usr/bin/env -S node --no-warnings=ExperimentalWarning
 
 import process from 'node:process';
-import { apiHandler } from './api';
+import { resolveAPI } from './api';
 import { argumentHander } from './cli/args-handler';
 import { createReactInk } from './cli/ui';
 import { useInkStore } from './cli/ui/store';
-import { Downloader } from './services/downloader';
+import { Downloader } from './core';
 import { parseSizeValue } from './utils/filters';
 import { setGlobalHeaders } from './utils/requests';
 
@@ -15,7 +15,7 @@ async function run() {
   const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
     argumentHander();
 
-  const filelist = await apiHandler(url);
+  const filelist = await resolveAPI(url);
 
   filelist
     .setDirPath(dir)

package/src/api/plain-curl.ts

@@ -1,9 +0,0 @@
-import { CoomerFile, CoomerFileList } from '../services/file';
-
-export async function getPlainFileData(url: string): Promise<CoomerFileList> {
-  const name = url.split('/').pop() as string;
-  const file = CoomerFile.from({ name, url });
-  const filelist = new CoomerFileList([file]);
-  filelist.dirName = '';
-  return filelist;
-}