coomer-downloader 3.1.0 → 3.3.2

package/src/cli/ui/components/spinner.tsx

@@ -0,0 +1,28 @@
+import type { SpinnerName } from 'cli-spinners';
+import spinners from 'cli-spinners';
+import { Text } from 'ink';
+import React, { useEffect, useState } from 'react';
+
+interface SpinnerProps {
+  type?: SpinnerName;
+}
+
+export function Spinner({ type = 'dots' }: SpinnerProps) {
+  const spinner = spinners[type];
+  const randomFrame = (spinner.frames.length * Math.random()) | 0;
+  const [frame, setFrame] = useState(randomFrame);
+
+  useEffect(() => {
+    const timer = setInterval(() => {
+      setFrame((previousFrame) => {
+        return (previousFrame + 1) % spinner.frames.length;
+      });
+    }, spinner.interval);
+
+    return () => {
+      clearInterval(timer);
+    };
+  }, [spinner]);
+
+  return <Text>{spinner.frames[frame]}</Text>;
+}

package/src/cli/ui/components/titlebar.tsx

@@ -0,0 +1,15 @@
+import { Box, Spacer, Text } from 'ink';
+import React from 'react';
+import { version } from '../../../../package.json';
+
+export function TitleBar() {
+  return (
+    <Box>
+      <Spacer></Spacer>
+      <Box borderColor={'magenta'} borderStyle={'arrow'}>
+        <Text color={'cyanBright'}>Coomer-Downloader {version}</Text>
+      </Box>
+      <Spacer></Spacer>
+    </Box>
+  );
+}

package/src/cli/ui/hooks/downloader.ts

@@ -0,0 +1,21 @@
+import { useEffect, useState } from 'react';
+import { useInkStore } from '../store';
+
+export const useDownloaderHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const filelist = downloader?.filelist;
+
+  const [_, setHelper] = useState(0);
+
+  useEffect(() => {
+    downloader?.subject.subscribe(({ type }) => {
+      if (
+        type === 'FILE_DOWNLOADING_START' ||
+        type === 'FILE_DOWNLOADING_END' ||
+        type === 'CHUNK_DOWNLOADING_UPDATE'
+      ) {
+        setHelper(Date.now());
+      }
+    });
+  });
+};

package/src/cli/ui/hooks/input.ts

@@ -0,0 +1,17 @@
+import { useInput } from 'ink';
+import { useInkStore } from '../store';
+
+// problems with tsx watch
+export const useInputHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const switchPreview = useInkStore((state) => state.switchPreview);
+
+  useInput((input) => {
+    if (input === 's') {
+      downloader?.skip();
+    }
+    if (input === 'p') {
+      switchPreview();
+    }
+  });
+};

package/src/cli/ui/index.tsx

@@ -0,0 +1,7 @@
+import { render } from 'ink';
+import React from 'react';
+import { App } from './app';
+
+export function createReactInk() {
+  return render(<App />);
+}

package/src/cli/ui/store/index.ts

@@ -0,0 +1,19 @@
+import { create } from 'zustand';
+import type { Downloader } from '../../../services/downloader';
+
+interface InkState {
+  preview: boolean;
+  switchPreview: () => void;
+  downloader?: Downloader;
+  setDownloader: (downloader: Downloader) => void;
+}
+
+export const useInkStore = create<InkState>((set) => ({
+  preview: false,
+  switchPreview: () =>
+    set((state) => ({
+      preview: !state.preview,
+    })),
+  downloader: undefined,
+  setDownloader: (downloader: Downloader) => set({ downloader }),
+}));

package/src/index.ts

@@ -1,36 +1,30 @@
-#!/usr/bin/env node
-import os from 'node:os';
-import path from 'node:path';
+#!/usr/bin/env -S node --no-warnings=ExperimentalWarning
+
 import process from 'node:process';
 import { apiHandler } from './api';
-import { argumentHander } from './args-handler';
-import type { ApiResult, MediaType } from './types';
-import { createMultibar, downloadFiles, filterKeywords, setGlobalHeaders } from './utils';
+import { argumentHander } from './cli/args-handler';
+import { createReactInk } from './cli/ui';
+import { useInkStore } from './cli/ui/store';
+import { Downloader } from './services/downloader';
+import { setGlobalHeaders } from './utils/requests';
 
 async function run() {
-  const { url, dir, media, include, exclude, skip } = argumentHander();
+  createReactInk();
 
-  const { dirName, files } = (await apiHandler(url, media as MediaType)) as ApiResult;
+  const { url, dir, media, include, exclude, skip } = argumentHander();
 
-  const downloadDir =
-    dir === './' ? path.resolve(dir, dirName) : path.join(os.homedir(), path.join(dir, dirName));
+  const filelist = await apiHandler(url);
 
-  const filteredFiles = filterKeywords(files.slice(skip), include, exclude);
+  filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
 
-  console.table([
-    {
-      found: files.length,
-      skip,
-      filtered: files.length - filteredFiles.length - skip,
-      folder: downloadDir,
-    },
-  ]);
+  await filelist.calculateFileSizes();
 
   setGlobalHeaders({ Referer: url });
 
-  createMultibar();
+  const downloader = new Downloader(filelist);
+  useInkStore.getState().setDownloader(downloader);
 
-  await downloadFiles(filteredFiles, downloadDir);
+  await downloader.downloadFiles();
 
   process.kill(process.pid, 'SIGINT');
 }

package/src/services/downloader.ts

@@ -0,0 +1,141 @@
+import fs from 'node:fs';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { Subject } from 'rxjs';
+import { tryFixCoomerUrl } from '../api/coomer-api';
+import type { DownloaderSubject } from '../types';
+import { getFileSize, mkdir } from '../utils/io';
+import { sleep } from '../utils/promise';
+import { fetchByteRange } from '../utils/requests';
+import { Timer } from '../utils/timer';
+import type { CoomerFile, CoomerFileList } from './file';
+
+export class Downloader {
+  public subject = new Subject<DownloaderSubject>();
+
+  private abortController = new AbortController();
+  public abortControllerSubject = new Subject<string>();
+
+  setAbortControllerListener() {
+    this.abortControllerSubject.subscribe((type) => {
+      this.abortController.abort(type);
+      this.abortController = new AbortController();
+    });
+  }
+
+  constructor(
+    public filelist: CoomerFileList,
+    public chunkTimeout = 30_000,
+    public chunkFetchRetries = 5,
+    public fetchRetries = 7,
+  ) {
+    this.setAbortControllerListener();
+  }
+
+  async fetchStream(
+    file: CoomerFile,
+    stream: Readable,
+    sizeOld = 0,
+    retries = this.chunkFetchRetries,
+  ): Promise<void> {
+    const signal = this.abortController.signal;
+    const subject = this.subject;
+    const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
+    let i: NodeJS.Timeout | undefined;
+
+    try {
+      const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
+
+      const progressStream = new Transform({
+        transform(chunk, _encoding, callback) {
+          this.push(chunk);
+          file.downloaded += chunk.length;
+          timer.reset();
+          subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE' });
+          callback();
+        },
+      });
+
+      subject.next({ type: 'CHUNK_DOWNLOADING_START' });
+      await pipeline(stream, progressStream, fileStream, { signal });
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+        if (signal.reason === 'TIMEOUT') {
+          if (retries === 0 && sizeOld < file.downloaded) {
+            retries += this.chunkFetchRetries;
+            sizeOld = file.downloaded;
+          }
+          if (retries === 0) return;
+          return await this.fetchStream(file, stream, sizeOld, retries - 1);
+        }
+      }
+      throw error;
+    } finally {
+      subject.next({ type: 'CHUNK_DOWNLOADING_END' });
+      timer.stop();
+      clearInterval(i);
+    }
+  }
+
+  public skip() {
+    this.abortControllerSubject.next('FILE_SKIP');
+  }
+
+  async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
+    const signal = this.abortController.signal;
+    try {
+      file.downloaded = await getFileSize(file.filepath as string);
+
+      const response = await fetchByteRange(file.url, file.downloaded, signal);
+
+      if (!response?.ok && response?.status !== 416) {
+        throw new Error(`HTTP error! status: ${response?.status}`);
+      }
+
+      const contentLength = response.headers.get('Content-Length') as string;
+
+      if (!contentLength && file.downloaded > 0) return;
+
+      const restFileSize = parseInt(contentLength);
+      file.size = restFileSize + file.downloaded;
+
+      if (file.size > file.downloaded && response.body) {
+        const stream = Readable.fromWeb(response.body);
+        stream.setMaxListeners(20);
+        await this.fetchStream(file, stream, file.downloaded);
+      }
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+      }
+      if (retries > 0) {
+        if (/coomer|kemono/.test(file.url)) {
+          file.url = tryFixCoomerUrl(file.url, retries);
+        }
+        await sleep(1000);
+        return await this.downloadFile(file, retries - 1);
+      }
+      throw error;
+    }
+  }
+
+  async downloadFiles(): Promise<void> {
+    mkdir(this.filelist.dirPath as string);
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_START' });
+    for (const file of this.filelist.files) {
+      file.active = true;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_START' });
+
+      await this.downloadFile(file);
+
+      file.active = false;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_END' });
+    }
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_END' });
+  }
+}

package/src/services/file.ts

@@ -0,0 +1,95 @@
+import os from 'node:os';
+import path from 'node:path';
+import type { MediaType } from '../types';
+import { filterString, testMediaType } from '../utils/filters';
+import { getFileSize } from '../utils/io';
+
+interface ICoomerFile {
+  name: string;
+  url: string;
+  filepath?: string;
+  size?: number;
+  downloaded?: number;
+  content?: string;
+}
+
+export class CoomerFile {
+  public active = false;
+
+  constructor(
+    public name: string,
+    public url: string,
+    public filepath?: string,
+    public size?: number,
+    public downloaded = 0,
+    public content?: string,
+  ) {}
+
+  public async getDownloadedSize() {
+    this.downloaded = await getFileSize(this.filepath as string);
+    return this;
+  }
+
+  public get textContent() {
+    const text = `${this.name || ''} ${this.content || ''}`.toLowerCase();
+    return text;
+  }
+
+  public static from(f: ICoomerFile) {
+    return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
+  }
+}
+
+export class CoomerFileList {
+  public dirPath?: string;
+  public dirName?: string;
+
+  constructor(public files: CoomerFile[] = []) {}
+
+  public setDirPath(dir: string, dirName?: string) {
+    dirName = dirName || (this.dirName as string);
+
+    if (dir === './') {
+      this.dirPath = path.resolve(dir, dirName);
+    } else {
+      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
+    }
+
+    this.files.forEach((file) => {
+      file.filepath = path.join(this.dirPath as string, file.name);
+    });
+
+    return this;
+  }
+
+  public filterByText(include: string, exclude: string) {
+    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
+    return this;
+  }
+
+  public filterByMediaType(media?: string) {
+    if (media) {
+      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
+    }
+    return this;
+  }
+
+  public skip(n: number) {
+    this.files = this.files.slice(n);
+    return this;
+  }
+
+  public async calculateFileSizes() {
+    for (const file of this.files) {
+      await file.getDownloadedSize();
+    }
+  }
+
+  public getActiveFiles() {
+    return this.files.filter((f) => f.active);
+  }
+
+  public getDownloaded() {
+    return this.files.filter((f) => f.size && f.size <= f.downloaded);
+  }
+}

package/src/types/index.ts

@@ -1,23 +1,15 @@
-export type File = {
-  name: string;
-  url: string;
-  filepath?: string;
-  size?: number;
-  downloaded?: number;
-  content?: string;
-};
+export type MediaType = 'video' | 'image' | 'all';
 
-export type DownloaderSubject = {
-  type: string;
-  file?: File;
-  index?: number;
-  filesCount?: number;
-};
+export type DownloaderSubjectSignal =
+  | 'FILES_DOWNLOADING_START'
+  | 'FILES_DOWNLOADING_END'
+  | 'FILE_DOWNLOADING_START'
+  | 'FILE_DOWNLOADING_END'
+  | 'FILE_SKIP'
+  | 'CHUNK_DOWNLOADING_START'
+  | 'CHUNK_DOWNLOADING_UPDATE'
+  | 'CHUNK_DOWNLOADING_END';
 
-export type ApiResult = {
-  files: File[];
-  // should merge into filepaths?
-  dirName: string;
+export type DownloaderSubject = {
+  type: DownloaderSubjectSignal;
 };
-
-export type MediaType = 'video' | 'image' | 'all';

package/src/utils/filters.ts

@@ -1,12 +1,16 @@
-import type { File, MediaType } from '../types';
+import type { MediaType } from '../types';
 
-export const isImage = (name: string) => /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+export function isImage(name: string) {
+  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+}
 
-export const isVideo = (name: string) =>
-  /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+export function isVideo(name: string) {
+  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+}
 
-export const testMediaType = (name: string, type: MediaType) =>
-  type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
+export function testMediaType(name: string, type: MediaType) {
+  return type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
+}
 
 function includesAllWords(str: string, words: string[]) {
   if (!words.length) return true;
@@ -25,13 +29,6 @@ function parseQuery(query: string) {
     .filter((_) => _);
 }
 
-function filterString(text: string, include: string, exclude: string): boolean {
+export function filterString(text: string, include: string, exclude: string): boolean {
   return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
 }
-
-export function filterKeywords(files: File[], include: string, exclude: string) {
-  return files.filter((f) => {
-    const text = `${f.name || ''} ${f.content || ''}`.toLowerCase();
-    return filterString(text, include, exclude);
-  });
-}

package/src/utils/promise.ts

@@ -1,53 +1,3 @@
 export async function sleep(time: number) {
   return new Promise((resolve) => setTimeout(resolve, time));
 }
-
-type PromiseRetryCallback = (retries: number, error: Error) => void | { newRetries?: number };
-
-interface PromiseRetryOptions {
-  retries?: number;
-  callback?: PromiseRetryCallback;
-  delay?: number;
-}
-
-export class PromiseRetry {
-  private retries: number;
-  private delay: number;
-  private callback?: PromiseRetryCallback;
-
-  constructor(options: PromiseRetryOptions) {
-    this.retries = options.retries || 3;
-    this.delay = options.delay || 1000;
-    this.callback = options.callback;
-  }
-
-  async execute(fn: () => Promise<void>) {
-    let retries = this.retries;
-
-    while (true) {
-      try {
-        return await fn();
-      } catch (error) {
-        if (retries <= 0) {
-          throw error;
-        }
-
-        if (this.callback) {
-          const res = this.callback(retries, error as Error);
-          if (res) {
-            const { newRetries } = res;
-            if (newRetries === 0) throw error;
-            this.retries = newRetries || retries;
-          }
-        }
-
-        await sleep(this.delay);
-        retries--;
-      }
-    }
-  }
-
-  static create(options: PromiseRetryOptions) {
-    return new PromiseRetry(options);
-  }
-}

package/src/utils/requests.ts

@@ -29,8 +29,8 @@ export function fetchWithGlobalHeader(url: string) {
   return fetch(url, { headers: requestHeaders });
 }
 
-export function fetchByteRange(url: string, downloadedSize: number) {
+export function fetchByteRange(url: string, downloadedSize: number, signal?: AbortSignal) {
   const requestHeaders = new Headers(HeadersDefault);
   requestHeaders.set('Range', `bytes=${downloadedSize}-`);
-  return fetch(url, { headers: requestHeaders });
+  return fetch(url, { headers: requestHeaders, signal });
 }

package/src/utils/strings.ts

@@ -1,21 +1,3 @@
 export function b2mb(bytes: number) {
-  return Number.parseFloat((bytes / 1048576).toFixed(2));
-}
-
-export function sanitizeString(str: string) {
-  return (
-    str
-      .match(/(\w| |-)/g)
-      ?.join('')
-      .replace(/ +/g, ' ') || ''
-  );
-}
-
-export function formatNameStdout(pathname: string) {
-  const name = pathname.split('/').pop() || '';
-  const consoleWidth = process.stdout.columns;
-  const width = Math.max((consoleWidth / 2) | 0, 40);
-  if (name.length < width) return name.trim();
-  const result = `${name.slice(0, width - 15)} ... ${name.slice(-10)}`.replace(/ +/g, ' ');
-  return result;
+  return (bytes / 1048576).toFixed(2);
 }

package/src/utils/timer.ts

@@ -1,5 +1,7 @@
+import type { Subject } from 'rxjs';
+
 export class Timer {
-  private timer: NodeJS.Timeout | undefined = undefined;
+  private timer: NodeJS.Timeout | undefined;
 
   constructor(
     private timeout = 10_000,
@@ -30,18 +32,17 @@ export class Timer {
     return this;
   }
 
-  static withSignal(timeout?: number, message?: string) {
-    const controller = new AbortController();
-
+  static withAbortController(
+    timeout: number,
+    abortControllerSubject: Subject<string>,
+    message: string = 'Timeout',
+  ) {
     const callback = () => {
-      controller.abort(message);
+      abortControllerSubject.next(message);
     };
 
     const timer = new Timer(timeout, callback).start();
 
-    return {
-      timer,
-      signal: controller.signal,
-    };
+    return { timer };
   }
 }

package/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "jsx": "react",
     "target": "ESNext",
     "useDefineForClassFields": true,
     "module": "esnext",
@@ -9,7 +10,7 @@
     "moduleResolution": "bundler",
     "isolatedModules": true,
     "moduleDetection": "force",
-    
+
     "declaration": true,
     "typeRoots": ["./dist/index.d.ts", "./src/types", "./node_modules/@types"],
     "outDir": "./dist",