coomer-downloader 3.2.0 → 3.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.2.0",
+  "version": "3.3.2",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",
@@ -22,10 +22,11 @@
     "type": "git",
     "url": "git+https://github.com/smartacephale/coomer-downloader.git"
   },
-  "main": "./src/index.ts",
+  "main": "./src/index.tsx",
   "type": "module",
   "scripts": {
     "start": "tsx ./src/index.ts",
+    "start-watch": "tsx watch ./src/index.ts",
     "build": "node build.js",
     "start-build": "npm run build && node ./dist/index.js"
   },
@@ -38,18 +39,25 @@
   },
   "dependencies": {
     "cheerio": "1.1.0",
-    "cli-progress": "^3.12.0",
+    "cli-spinners": "^3.3.0",
     "http-cookie-agent": "^6.0.0",
+    "ink": "^6.5.1",
+    "ink-picture": "^1.3.3",
+    "react": "^19.2.0",
     "rxjs": "^7.8.2",
     "tough-cookie": "5.1.2",
     "undici": "^6.22.0",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "zustand": "^5.0.8"
   },
   "devDependencies": {
-    "@types/cli-progress": "^3.11.6",
     "@types/node": "^24.10.0",
+    "@types/react": "^19.2.6",
     "@types/yargs": "^17.0.34",
     "esbuild": "^0.27.0",
+    "eslint-config-xo-react": "^0.27.0",
+    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
     "tsx": "^4.20.6"
   }
 }

package/src/api/bunkr.ts

@@ -1,6 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 type EncData = { url: string; timestamp: number };
 

package/src/api/coomer-api.ts

@@ -1,5 +1,6 @@
-import { CoomerFile, CoomerFileList } from '../utils/file.js';
-import { fetchWithGlobalHeader, isImage, setGlobalHeaders } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../services/file';
+import { isImage } from '../utils/filters';
+import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
 
 type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
 type CoomerAPIUserData = { name: string };

package/src/api/gofile.ts

@@ -1,6 +1,6 @@
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file.js';
-import { setGlobalHeaders } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../services/file';
+import { setGlobalHeaders } from '../utils/requests';
 
 type GoFileAPIToken = { status: string; data: { token: string } };
 type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };

package/src/api/index.ts

@@ -1,4 +1,4 @@
-import type { CoomerFileList } from '../utils/file';
+import type { CoomerFileList } from '../services/file';
 import { getBunkrData } from './bunkr';
 import { getCoomerData } from './coomer-api';
 import { getGofileData } from './gofile';
@@ -11,15 +11,19 @@ export async function apiHandler(url_: string): Promise<CoomerFileList> {
   if (/^u\/\w+$/.test(url.origin)) {
     return getRedditData(url.href);
   }
+
   if (/coomer|kemono/.test(url.origin)) {
     return getCoomerData(url.href);
   }
+
   if (/bunkr/.test(url.origin)) {
     return getBunkrData(url.href);
   }
+
   if (/gofile\.io/.test(url.origin)) {
     return getGofileData(url.href);
   }
+
   if (/\.\w+/.test(url.pathname)) {
     return getPlainFileData(url.href);
   }

package/src/api/nsfw.xxx.ts

@@ -1,6 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;

package/src/api/plain-curl.ts

@@ -1,4 +1,4 @@
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 export async function getPlainFileData(url: string): Promise<CoomerFileList> {
   const name = url.split('/').pop() as string;

package/src/cli/ui/app.tsx

@@ -0,0 +1,40 @@
+import { Box } from 'ink';
+import React from 'react';
+import { CoomerFileList } from '../../services/file';
+import { FileBox, FileListStateBox, KeyboardControlsInfo, Loading, TitleBar } from './components';
+import { useDownloaderHook } from './hooks/downloader';
+import { useInputHook } from './hooks/input';
+import { useInkStore } from './store';
+
+export function App() {
+  useInputHook();
+  useDownloaderHook();
+
+  const downloader = useInkStore((state) => state.downloader);
+  const filelist = downloader?.filelist;
+  const isFilelist = filelist instanceof CoomerFileList;
+
+  return (
+    <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
+      <TitleBar />
+      {!isFilelist ? (
+        <Loading />
+      ) : (
+        <>
+          <Box>
+            <Box>
+              <FileListStateBox filelist={filelist} />
+            </Box>
+            <Box flexBasis={29}>
+              <KeyboardControlsInfo />
+            </Box>
+          </Box>
+
+          {filelist.getActiveFiles().map((file) => {
+            return <FileBox file={file} key={file.name} />;
+          })}
+        </>
+      )}
+    </Box>
+  );
+}

package/src/cli/ui/components/file.tsx

@@ -0,0 +1,44 @@
+import { Box, Spacer, Text } from 'ink';
+import React from 'react';
+import type { CoomerFile } from '../../../services/file';
+import { b2mb } from '../../../utils/strings';
+import { Preview } from './preview';
+import { Spinner } from './spinner';
+
+interface FileBoxProps {
+  file: CoomerFile;
+}
+
+export function FileBox({ file }: FileBoxProps) {
+  const percentage = Number((file.downloaded / (file.size as number)) * 100).toFixed(2);
+  return (
+    <>
+      <Box
+        borderStyle="single"
+        borderColor="magentaBright"
+        borderDimColor
+        paddingX={1}
+        flexDirection="column"
+      >
+        <Box>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {file.name}
+          </Text>
+        </Box>
+        <Box flexDirection="row-reverse">
+          <Text color="cyan" dimColor>
+            {b2mb(file.downloaded)}/{file.size ? b2mb(file.size) : '∞'} MB
+          </Text>
+          <Text color="redBright" dimColor>
+            {file.size ? `  ${percentage}% ` : ''}
+          </Text>
+          <Spacer />
+          <Text color={'green'} dimColor>
+            <Spinner></Spinner>
+          </Text>
+        </Box>
+      </Box>
+      <Preview file={file} />
+    </>
+  );
+}

package/src/cli/ui/components/filelist.tsx

@@ -0,0 +1,52 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import type { CoomerFileList } from '../../../services/file';
+
+interface FileListStateBoxProps {
+  filelist: CoomerFileList;
+}
+
+export function FileListStateBox({ filelist }: FileListStateBoxProps) {
+  return (
+    <Box
+      paddingX={1}
+      flexDirection="column"
+      borderStyle={'single'}
+      borderColor={'magenta'}
+      borderDimColor
+    >
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Found:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist.files.length}
+        </Text>
+      </Box>
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Downloaded:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist.getDownloaded().length}
+        </Text>
+      </Box>
+      <Box>
+        <Box width={9}>
+          <Text color="cyanBright" dimColor>
+            Folder:
+          </Text>
+        </Box>
+        <Box flexGrow={1}>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {filelist.dirPath}
+          </Text>
+        </Box>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/index.ts

@@ -0,0 +1,6 @@
+export { FileBox } from './file';
+export { FileListStateBox } from './filelist';
+export { KeyboardControlsInfo } from './keyboardinfo';
+export { Loading } from './loading';
+export { Spinner } from './spinner';
+export { TitleBar } from './titlebar';

package/src/cli/ui/components/keyboardinfo.tsx

@@ -0,0 +1,41 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+
+const info = {
+  's ': 'skip current file',
+  p: 'on/off image preview',
+};
+
+export function KeyboardControlsInfo() {
+  const infoRender = Object.entries(info).map(([key, value]) => {
+    return (
+      <Box key={key}>
+        <Box marginRight={2}>
+          <Text color={'red'} dimColor bold>
+            {key}
+          </Text>
+        </Box>
+        <Text dimColor bold={false}>
+          {value}
+        </Text>
+      </Box>
+    );
+  });
+
+  return (
+    <Box
+      flexDirection="column"
+      paddingX={1}
+      borderStyle={'single'}
+      borderColor={'gray'}
+      borderDimColor
+    >
+      <Box>
+        <Text color={'red'} dimColor bold>
+          Keyboard controls:
+        </Text>
+      </Box>
+      {infoRender}
+    </Box>
+  );
+}

package/src/cli/ui/components/loading.tsx

@@ -0,0 +1,20 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import { Spinner } from './spinner';
+
+export function Loading() {
+  return (
+    <Box paddingX={1} borderDimColor flexDirection="column">
+      <Box alignSelf="center">
+        <Text dimColor color={'redBright'}>
+          Fetching Data
+        </Text>
+      </Box>
+      <Box alignSelf="center">
+        <Text color={'blueBright'} dimColor>
+          <Spinner type="grenade"></Spinner>
+        </Text>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/preview.tsx

@@ -0,0 +1,32 @@
+import { Box } from 'ink';
+import Image, { TerminalInfoProvider } from 'ink-picture';
+import React from 'react';
+import type { CoomerFile } from '../../../services/file';
+import { isImage } from '../../../utils/filters';
+import { useInkStore } from '../store';
+
+interface PreviewProps {
+  file: CoomerFile;
+}
+
+export function Preview({ file }: PreviewProps) {
+  const previewEnabled = useInkStore((state) => state.preview);
+  const bigEnough = file.downloaded > 50 * 1024;
+  const shouldShow = previewEnabled && bigEnough && isImage(file.filepath as string);
+  const imgInfo = `
+    can't read partial images yet...
+    actual size: ${file.size}}
+    downloaded: ${file.downloaded}}
+  `;
+  return (
+    shouldShow && (
+      <Box paddingX={1}>
+        <TerminalInfoProvider>
+          <Box width={30} height={15}>
+            <Image src={file.filepath as string} alt={imgInfo} />
+          </Box>
+        </TerminalInfoProvider>
+      </Box>
+    )
+  );
+}

package/src/cli/ui/components/spinner.tsx

@@ -0,0 +1,28 @@
+import type { SpinnerName } from 'cli-spinners';
+import spinners from 'cli-spinners';
+import { Text } from 'ink';
+import React, { useEffect, useState } from 'react';
+
+interface SpinnerProps {
+  type?: SpinnerName;
+}
+
+export function Spinner({ type = 'dots' }: SpinnerProps) {
+  const spinner = spinners[type];
+  const randomFrame = (spinner.frames.length * Math.random()) | 0;
+  const [frame, setFrame] = useState(randomFrame);
+
+  useEffect(() => {
+    const timer = setInterval(() => {
+      setFrame((previousFrame) => {
+        return (previousFrame + 1) % spinner.frames.length;
+      });
+    }, spinner.interval);
+
+    return () => {
+      clearInterval(timer);
+    };
+  }, [spinner]);
+
+  return <Text>{spinner.frames[frame]}</Text>;
+}

package/src/cli/ui/components/titlebar.tsx

@@ -0,0 +1,15 @@
+import { Box, Spacer, Text } from 'ink';
+import React from 'react';
+import { version } from '../../../../package.json';
+
+export function TitleBar() {
+  return (
+    <Box>
+      <Spacer></Spacer>
+      <Box borderColor={'magenta'} borderStyle={'arrow'}>
+        <Text color={'cyanBright'}>Coomer-Downloader {version}</Text>
+      </Box>
+      <Spacer></Spacer>
+    </Box>
+  );
+}

package/src/cli/ui/hooks/downloader.ts

@@ -0,0 +1,21 @@
+import { useEffect, useState } from 'react';
+import { useInkStore } from '../store';
+
+export const useDownloaderHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const filelist = downloader?.filelist;
+
+  const [_, setHelper] = useState(0);
+
+  useEffect(() => {
+    downloader?.subject.subscribe(({ type }) => {
+      if (
+        type === 'FILE_DOWNLOADING_START' ||
+        type === 'FILE_DOWNLOADING_END' ||
+        type === 'CHUNK_DOWNLOADING_UPDATE'
+      ) {
+        setHelper(Date.now());
+      }
+    });
+  });
+};

package/src/cli/ui/hooks/input.ts

@@ -0,0 +1,17 @@
+import { useInput } from 'ink';
+import { useInkStore } from '../store';
+
+// problems with tsx watch
+export const useInputHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const switchPreview = useInkStore((state) => state.switchPreview);
+
+  useInput((input) => {
+    if (input === 's') {
+      downloader?.skip();
+    }
+    if (input === 'p') {
+      switchPreview();
+    }
+  });
+};

package/src/cli/ui/index.tsx

@@ -0,0 +1,7 @@
+import { render } from 'ink';
+import React from 'react';
+import { App } from './app';
+
+export function createReactInk() {
+  return render(<App />);
+}

package/src/cli/ui/store/index.ts

@@ -0,0 +1,19 @@
+import { create } from 'zustand';
+import type { Downloader } from '../../../services/downloader';
+
+interface InkState {
+  preview: boolean;
+  switchPreview: () => void;
+  downloader?: Downloader;
+  setDownloader: (downloader: Downloader) => void;
+}
+
+export const useInkStore = create<InkState>((set) => ({
+  preview: false,
+  switchPreview: () =>
+    set((state) => ({
+      preview: !state.preview,
+    })),
+  downloader: undefined,
+  setDownloader: (downloader: Downloader) => set({ downloader }),
+}));

package/src/index.ts

@@ -1,34 +1,30 @@
-#!/usr/bin/env node
+#!/usr/bin/env -S node --no-warnings=ExperimentalWarning
+
 import process from 'node:process';
 import { apiHandler } from './api';
-import { argumentHander } from './args-handler';
-import { createMultibar, Downloader, setGlobalHeaders } from './utils';
+import { argumentHander } from './cli/args-handler';
+import { createReactInk } from './cli/ui';
+import { useInkStore } from './cli/ui/store';
+import { Downloader } from './services/downloader';
+import { setGlobalHeaders } from './utils/requests';
 
 async function run() {
+  createReactInk();
+
   const { url, dir, media, include, exclude, skip } = argumentHander();
 
   const filelist = await apiHandler(url);
 
-  const found = filelist.files.length;
-  filelist.setDirPath(dir);
-  filelist.skip(skip);
-  filelist.filterByText(include, exclude);
-  filelist.filterByMediaType(media);
-
-  console.table([
-    {
-      found,
-      skip,
-      filtered: found - filelist.files.length,
-      folder: filelist.dirPath,
-    },
-  ]);
+  filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
+
+  await filelist.calculateFileSizes();
 
   setGlobalHeaders({ Referer: url });
 
-  const downloader = new Downloader();
-  createMultibar(downloader);
-  await downloader.downloadFiles(filelist);
+  const downloader = new Downloader(filelist);
+  useInkStore.getState().setDownloader(downloader);
+
+  await downloader.downloadFiles();
 
   process.kill(process.pid, 'SIGINT');
 }

package/src/services/downloader.ts

@@ -0,0 +1,141 @@
+import fs from 'node:fs';
+import { Readable, Transform } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { Subject } from 'rxjs';
+import { tryFixCoomerUrl } from '../api/coomer-api';
+import type { DownloaderSubject } from '../types';
+import { getFileSize, mkdir } from '../utils/io';
+import { sleep } from '../utils/promise';
+import { fetchByteRange } from '../utils/requests';
+import { Timer } from '../utils/timer';
+import type { CoomerFile, CoomerFileList } from './file';
+
+export class Downloader {
+  public subject = new Subject<DownloaderSubject>();
+
+  private abortController = new AbortController();
+  public abortControllerSubject = new Subject<string>();
+
+  setAbortControllerListener() {
+    this.abortControllerSubject.subscribe((type) => {
+      this.abortController.abort(type);
+      this.abortController = new AbortController();
+    });
+  }
+
+  constructor(
+    public filelist: CoomerFileList,
+    public chunkTimeout = 30_000,
+    public chunkFetchRetries = 5,
+    public fetchRetries = 7,
+  ) {
+    this.setAbortControllerListener();
+  }
+
+  async fetchStream(
+    file: CoomerFile,
+    stream: Readable,
+    sizeOld = 0,
+    retries = this.chunkFetchRetries,
+  ): Promise<void> {
+    const signal = this.abortController.signal;
+    const subject = this.subject;
+    const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
+    let i: NodeJS.Timeout | undefined;
+
+    try {
+      const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
+
+      const progressStream = new Transform({
+        transform(chunk, _encoding, callback) {
+          this.push(chunk);
+          file.downloaded += chunk.length;
+          timer.reset();
+          subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE' });
+          callback();
+        },
+      });
+
+      subject.next({ type: 'CHUNK_DOWNLOADING_START' });
+      await pipeline(stream, progressStream, fileStream, { signal });
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+        if (signal.reason === 'TIMEOUT') {
+          if (retries === 0 && sizeOld < file.downloaded) {
+            retries += this.chunkFetchRetries;
+            sizeOld = file.downloaded;
+          }
+          if (retries === 0) return;
+          return await this.fetchStream(file, stream, sizeOld, retries - 1);
+        }
+      }
+      throw error;
+    } finally {
+      subject.next({ type: 'CHUNK_DOWNLOADING_END' });
+      timer.stop();
+      clearInterval(i);
+    }
+  }
+
+  public skip() {
+    this.abortControllerSubject.next('FILE_SKIP');
+  }
+
+  async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
+    const signal = this.abortController.signal;
+    try {
+      file.downloaded = await getFileSize(file.filepath as string);
+
+      const response = await fetchByteRange(file.url, file.downloaded, signal);
+
+      if (!response?.ok && response?.status !== 416) {
+        throw new Error(`HTTP error! status: ${response?.status}`);
+      }
+
+      const contentLength = response.headers.get('Content-Length') as string;
+
+      if (!contentLength && file.downloaded > 0) return;
+
+      const restFileSize = parseInt(contentLength);
+      file.size = restFileSize + file.downloaded;
+
+      if (file.size > file.downloaded && response.body) {
+        const stream = Readable.fromWeb(response.body);
+        stream.setMaxListeners(20);
+        await this.fetchStream(file, stream, file.downloaded);
+      }
+    } catch (error) {
+      if (signal.aborted) {
+        if (signal.reason === 'FILE_SKIP') return;
+      }
+      if (retries > 0) {
+        if (/coomer|kemono/.test(file.url)) {
+          file.url = tryFixCoomerUrl(file.url, retries);
+        }
+        await sleep(1000);
+        return await this.downloadFile(file, retries - 1);
+      }
+      throw error;
+    }
+  }
+
+  async downloadFiles(): Promise<void> {
+    mkdir(this.filelist.dirPath as string);
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_START' });
+    for (const file of this.filelist.files) {
+      file.active = true;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_START' });
+
+      await this.downloadFile(file);
+
+      file.active = false;
+
+      this.subject.next({ type: 'FILE_DOWNLOADING_END' });
+    }
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_END' });
+  }
+}