coomer-downloader 3.2.0 → 3.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.2.0",
+  "version": "3.4.0",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",
@@ -22,10 +22,11 @@
     "type": "git",
     "url": "git+https://github.com/smartacephale/coomer-downloader.git"
   },
-  "main": "./src/index.ts",
+  "main": "./src/index.tsx",
   "type": "module",
   "scripts": {
     "start": "tsx ./src/index.ts",
+    "start-watch": "tsx watch ./src/index.ts",
     "build": "node build.js",
     "start-build": "npm run build && node ./dist/index.js"
   },
@@ -38,18 +39,26 @@
   },
   "dependencies": {
     "cheerio": "1.1.0",
-    "cli-progress": "^3.12.0",
+    "cli-spinners": "^3.3.0",
     "http-cookie-agent": "^6.0.0",
+    "ink": "^6.5.1",
+    "ink-picture": "^1.3.3",
+    "react": "^19.2.0",
     "rxjs": "^7.8.2",
     "tough-cookie": "5.1.2",
     "undici": "^6.22.0",
-    "yargs": "^17.7.2"
+    "yargs": "^17.7.2",
+    "zustand": "^5.0.8"
   },
   "devDependencies": {
-    "@types/cli-progress": "^3.11.6",
     "@types/node": "^24.10.0",
+    "@types/react": "^19.2.6",
     "@types/yargs": "^17.0.34",
     "esbuild": "^0.27.0",
+    "eslint-config-xo-react": "^0.27.0",
+    "eslint-plugin-react": "^7.32.2",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "pino": "^10.1.0",
     "tsx": "^4.20.6"
   }
 }

package/src/api/bunkr.ts

@@ -1,6 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 type EncData = { url: string; timestamp: number };
 

package/src/api/coomer-api.ts

@@ -1,5 +1,6 @@
-import { CoomerFile, CoomerFileList } from '../utils/file.js';
-import { fetchWithGlobalHeader, isImage, setGlobalHeaders } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../services/file';
+import { isImage } from '../utils/mediatypes';
+import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
 
 type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
 type CoomerAPIUserData = { name: string };
@@ -33,7 +34,10 @@ async function getUserProfileData(user: CoomerAPIUser): Promise<CoomerAPIUserDat
   return result as CoomerAPIUserData;
 }
 
-async function getUserPostsAPI(user: CoomerAPIUser, offset: number): Promise<CoomerAPIPost[]> {
+async function getUserPostsAPI(
+  user: CoomerAPIUser,
+  offset: number,
+): Promise<CoomerAPIPost[]> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
   const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
   return posts as CoomerAPIPost[];
@@ -43,10 +47,10 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
   const userPosts = [];
 
   const offset = 50;
-  for (let i = 0; i < 1000; i++) {
+  for (let i = 0; i < 10_000; i++) {
     const posts = await getUserPostsAPI(user, i * offset);
     userPosts.push(...posts);
-    if (posts.length < 50) break;
+    if (posts.length < offset) break;
   }
 
   const filelist = new CoomerFileList();
@@ -62,7 +66,7 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
       .map((f, i) => {
         const ext = f.name.split('.').pop();
         const name = `${datentitle} ${i + 1}.${ext}`;
-        const url = `${user.domain}/${f.path}`;
+        const url = getUrl(f, user);
         return CoomerFile.from({ name, url, content });
       });
 
@@ -72,6 +76,19 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
   return filelist;
 }
 
+function getUrl(f: CoomerAPIFile, user: CoomerAPIUser) {
+  // Normalize f.path to avoid protocol-relative or multiple-leading-slash paths
+  const normalizedPath = f.path.replace(/^\/+/, '/');
+  let url = '';
+  try {
+    url = new URL(normalizedPath, user.domain).toString();
+  } catch (_) {
+    // Fallback: join with a single slash
+    url = `${user.domain}/${normalizedPath.replace(/^\//, '')}`;
+  }
+  return url;
+}
+
 async function parseUser(url: string): Promise<CoomerAPIUser> {
   const [_, domain, service, id] = url.match(
     /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/,

package/src/api/gofile.ts

@@ -1,6 +1,6 @@
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file.js';
-import { setGlobalHeaders } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../services/file';
+import { setGlobalHeaders } from '../utils/requests';
 
 type GoFileAPIToken = { status: string; data: { token: string } };
 type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };

package/src/api/index.ts

@@ -1,4 +1,4 @@
-import type { CoomerFileList } from '../utils/file';
+import type { CoomerFileList } from '../services/file';
 import { getBunkrData } from './bunkr';
 import { getCoomerData } from './coomer-api';
 import { getGofileData } from './gofile';
@@ -11,15 +11,19 @@ export async function apiHandler(url_: string): Promise<CoomerFileList> {
   if (/^u\/\w+$/.test(url.origin)) {
     return getRedditData(url.href);
   }
+
   if (/coomer|kemono/.test(url.origin)) {
     return getCoomerData(url.href);
   }
+
   if (/bunkr/.test(url.origin)) {
     return getBunkrData(url.href);
   }
+
   if (/gofile\.io/.test(url.origin)) {
     return getGofileData(url.href);
   }
+
   if (/\.\w+/.test(url.pathname)) {
     return getPlainFileData(url.href);
   }

package/src/api/nsfw.xxx.ts

@@ -1,6 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -8,7 +8,6 @@ async function getUserPage(user: string, offset: number) {
 }
 
 async function getUserPosts(user: string): Promise<string[]> {
-  console.log('Fetching user posts...');
   const posts = [];
   for (let i = 1; i < 100000; i++) {
     const page = await getUserPage(user, i);
@@ -26,7 +25,6 @@ async function getUserPosts(user: string): Promise<string[]> {
 }
 
 async function getPostsData(posts: string[]): Promise<CoomerFileList> {
-  console.log('Fetching posts data...');
   const filelist = new CoomerFileList();
   for (const post of posts) {
     const page = await fetch(post).then((r) => r.text());
@@ -53,7 +51,9 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
 
 export async function getRedditData(url: string): Promise<CoomerFileList> {
   const user = url.match(/u\/(\w+)/)?.[1] as string;
+  console.log('Fetching user posts...');
   const posts = await getUserPosts(user);
+  console.log('Fetching posts data...');
   const filelist = await getPostsData(posts);
   filelist.dirName = `${user}-reddit`;
   return filelist;

package/src/api/plain-curl.ts

@@ -1,4 +1,4 @@
-import { CoomerFile, CoomerFileList } from '../utils/file';
+import { CoomerFile, CoomerFileList } from '../services/file';
 
 export async function getPlainFileData(url: string): Promise<CoomerFileList> {
   const name = url.split('/').pop() as string;

package/src/{args-handler.ts → cli/args-handler.ts}

@@ -1,16 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-type ArgumentHandlerResult = {
-  url: string;
-  dir: string;
-  media: string;
-  include: string;
-  exclude: string;
-  skip: number;
-};
-
-export function argumentHander(): ArgumentHandlerResult {
+export function argumentHander() {
   return yargs(hideBin(process.argv))
     .option('url', {
       alias: 'u',
@@ -26,8 +17,7 @@ export function argumentHander(): ArgumentHandlerResult {
     })
     .option('media', {
       type: 'string',
-      choices: ['video', 'image', 'all'],
-      default: 'all',
+      choices: ['video', 'image'],
       description:
         "The type of media to download: 'video', 'image', or 'all'. 'all' is the default.",
     })
@@ -41,11 +31,26 @@ export function argumentHander(): ArgumentHandlerResult {
       default: '',
       description: 'Filter file names by a comma-separated list of keywords to exclude',
     })
+    .option('min-size', {
+      type: 'string',
+      default: '',
+      description: 'Minimum file size to download. Example: "1mb" or "500kb"',
+    })
+    .option('max-size', {
+      type: 'string',
+      default: '',
+      description: 'Maximum file size to download. Example: "1mb" or "500kb"',
+    })
     .option('skip', {
       type: 'number',
       default: 0,
       description: 'Skips the first N files in the download queue',
     })
+    .option('remove-dupilicates', {
+      type: 'boolean',
+      default: true,
+      description: 'removes duplicates by url and file hash',
+    })
     .help()
     .alias('help', 'h')
     .parseSync();

package/src/cli/ui/app.tsx

@@ -0,0 +1,40 @@
+import { Box } from 'ink';
+import React from 'react';
+import { CoomerFileList } from '../../services/file';
+import { FileBox, FileListStateBox, KeyboardControlsInfo, Loading, TitleBar } from './components';
+import { useDownloaderHook } from './hooks/downloader';
+import { useInputHook } from './hooks/input';
+import { useInkStore } from './store';
+
+export function App() {
+  useInputHook();
+  useDownloaderHook();
+
+  const downloader = useInkStore((state) => state.downloader);
+  const filelist = downloader?.filelist;
+  const isFilelist = filelist instanceof CoomerFileList;
+
+  return (
+    <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
+      <TitleBar />
+      {!isFilelist ? (
+        <Loading />
+      ) : (
+        <>
+          <Box>
+            <Box>
+              <FileListStateBox filelist={filelist} />
+            </Box>
+            <Box flexBasis={30}>
+              <KeyboardControlsInfo />
+            </Box>
+          </Box>
+
+          {filelist.getActiveFiles().map((file) => {
+            return <FileBox file={file} key={file.name} />;
+          })}
+        </>
+      )}
+    </Box>
+  );
+}

package/src/cli/ui/components/file.tsx

@@ -0,0 +1,44 @@
+import { Box, Spacer, Text } from 'ink';
+import React from 'react';
+import type { CoomerFile } from '../../../services/file';
+import { b2mb } from '../../../utils/strings';
+import { Preview } from './preview';
+import { Spinner } from './spinner';
+
+interface FileBoxProps {
+  file: CoomerFile;
+}
+
+export function FileBox({ file }: FileBoxProps) {
+  const percentage = Number((file.downloaded / (file.size as number)) * 100).toFixed(2);
+  return (
+    <>
+      <Box
+        borderStyle="single"
+        borderColor="magentaBright"
+        borderDimColor
+        paddingX={1}
+        flexDirection="column"
+      >
+        <Box>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {file.name}
+          </Text>
+        </Box>
+        <Box flexDirection="row-reverse">
+          <Text color="cyan" dimColor>
+            {b2mb(file.downloaded)}/{file.size ? b2mb(file.size) : '∞'} MB
+          </Text>
+          <Text color="redBright" dimColor>
+            {file.size ? `  ${percentage}% ` : ''}
+          </Text>
+          <Spacer />
+          <Text color={'green'} dimColor>
+            <Spinner></Spinner>
+          </Text>
+        </Box>
+      </Box>
+      <Preview file={file} />
+    </>
+  );
+}

package/src/cli/ui/components/filelist.tsx

@@ -0,0 +1,52 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import type { CoomerFileList } from '../../../services/file';
+
+interface FileListStateBoxProps {
+  filelist: CoomerFileList;
+}
+
+export function FileListStateBox({ filelist }: FileListStateBoxProps) {
+  return (
+    <Box
+      paddingX={1}
+      flexDirection="column"
+      borderStyle={'single'}
+      borderColor={'magenta'}
+      borderDimColor
+    >
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Found:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist.files.length}
+        </Text>
+      </Box>
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Downloaded:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist.getDownloaded().length}
+        </Text>
+      </Box>
+      <Box>
+        <Box width={9}>
+          <Text color="cyanBright" dimColor>
+            Folder:
+          </Text>
+        </Box>
+        <Box flexGrow={1}>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {filelist.dirPath}
+          </Text>
+        </Box>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/index.ts

@@ -0,0 +1,6 @@
+export { FileBox } from './file';
+export { FileListStateBox } from './filelist';
+export { KeyboardControlsInfo } from './keyboardinfo';
+export { Loading } from './loading';
+export { Spinner } from './spinner';
+export { TitleBar } from './titlebar';

package/src/cli/ui/components/keyboardinfo.tsx

@@ -0,0 +1,41 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+
+const info = {
+  's ': 'skip current file',
+  p: 'on/off image preview',
+};
+
+export function KeyboardControlsInfo() {
+  const infoRender = Object.entries(info).map(([key, value]) => {
+    return (
+      <Box key={key}>
+        <Box marginRight={2}>
+          <Text color={'red'} dimColor bold>
+            {key}
+          </Text>
+        </Box>
+        <Text dimColor bold={false}>
+          {value}
+        </Text>
+      </Box>
+    );
+  });
+
+  return (
+    <Box
+      flexDirection="column"
+      paddingX={1}
+      borderStyle={'single'}
+      borderColor={'gray'}
+      borderDimColor
+    >
+      <Box>
+        <Text color={'red'} dimColor bold>
+          Keyboard controls:
+        </Text>
+      </Box>
+      {infoRender}
+    </Box>
+  );
+}

package/src/cli/ui/components/loading.tsx

@@ -0,0 +1,20 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import { Spinner } from './spinner';
+
+export function Loading() {
+  return (
+    <Box paddingX={1} borderDimColor flexDirection="column">
+      <Box alignSelf="center">
+        <Text dimColor color={'redBright'}>
+          Fetching Data
+        </Text>
+      </Box>
+      <Box alignSelf="center">
+        <Text color={'blueBright'} dimColor>
+          <Spinner type="grenade"></Spinner>
+        </Text>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/preview.tsx

@@ -0,0 +1,32 @@
+import { Box } from 'ink';
+import Image, { TerminalInfoProvider } from 'ink-picture';
+import React from 'react';
+import type { CoomerFile } from '../../../services/file';
+import { isImage } from '../../../utils/mediatypes';
+import { useInkStore } from '../store';
+
+interface PreviewProps {
+  file: CoomerFile;
+}
+
+export function Preview({ file }: PreviewProps) {
+  const previewEnabled = useInkStore((state) => state.preview);
+  const bigEnough = file.downloaded > 50 * 1024;
+  const shouldShow = previewEnabled && bigEnough && isImage(file.filepath as string);
+  const imgInfo = `
+    can't read partial images yet...
+    actual size: ${file.size}}
+    downloaded: ${file.downloaded}}
+  `;
+  return (
+    shouldShow && (
+      <Box paddingX={1}>
+        <TerminalInfoProvider>
+          <Box width={30} height={15}>
+            <Image src={file.filepath as string} alt={imgInfo} />
+          </Box>
+        </TerminalInfoProvider>
+      </Box>
+    )
+  );
+}

package/src/cli/ui/components/spinner.tsx

@@ -0,0 +1,28 @@
+import type { SpinnerName } from 'cli-spinners';
+import spinners from 'cli-spinners';
+import { Text } from 'ink';
+import React, { useEffect, useState } from 'react';
+
+interface SpinnerProps {
+  type?: SpinnerName;
+}
+
+export function Spinner({ type = 'dots' }: SpinnerProps) {
+  const spinner = spinners[type];
+  const randomFrame = (spinner.frames.length * Math.random()) | 0;
+  const [frame, setFrame] = useState(randomFrame);
+
+  useEffect(() => {
+    const timer = setInterval(() => {
+      setFrame((previousFrame) => {
+        return (previousFrame + 1) % spinner.frames.length;
+      });
+    }, spinner.interval);
+
+    return () => {
+      clearInterval(timer);
+    };
+  }, [spinner]);
+
+  return <Text>{spinner.frames[frame]}</Text>;
+}

package/src/cli/ui/components/titlebar.tsx

@@ -0,0 +1,15 @@
+import { Box, Spacer, Text } from 'ink';
+import React from 'react';
+import { version } from '../../../../package.json';
+
+export function TitleBar() {
+  return (
+    <Box>
+      <Spacer></Spacer>
+      <Box borderColor={'magenta'} borderStyle={'arrow'}>
+        <Text color={'cyanBright'}>Coomer-Downloader {version}</Text>
+      </Box>
+      <Spacer></Spacer>
+    </Box>
+  );
+}

package/src/cli/ui/hooks/downloader.ts

@@ -0,0 +1,21 @@
+import { useEffect, useState } from 'react';
+import { useInkStore } from '../store';
+
+export const useDownloaderHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const filelist = downloader?.filelist;
+
+  const [_, setHelper] = useState(0);
+
+  useEffect(() => {
+    downloader?.subject.subscribe(({ type }) => {
+      if (
+        type === 'FILE_DOWNLOADING_START' ||
+        type === 'FILE_DOWNLOADING_END' ||
+        type === 'CHUNK_DOWNLOADING_UPDATE'
+      ) {
+        setHelper(Date.now());
+      }
+    });
+  });
+};

package/src/cli/ui/hooks/input.ts

@@ -0,0 +1,17 @@
+import { useInput } from 'ink';
+import { useInkStore } from '../store';
+
+// problems with tsx watch
+export const useInputHook = () => {
+  const downloader = useInkStore((state) => state.downloader);
+  const switchPreview = useInkStore((state) => state.switchPreview);
+
+  useInput((input) => {
+    if (input === 's') {
+      downloader?.skip();
+    }
+    if (input === 'p') {
+      switchPreview();
+    }
+  });
+};

package/src/cli/ui/index.tsx

@@ -0,0 +1,7 @@
+import { render } from 'ink';
+import React from 'react';
+import { App } from './app';
+
+export function createReactInk() {
+  return render(<App />);
+}

package/src/cli/ui/store/index.ts

@@ -0,0 +1,19 @@
+import { create } from 'zustand';
+import type { Downloader } from '../../../services/downloader';
+
+interface InkState {
+  preview: boolean;
+  switchPreview: () => void;
+  downloader?: Downloader;
+  setDownloader: (downloader: Downloader) => void;
+}
+
+export const useInkStore = create<InkState>((set) => ({
+  preview: false,
+  switchPreview: () =>
+    set((state) => ({
+      preview: !state.preview,
+    })),
+  downloader: undefined,
+  setDownloader: (downloader: Downloader) => set({ downloader }),
+}));