coomer-downloader 3.4.1 → 3.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.4.1",
+  "version": "3.4.2",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",

package/src/api/index.ts

@@ -1,32 +1,6 @@
-import type { CoomerFileList } from '../services/filelist';
-import { getBunkrData } from './bunkr';
-import { getCoomerData } from './coomer-api';
-import { getGofileData } from './gofile';
-import { getRedditData } from './nsfw.xxx';
-import { getPlainFileData } from './plain-curl';
-
-export async function apiHandler(url_: string): Promise<CoomerFileList> {
-  const url = new URL(url_);
-
-  if (/^u\/\w+$/.test(url.origin)) {
-    return getRedditData(url.href);
-  }
-
-  if (/coomer|kemono/.test(url.origin)) {
-    return getCoomerData(url.href);
-  }
-
-  if (/bunkr/.test(url.origin)) {
-    return getBunkrData(url.href);
-  }
-
-  if (/gofile\.io/.test(url.origin)) {
-    return getGofileData(url.href);
-  }
-
-  if (/\.\w+/.test(url.pathname)) {
-    return getPlainFileData(url.href);
-  }
-
-  throw Error('Invalid URL');
-}
+export { BunkrAPI } from './providers/bunkr';
+export { CoomerAPI } from './providers/coomer';
+export { GofileAPI } from './providers/gofile';
+export { PlainFileAPI } from './providers/plainfile';
+export { RedditAPI } from './providers/reddit';
+export { resolveAPI } from './resolver';

package/src/api/provider.ts

@@ -0,0 +1,7 @@
+import type { CoomerFileList } from '../core/filelist';
+
+export interface ProviderAPI {
+  fixURL?(url: string, retries: number): string;
+  testURL(url: URL): boolean;
+  getData(url: string): Promise<CoomerFileList>;
+}

package/src/api/{bunkr.ts → providers/bunkr.ts}

@@ -1,7 +1,8 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile } from '../services/file';
-import { CoomerFileList } from '../services/filelist';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import type { ProviderAPI } from '../provider';
 
 type EncData = { url: string; timestamp: number };
 
@@ -64,7 +65,13 @@ async function getGalleryFiles(url: string): Promise<CoomerFileList> {
   return filelist;
 }
 
-export async function getBunkrData(url: string): Promise<CoomerFileList> {
-  const filelist = await getGalleryFiles(url);
-  return filelist;
+export class BunkrAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /bunkr/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const filelist = await getGalleryFiles(url);
+    return filelist;
+  }
 }

package/src/api/{coomer-api.ts → providers/coomer.ts}

@@ -1,34 +1,27 @@
-import { CoomerFile } from '../services/file';
-import { CoomerFileList } from '../services/filelist';
-import { isImage } from '../utils/mediatypes';
-import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
-
-type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
-type CoomerAPIUserData = { name: string };
-type CoomerAPIFile = { path: string; name: string };
-type CoomerAPIPost = {
-  title: string;
-  content: string;
-  published: string;
-  attachments: CoomerAPIFile[];
-  file: CoomerAPIFile;
-};
-
-const SERVERS = ['n1', 'n2', 'n3', 'n4'];
-
-export function tryFixCoomerUrl(url: string, attempts: number) {
-  if (attempts < 2 && isImage(url)) {
-    return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
-  }
-  const server = url.match(/n\d\./)?.[0].slice(0, 2) as string;
-  const i = SERVERS.indexOf(server);
-  if (i !== -1) {
-    const newServer = SERVERS[(i + 1) % SERVERS.length];
-    return url.replace(/n\d./, `${newServer}.`);
-  }
-  return url;
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import { isImage } from '../../utils/mediatypes';
+import { fetchWithGlobalHeader, setGlobalHeaders } from '../../utils/requests';
+import type { ProviderAPI } from '../provider';
+
+interface CoomerServiceAPI {
+  user: { domain: string; service: string; id: string; name?: string };
+  userData: { name: string };
+  file: { path: string; name: string };
+  post: {
+    title: string;
+    content: string;
+    published: string;
+    attachments: CoomerServiceAPI['file'][];
+    file: CoomerServiceAPI['file'];
+  };
 }
 
+type CoomerAPIUser = CoomerServiceAPI['user'];
+type CoomerAPIUserData = CoomerServiceAPI['userData'];
+type CoomerAPIFile = CoomerServiceAPI['file'];
+type CoomerAPIPost = CoomerServiceAPI['post'];
+
 async function getUserProfileData(user: CoomerAPIUser): Promise<CoomerAPIUserData> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
   const result = await fetchWithGlobalHeader(url).then((r) => r.json());
@@ -101,10 +94,31 @@ async function parseUser(url: string): Promise<CoomerAPIUser> {
   return { domain, service, id, name };
 }
 
-export async function getCoomerData(url: string): Promise<CoomerFileList> {
-  setGlobalHeaders({ accept: 'text/css' });
-  const user = await parseUser(url);
-  const filelist = await getUserFiles(user);
-  filelist.dirName = `${user.name}-${user.service}`;
-  return filelist;
+export class CoomerAPI implements ProviderAPI {
+  private static readonly SERVERS = ['n1', 'n2', 'n3', 'n4'];
+
+  public fixURL(url: string, retries: number): string {
+    if (retries < 2 && isImage(url)) {
+      return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
+    }
+    const server = url.match(/n\d\./)?.[0].slice(0, 2) as string;
+    const i = CoomerAPI.SERVERS.indexOf(server);
+    if (i !== -1) {
+      const newServer = CoomerAPI.SERVERS[(i + 1) % CoomerAPI.SERVERS.length];
+      return url.replace(/n\d./, `${newServer}.`);
+    }
+    return url;
+  }
+
+  public testURL(url: URL) {
+    return /coomer|kemono/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    setGlobalHeaders({ accept: 'text/css' });
+    const user = await parseUser(url);
+    const filelist = await getUserFiles(user);
+    filelist.dirName = `${user.name}-${user.service}`;
+    return filelist;
+  }
 }

package/src/api/{gofile.ts → providers/gofile.ts}

@@ -1,7 +1,8 @@
 import { fetch } from 'undici';
-import { CoomerFile } from '../services/file';
-import { CoomerFileList } from '../services/filelist';
-import { setGlobalHeaders } from '../utils/requests';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import { setGlobalHeaders } from '../../utils/requests';
+import type { ProviderAPI } from '../provider';
 
 type GoFileAPIToken = { status: string; data: { token: string } };
 type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };
@@ -14,7 +15,8 @@ async function getToken(): Promise<string> {
   if (data.status === 'ok') {
     return data.data.token;
   }
-  throw new Error('cannot get token');
+
+  throw new Error('Token Not Found');
 }
 
 async function getWebsiteToken() {
@@ -54,16 +56,22 @@ async function getFolderFiles(
   return new CoomerFileList(files);
 }
 
-export async function getGofileData(url: string): Promise<CoomerFileList> {
-  const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
+export class GofileAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /gofile\.io/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
 
-  const token = await getToken();
-  const websiteToken = await getWebsiteToken();
+    const token = await getToken();
+    const websiteToken = await getWebsiteToken();
 
-  const filelist = await getFolderFiles(id, token, websiteToken);
-  filelist.dirName = `gofile-${id}`;
+    const filelist = await getFolderFiles(id, token, websiteToken);
+    filelist.dirName = `gofile-${id}`;
 
-  setGlobalHeaders({ Cookie: `accountToken=${token}` });
+    setGlobalHeaders({ Cookie: `accountToken=${token}` });
 
-  return filelist;
+    return filelist;
+  }
 }

package/src/api/providers/plainfile.ts

@@ -0,0 +1,17 @@
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import type { ProviderAPI } from '../provider';
+
+export class PlainFileAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /\.\w+/.test(url.pathname);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const name = url.split('/').pop() as string;
+    const file = CoomerFile.from({ name, url });
+    const filelist = new CoomerFileList([file]);
+    filelist.dirName = '';
+    return filelist;
+  }
+}

package/src/api/{nsfw.xxx.ts → providers/reddit.ts}

@@ -1,7 +1,9 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile } from '../services/file';
-import { CoomerFileList } from '../services/filelist';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import logger from '../../utils/logger';
+import type { ProviderAPI } from '../provider';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -20,6 +22,7 @@ async function getUserPosts(user: string): Promise<string[]> {
       .get()
       .filter((href) => href?.startsWith('https://nsfw.xxx/post'));
 
+    logger.debug({ count: posts.length });
     posts.push(...newPosts);
   }
   return posts;
@@ -45,18 +48,23 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;
 
+    logger.debug({ hehe: filelist.files.length, src });
     filelist.files.push(CoomerFile.from({ name, url: src }));
   }
 
   return filelist;
 }
 
-export async function getRedditData(url: string): Promise<CoomerFileList> {
-  const user = url.match(/u\/(\w+)/)?.[1] as string;
-  console.log('Fetching user posts...');
-  const posts = await getUserPosts(user);
-  console.log('Fetching posts data...');
-  const filelist = await getPostsData(posts);
-  filelist.dirName = `${user}-reddit`;
-  return filelist;
+export class RedditAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /^\/user\/[\w-]+$/.test(url.pathname);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const user = url.match(/^\/user\/([\w-]+)/)?.[1] as string;
+    const posts = await getUserPosts(user);
+    const filelist = await getPostsData(posts);
+    filelist.dirName = `${user}-reddit`;
+    return filelist;
+  }
 }

package/src/api/resolver.ts

@@ -0,0 +1,23 @@
+import type { CoomerFileList } from '../core/filelist';
+import { BunkrAPI } from './providers/bunkr';
+import { CoomerAPI } from './providers/coomer';
+import { GofileAPI } from './providers/gofile';
+import { PlainFileAPI } from './providers/plainfile';
+import { RedditAPI } from './providers/reddit';
+
+const providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
+
+export async function resolveAPI(url_: string): Promise<CoomerFileList> {
+  const url = new URL(url_);
+
+  for (const p of providers) {
+    const provider = new p();
+    if (provider.testURL(url)) {
+      const filelist = await provider.getData(url.toString());
+      filelist.provider = provider;
+      return filelist;
+    }
+  }
+
+  throw Error('Invalid URL');
+}

package/src/cli/ui/app.tsx

@@ -1,8 +1,8 @@
 import { Box } from 'ink';
 import React from 'react';
-import { CoomerFileList } from '../../services/filelist';
+import { CoomerFileList } from '../../core';
 import {
-  FileBox,
+  FileListBox,
   FileListStateBox,
   KeyboardControlsInfo,
   Loading,
@@ -13,7 +13,7 @@ import { useInputHook } from './hooks/input';
 
 export function App() {
   useInputHook();
-  const filelist = useDownloaderHook();
+  const filelist = useDownloaderHook(['FILES_DOWNLOADING_START']);
 
   return (
     <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
@@ -24,16 +24,14 @@ export function App() {
         <>
           <Box>
             <Box>
-              <FileListStateBox filelist={filelist} />
+              <FileListStateBox />
             </Box>
             <Box flexBasis={30}>
               <KeyboardControlsInfo />
             </Box>
           </Box>
 
-          {filelist?.getActiveFiles().map((file) => {
-            return <FileBox file={file} key={file.name} />;
-          })}
+          <FileListBox />
         </>
       )}
     </Box>

package/src/cli/ui/components/file.tsx

@@ -1,7 +1,8 @@
 import { Box, Spacer, Text } from 'ink';
 import React from 'react';
-import type { CoomerFile } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
 import { b2mb } from '../../../utils/strings';
+import { useDownloaderHook } from '../hooks/downloader';
 import { Preview } from './preview';
 import { Spinner } from './spinner';
 
@@ -9,8 +10,11 @@ interface FileBoxProps {
   file: CoomerFile;
 }
 
-export function FileBox({ file }: FileBoxProps) {
+export const FileBox = React.memo(({ file }: FileBoxProps) => {
+  useDownloaderHook(['CHUNK_DOWNLOADING_UPDATE']);
+
   const percentage = Number((file.downloaded / (file.size as number)) * 100).toFixed(2);
+
   return (
     <>
       <Box
@@ -41,4 +45,4 @@ export function FileBox({ file }: FileBoxProps) {
       <Preview file={file} />
     </>
   );
-}
+});

package/src/cli/ui/components/filelist-state.tsx

@@ -0,0 +1,48 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import { useDownloaderHook } from '../hooks/downloader';
+
+export function FileListStateBox() {
+  const filelist = useDownloaderHook(['FILE_DOWNLOADING_START', 'FILE_DOWNLOADING_END']);
+
+  return (
+    <Box
+      paddingX={1}
+      flexDirection="column"
+      borderStyle={'single'}
+      borderColor={'magenta'}
+      borderDimColor
+    >
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Found:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist?.files.length}
+        </Text>
+      </Box>
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Downloaded:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap"></Text>
+      </Box>
+      <Box>
+        <Box width={9}>
+          <Text color="cyanBright" dimColor>
+            Folder:
+          </Text>
+        </Box>
+        <Box flexGrow={1}>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {filelist?.dirPath}
+          </Text>
+        </Box>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/filelist.tsx

@@ -1,52 +1,16 @@
-import { Box, Text } from 'ink';
 import React from 'react';
-import type { CoomerFileList } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
+import { useDownloaderHook } from '../hooks/downloader';
+import { FileBox } from './file';
 
-interface FileListStateBoxProps {
-  filelist: CoomerFileList;
-}
+export function FileListBox() {
+  const filelist = useDownloaderHook(['FILE_DOWNLOADING_START', 'FILE_DOWNLOADING_END']);
 
-export function FileListStateBox({ filelist }: FileListStateBoxProps) {
   return (
-    <Box
-      paddingX={1}
-      flexDirection="column"
-      borderStyle={'single'}
-      borderColor={'magenta'}
-      borderDimColor
-    >
-      <Box>
-        <Box marginRight={1}>
-          <Text color="cyanBright" dimColor>
-            Found:
-          </Text>
-        </Box>
-        <Text color="blue" dimColor wrap="wrap">
-          {filelist.files.length}
-        </Text>
-      </Box>
-      <Box>
-        <Box marginRight={1}>
-          <Text color="cyanBright" dimColor>
-            Downloaded:
-          </Text>
-        </Box>
-        <Text color="blue" dimColor wrap="wrap">
-          {filelist.getDownloaded().length}
-        </Text>
-      </Box>
-      <Box>
-        <Box width={9}>
-          <Text color="cyanBright" dimColor>
-            Folder:
-          </Text>
-        </Box>
-        <Box flexGrow={1}>
-          <Text color="blue" dimColor wrap="truncate-middle">
-            {filelist.dirPath}
-          </Text>
-        </Box>
-      </Box>
-    </Box>
+    <>
+      {filelist?.getActiveFiles().map((file: CoomerFile) => {
+        return <FileBox file={file} key={file.name} />;
+      })}
+    </>
   );
 }

package/src/cli/ui/components/index.ts

@@ -1,5 +1,6 @@
 export { FileBox } from './file';
-export { FileListStateBox } from './filelist';
+export { FileListBox } from './filelist';
+export { FileListStateBox } from './filelist-state';
 export { KeyboardControlsInfo } from './keyboardinfo';
 export { Loading } from './loading';
 export { Spinner } from './spinner';

package/src/cli/ui/components/preview.tsx

@@ -1,7 +1,7 @@
 import { Box } from 'ink';
 import Image, { TerminalInfoProvider } from 'ink-picture';
 import React from 'react';
-import type { CoomerFile } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
 import { isImage } from '../../../utils/mediatypes';
 import { useInkStore } from '../store';
 

package/src/cli/ui/hooks/downloader.ts

@@ -1,7 +1,8 @@
 import { useRef, useSyncExternalStore } from 'react';
+import type { DownloaderSubject, DownloaderSubjectSignal } from '../../../types';
 import { useInkStore } from '../store';
 
-export const useDownloaderHook = () => {
+export const useDownloaderHook = (subjectEvents: DownloaderSubjectSignal[]) => {
   const downloader = useInkStore((state) => state.downloader);
 
   const versionRef = useRef(0);
@@ -10,14 +11,8 @@ export const useDownloaderHook = () => {
     (onStoreChange) => {
       if (!downloader) return () => {};
 
-      const sub = downloader.subject.subscribe(({ type }) => {
-        const targets = [
-          'FILE_DOWNLOADING_START',
-          'FILE_DOWNLOADING_END',
-          'CHUNK_DOWNLOADING_UPDATE',
-        ];
-
-        if (targets.includes(type)) {
+      const sub = downloader.subject.subscribe(({ type }: DownloaderSubject) => {
+        if (subjectEvents.includes(type)) {
           versionRef.current++;
           onStoreChange();
         }

package/src/cli/ui/hooks/input.ts

@@ -1,7 +1,6 @@
 import { useInput } from 'ink';
 import { useInkStore } from '../store';
 
-// problems with tsx watch
 export const useInputHook = () => {
   const downloader = useInkStore((state) => state.downloader);
   const switchPreview = useInkStore((state) => state.switchPreview);

package/src/cli/ui/store/index.ts

@@ -1,5 +1,5 @@
 import { create } from 'zustand';
-import type { Downloader } from '../../../services/downloader';
+import type { Downloader } from '../../../core';
 
 interface InkState {
   preview: boolean;

package/src/{services → core}/downloader.ts

@@ -2,7 +2,6 @@ import fs from 'node:fs';
 import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
-import { tryFixCoomerUrl } from '../api/coomer-api';
 import type { AbortControllerSubject, DownloaderSubject } from '../types';
 import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
@@ -35,7 +34,7 @@ export class Downloader {
     this.setAbortControllerListener();
   }
 
-  async fetchStream(
+  private async fetchStream(
     file: CoomerFile,
     stream: Readable,
     sizeOld = 0,
@@ -100,7 +99,10 @@ export class Downloader {
     }
   }
 
-  async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
+  public async downloadFile(
+    file: CoomerFile,
+    retries = this.fetchRetries,
+  ): Promise<void> {
     const signal = this.abortController.signal;
     try {
       file.downloaded = await getFileSize(file.filepath as string);
@@ -130,8 +132,8 @@ export class Downloader {
         if (signal.reason === 'FILE_SKIP') return;
       }
       if (retries > 0) {
-        if (/coomer|kemono/.test(file.url)) {
-          file.url = tryFixCoomerUrl(file.url, retries);
+        if (this.filelist.provider?.fixURL) {
+          file.url = this.filelist.provider.fixURL(file.url, retries);
         }
         await sleep(1000);
         return await this.downloadFile(file, retries - 1);
@@ -140,7 +142,7 @@ export class Downloader {
     }
   }
 
-  async downloadFiles(): Promise<void> {
+  public async downloadFiles(): Promise<void> {
     mkdir(this.filelist.dirPath as string);
 
     this.subject.next({ type: 'FILES_DOWNLOADING_START' });

package/src/{services → core}/filelist.ts

@@ -1,6 +1,6 @@
 import os from 'node:os';
 import path from 'node:path';
-import logger from '../logger';
+import type { ProviderAPI } from '../api/provider';
 import type { MediaType } from '../types';
 import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
 import { filterString } from '../utils/filters';
@@ -11,6 +11,7 @@ import type { CoomerFile } from './file';
 export class CoomerFileList {
   public dirPath?: string;
   public dirName?: string;
+  public provider?: ProviderAPI;
 
   constructor(public files: CoomerFile[] = []) {}
 

package/src/core/index.ts

@@ -0,0 +1,3 @@
+export { Downloader } from './downloader';
+export { CoomerFile } from './file';
+export { CoomerFileList } from './filelist';