coomer-downloader 3.4.0 → 3.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.4.0",
+  "version": "3.4.2",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",

package/src/api/index.ts

@@ -1,32 +1,6 @@
-import type { CoomerFileList } from '../services/file';
-import { getBunkrData } from './bunkr';
-import { getCoomerData } from './coomer-api';
-import { getGofileData } from './gofile';
-import { getRedditData } from './nsfw.xxx';
-import { getPlainFileData } from './plain-curl';
-
-export async function apiHandler(url_: string): Promise<CoomerFileList> {
-  const url = new URL(url_);
-
-  if (/^u\/\w+$/.test(url.origin)) {
-    return getRedditData(url.href);
-  }
-
-  if (/coomer|kemono/.test(url.origin)) {
-    return getCoomerData(url.href);
-  }
-
-  if (/bunkr/.test(url.origin)) {
-    return getBunkrData(url.href);
-  }
-
-  if (/gofile\.io/.test(url.origin)) {
-    return getGofileData(url.href);
-  }
-
-  if (/\.\w+/.test(url.pathname)) {
-    return getPlainFileData(url.href);
-  }
-
-  throw Error('Invalid URL');
-}
+export { BunkrAPI } from './providers/bunkr';
+export { CoomerAPI } from './providers/coomer';
+export { GofileAPI } from './providers/gofile';
+export { PlainFileAPI } from './providers/plainfile';
+export { RedditAPI } from './providers/reddit';
+export { resolveAPI } from './resolver';

package/src/api/provider.ts

@@ -0,0 +1,7 @@
+import type { CoomerFileList } from '../core/filelist';
+
+export interface ProviderAPI {
+  fixURL?(url: string, retries: number): string;
+  testURL(url: URL): boolean;
+  getData(url: string): Promise<CoomerFileList>;
+}

package/src/api/{bunkr.ts → providers/bunkr.ts}

@@ -1,6 +1,8 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import type { ProviderAPI } from '../provider';
 
 type EncData = { url: string; timestamp: number };
 
@@ -18,7 +20,9 @@ function decryptEncryptedUrl(encryptionData: EncData) {
   const encryptedUrlBuffer = Buffer.from(encryptionData.url, 'base64');
   const secretKeyBuffer = Buffer.from(secretKey, 'utf-8');
   return Array.from(encryptedUrlBuffer)
-    .map((byte, i) => String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]))
+    .map((byte, i) =>
+      String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]),
+    )
     .join('');
 }
 
@@ -61,7 +65,13 @@ async function getGalleryFiles(url: string): Promise<CoomerFileList> {
   return filelist;
 }
 
-export async function getBunkrData(url: string): Promise<CoomerFileList> {
-  const filelist = await getGalleryFiles(url);
-  return filelist;
+export class BunkrAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /bunkr/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const filelist = await getGalleryFiles(url);
+    return filelist;
+  }
 }

package/src/api/{coomer-api.ts → providers/coomer.ts}

@@ -1,33 +1,27 @@
-import { CoomerFile, CoomerFileList } from '../services/file';
-import { isImage } from '../utils/mediatypes';
-import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
-
-type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
-type CoomerAPIUserData = { name: string };
-type CoomerAPIFile = { path: string; name: string };
-type CoomerAPIPost = {
-  title: string;
-  content: string;
-  published: string;
-  attachments: CoomerAPIFile[];
-  file: CoomerAPIFile;
-};
-
-const SERVERS = ['n1', 'n2', 'n3', 'n4'];
-
-export function tryFixCoomerUrl(url: string, attempts: number) {
-  if (attempts < 2 && isImage(url)) {
-    return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
-  }
-  const server = url.match(/n\d\./)?.[0].slice(0, 2) as string;
-  const i = SERVERS.indexOf(server);
-  if (i !== -1) {
-    const newServer = SERVERS[(i + 1) % SERVERS.length];
-    return url.replace(/n\d./, `${newServer}.`);
-  }
-  return url;
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import { isImage } from '../../utils/mediatypes';
+import { fetchWithGlobalHeader, setGlobalHeaders } from '../../utils/requests';
+import type { ProviderAPI } from '../provider';
+
+interface CoomerServiceAPI {
+  user: { domain: string; service: string; id: string; name?: string };
+  userData: { name: string };
+  file: { path: string; name: string };
+  post: {
+    title: string;
+    content: string;
+    published: string;
+    attachments: CoomerServiceAPI['file'][];
+    file: CoomerServiceAPI['file'];
+  };
 }
 
+type CoomerAPIUser = CoomerServiceAPI['user'];
+type CoomerAPIUserData = CoomerServiceAPI['userData'];
+type CoomerAPIFile = CoomerServiceAPI['file'];
+type CoomerAPIPost = CoomerServiceAPI['post'];
+
 async function getUserProfileData(user: CoomerAPIUser): Promise<CoomerAPIUserData> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
   const result = await fetchWithGlobalHeader(url).then((r) => r.json());
@@ -100,10 +94,31 @@ async function parseUser(url: string): Promise<CoomerAPIUser> {
   return { domain, service, id, name };
 }
 
-export async function getCoomerData(url: string): Promise<CoomerFileList> {
-  setGlobalHeaders({ accept: 'text/css' });
-  const user = await parseUser(url);
-  const filelist = await getUserFiles(user);
-  filelist.dirName = `${user.name}-${user.service}`;
-  return filelist;
+export class CoomerAPI implements ProviderAPI {
+  private static readonly SERVERS = ['n1', 'n2', 'n3', 'n4'];
+
+  public fixURL(url: string, retries: number): string {
+    if (retries < 2 && isImage(url)) {
+      return url.replace(/\/data\//, '/thumbnail/data/').replace(/n\d\./, 'img.');
+    }
+    const server = url.match(/n\d\./)?.[0].slice(0, 2) as string;
+    const i = CoomerAPI.SERVERS.indexOf(server);
+    if (i !== -1) {
+      const newServer = CoomerAPI.SERVERS[(i + 1) % CoomerAPI.SERVERS.length];
+      return url.replace(/n\d./, `${newServer}.`);
+    }
+    return url;
+  }
+
+  public testURL(url: URL) {
+    return /coomer|kemono/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    setGlobalHeaders({ accept: 'text/css' });
+    const user = await parseUser(url);
+    const filelist = await getUserFiles(user);
+    filelist.dirName = `${user.name}-${user.service}`;
+    return filelist;
+  }
 }

package/src/api/{gofile.ts → providers/gofile.ts}

@@ -1,6 +1,8 @@
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
-import { setGlobalHeaders } from '../utils/requests';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import { setGlobalHeaders } from '../../utils/requests';
+import type { ProviderAPI } from '../provider';
 
 type GoFileAPIToken = { status: string; data: { token: string } };
 type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };
@@ -13,7 +15,8 @@ async function getToken(): Promise<string> {
   if (data.status === 'ok') {
     return data.data.token;
   }
-  throw new Error('cannot get token');
+
+  throw new Error('Token Not Found');
 }
 
 async function getWebsiteToken() {
@@ -53,16 +56,22 @@ async function getFolderFiles(
   return new CoomerFileList(files);
 }
 
-export async function getGofileData(url: string): Promise<CoomerFileList> {
-  const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
+export class GofileAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /gofile\.io/.test(url.origin);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
 
-  const token = await getToken();
-  const websiteToken = await getWebsiteToken();
+    const token = await getToken();
+    const websiteToken = await getWebsiteToken();
 
-  const filelist = await getFolderFiles(id, token, websiteToken);
-  filelist.dirName = `gofile-${id}`;
+    const filelist = await getFolderFiles(id, token, websiteToken);
+    filelist.dirName = `gofile-${id}`;
 
-  setGlobalHeaders({ Cookie: `accountToken=${token}` });
+    setGlobalHeaders({ Cookie: `accountToken=${token}` });
 
-  return filelist;
+    return filelist;
+  }
 }

package/src/api/providers/plainfile.ts

@@ -0,0 +1,17 @@
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import type { ProviderAPI } from '../provider';
+
+export class PlainFileAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /\.\w+/.test(url.pathname);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const name = url.split('/').pop() as string;
+    const file = CoomerFile.from({ name, url });
+    const filelist = new CoomerFileList([file]);
+    filelist.dirName = '';
+    return filelist;
+  }
+}

package/src/api/{nsfw.xxx.ts → providers/reddit.ts}

@@ -1,6 +1,9 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../../core/file';
+import { CoomerFileList } from '../../core/filelist';
+import logger from '../../utils/logger';
+import type { ProviderAPI } from '../provider';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -19,6 +22,7 @@ async function getUserPosts(user: string): Promise<string[]> {
       .get()
       .filter((href) => href?.startsWith('https://nsfw.xxx/post'));
 
+    logger.debug({ count: posts.length });
     posts.push(...newPosts);
   }
   return posts;
@@ -38,23 +42,29 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
     if (!src) continue;
 
     const slug = post.split('post/')[1].split('?')[0];
-    const date = $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
+    const date =
+      $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
 
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;
 
+    logger.debug({ hehe: filelist.files.length, src });
     filelist.files.push(CoomerFile.from({ name, url: src }));
   }
 
   return filelist;
 }
 
-export async function getRedditData(url: string): Promise<CoomerFileList> {
-  const user = url.match(/u\/(\w+)/)?.[1] as string;
-  console.log('Fetching user posts...');
-  const posts = await getUserPosts(user);
-  console.log('Fetching posts data...');
-  const filelist = await getPostsData(posts);
-  filelist.dirName = `${user}-reddit`;
-  return filelist;
+export class RedditAPI implements ProviderAPI {
+  public testURL(url: URL) {
+    return /^\/user\/[\w-]+$/.test(url.pathname);
+  }
+
+  public async getData(url: string): Promise<CoomerFileList> {
+    const user = url.match(/^\/user\/([\w-]+)/)?.[1] as string;
+    const posts = await getUserPosts(user);
+    const filelist = await getPostsData(posts);
+    filelist.dirName = `${user}-reddit`;
+    return filelist;
+  }
 }

package/src/api/resolver.ts

@@ -0,0 +1,23 @@
+import type { CoomerFileList } from '../core/filelist';
+import { BunkrAPI } from './providers/bunkr';
+import { CoomerAPI } from './providers/coomer';
+import { GofileAPI } from './providers/gofile';
+import { PlainFileAPI } from './providers/plainfile';
+import { RedditAPI } from './providers/reddit';
+
+const providers = [RedditAPI, CoomerAPI, BunkrAPI, GofileAPI, PlainFileAPI];
+
+export async function resolveAPI(url_: string): Promise<CoomerFileList> {
+  const url = new URL(url_);
+
+  for (const p of providers) {
+    const provider = new p();
+    if (provider.testURL(url)) {
+      const filelist = await provider.getData(url.toString());
+      filelist.provider = provider;
+      return filelist;
+    }
+  }
+
+  throw Error('Invalid URL');
+}

package/src/cli/ui/app.tsx

@@ -1,38 +1,37 @@
 import { Box } from 'ink';
 import React from 'react';
-import { CoomerFileList } from '../../services/file';
-import { FileBox, FileListStateBox, KeyboardControlsInfo, Loading, TitleBar } from './components';
+import { CoomerFileList } from '../../core';
+import {
+  FileListBox,
+  FileListStateBox,
+  KeyboardControlsInfo,
+  Loading,
+  TitleBar,
+} from './components';
 import { useDownloaderHook } from './hooks/downloader';
 import { useInputHook } from './hooks/input';
-import { useInkStore } from './store';
 
 export function App() {
   useInputHook();
-  useDownloaderHook();
-
-  const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
-  const isFilelist = filelist instanceof CoomerFileList;
+  const filelist = useDownloaderHook(['FILES_DOWNLOADING_START']);
 
   return (
     <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
       <TitleBar />
-      {!isFilelist ? (
+      {!(filelist instanceof CoomerFileList) ? (
         <Loading />
       ) : (
         <>
           <Box>
             <Box>
-              <FileListStateBox filelist={filelist} />
+              <FileListStateBox />
             </Box>
             <Box flexBasis={30}>
               <KeyboardControlsInfo />
             </Box>
           </Box>
 
-          {filelist.getActiveFiles().map((file) => {
-            return <FileBox file={file} key={file.name} />;
-          })}
+          <FileListBox />
         </>
       )}
     </Box>

package/src/cli/ui/components/file.tsx

@@ -1,7 +1,8 @@
 import { Box, Spacer, Text } from 'ink';
 import React from 'react';
-import type { CoomerFile } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
 import { b2mb } from '../../../utils/strings';
+import { useDownloaderHook } from '../hooks/downloader';
 import { Preview } from './preview';
 import { Spinner } from './spinner';
 
@@ -9,8 +10,11 @@ interface FileBoxProps {
   file: CoomerFile;
 }
 
-export function FileBox({ file }: FileBoxProps) {
+export const FileBox = React.memo(({ file }: FileBoxProps) => {
+  useDownloaderHook(['CHUNK_DOWNLOADING_UPDATE']);
+
   const percentage = Number((file.downloaded / (file.size as number)) * 100).toFixed(2);
+
   return (
     <>
       <Box
@@ -41,4 +45,4 @@ export function FileBox({ file }: FileBoxProps) {
       <Preview file={file} />
     </>
   );
-}
+});

package/src/cli/ui/components/filelist-state.tsx

@@ -0,0 +1,48 @@
+import { Box, Text } from 'ink';
+import React from 'react';
+import { useDownloaderHook } from '../hooks/downloader';
+
+export function FileListStateBox() {
+  const filelist = useDownloaderHook(['FILE_DOWNLOADING_START', 'FILE_DOWNLOADING_END']);
+
+  return (
+    <Box
+      paddingX={1}
+      flexDirection="column"
+      borderStyle={'single'}
+      borderColor={'magenta'}
+      borderDimColor
+    >
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Found:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap">
+          {filelist?.files.length}
+        </Text>
+      </Box>
+      <Box>
+        <Box marginRight={1}>
+          <Text color="cyanBright" dimColor>
+            Downloaded:
+          </Text>
+        </Box>
+        <Text color="blue" dimColor wrap="wrap"></Text>
+      </Box>
+      <Box>
+        <Box width={9}>
+          <Text color="cyanBright" dimColor>
+            Folder:
+          </Text>
+        </Box>
+        <Box flexGrow={1}>
+          <Text color="blue" dimColor wrap="truncate-middle">
+            {filelist?.dirPath}
+          </Text>
+        </Box>
+      </Box>
+    </Box>
+  );
+}

package/src/cli/ui/components/filelist.tsx

@@ -1,52 +1,16 @@
-import { Box, Text } from 'ink';
 import React from 'react';
-import type { CoomerFileList } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
+import { useDownloaderHook } from '../hooks/downloader';
+import { FileBox } from './file';
 
-interface FileListStateBoxProps {
-  filelist: CoomerFileList;
-}
+export function FileListBox() {
+  const filelist = useDownloaderHook(['FILE_DOWNLOADING_START', 'FILE_DOWNLOADING_END']);
 
-export function FileListStateBox({ filelist }: FileListStateBoxProps) {
   return (
-    <Box
-      paddingX={1}
-      flexDirection="column"
-      borderStyle={'single'}
-      borderColor={'magenta'}
-      borderDimColor
-    >
-      <Box>
-        <Box marginRight={1}>
-          <Text color="cyanBright" dimColor>
-            Found:
-          </Text>
-        </Box>
-        <Text color="blue" dimColor wrap="wrap">
-          {filelist.files.length}
-        </Text>
-      </Box>
-      <Box>
-        <Box marginRight={1}>
-          <Text color="cyanBright" dimColor>
-            Downloaded:
-          </Text>
-        </Box>
-        <Text color="blue" dimColor wrap="wrap">
-          {filelist.getDownloaded().length}
-        </Text>
-      </Box>
-      <Box>
-        <Box width={9}>
-          <Text color="cyanBright" dimColor>
-            Folder:
-          </Text>
-        </Box>
-        <Box flexGrow={1}>
-          <Text color="blue" dimColor wrap="truncate-middle">
-            {filelist.dirPath}
-          </Text>
-        </Box>
-      </Box>
-    </Box>
+    <>
+      {filelist?.getActiveFiles().map((file: CoomerFile) => {
+        return <FileBox file={file} key={file.name} />;
+      })}
+    </>
   );
 }

package/src/cli/ui/components/index.ts

@@ -1,5 +1,6 @@
 export { FileBox } from './file';
-export { FileListStateBox } from './filelist';
+export { FileListBox } from './filelist';
+export { FileListStateBox } from './filelist-state';
 export { KeyboardControlsInfo } from './keyboardinfo';
 export { Loading } from './loading';
 export { Spinner } from './spinner';

package/src/cli/ui/components/preview.tsx

@@ -1,7 +1,7 @@
 import { Box } from 'ink';
 import Image, { TerminalInfoProvider } from 'ink-picture';
 import React from 'react';
-import type { CoomerFile } from '../../../services/file';
+import type { CoomerFile } from '../../../core';
 import { isImage } from '../../../utils/mediatypes';
 import { useInkStore } from '../store';
 

package/src/cli/ui/hooks/downloader.ts

@@ -1,21 +1,26 @@
-import { useEffect, useState } from 'react';
+import { useRef, useSyncExternalStore } from 'react';
+import type { DownloaderSubject, DownloaderSubjectSignal } from '../../../types';
 import { useInkStore } from '../store';
 
-export const useDownloaderHook = () => {
+export const useDownloaderHook = (subjectEvents: DownloaderSubjectSignal[]) => {
   const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
 
-  const [_, setHelper] = useState(0);
+  const versionRef = useRef(0);
 
-  useEffect(() => {
-    downloader?.subject.subscribe(({ type }) => {
-      if (
-        type === 'FILE_DOWNLOADING_START' ||
-        type === 'FILE_DOWNLOADING_END' ||
-        type === 'CHUNK_DOWNLOADING_UPDATE'
-      ) {
-        setHelper(Date.now());
-      }
-    });
-  });
+  useSyncExternalStore(
+    (onStoreChange) => {
+      if (!downloader) return () => {};
+
+      const sub = downloader.subject.subscribe(({ type }: DownloaderSubject) => {
+        if (subjectEvents.includes(type)) {
+          versionRef.current++;
+          onStoreChange();
+        }
+      });
+      return () => sub.unsubscribe();
+    },
+    () => versionRef.current,
+  );
+
+  return downloader?.filelist;
 };

package/src/cli/ui/hooks/input.ts

@@ -1,7 +1,6 @@
 import { useInput } from 'ink';
 import { useInkStore } from '../store';
 
-// problems with tsx watch
 export const useInputHook = () => {
   const downloader = useInkStore((state) => state.downloader);
   const switchPreview = useInkStore((state) => state.switchPreview);

package/src/cli/ui/store/index.ts

@@ -1,5 +1,5 @@
 import { create } from 'zustand';
-import type { Downloader } from '../../../services/downloader';
+import type { Downloader } from '../../../core';
 
 interface InkState {
   preview: boolean;