coomer-downloader 3.3.2 → 3.4.1

package/src/api/nsfw.xxx.ts

@@ -1,6 +1,7 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -8,7 +9,6 @@ async function getUserPage(user: string, offset: number) {
 }
 
 async function getUserPosts(user: string): Promise<string[]> {
-  console.log('Fetching user posts...');
   const posts = [];
   for (let i = 1; i < 100000; i++) {
     const page = await getUserPage(user, i);
@@ -26,7 +26,6 @@ async function getUserPosts(user: string): Promise<string[]> {
 }
 
 async function getPostsData(posts: string[]): Promise<CoomerFileList> {
-  console.log('Fetching posts data...');
   const filelist = new CoomerFileList();
   for (const post of posts) {
     const page = await fetch(post).then((r) => r.text());
@@ -40,7 +39,8 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
     if (!src) continue;
 
     const slug = post.split('post/')[1].split('?')[0];
-    const date = $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
+    const date =
+      $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
 
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;
@@ -53,7 +53,9 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
 
 export async function getRedditData(url: string): Promise<CoomerFileList> {
   const user = url.match(/u\/(\w+)/)?.[1] as string;
+  console.log('Fetching user posts...');
   const posts = await getUserPosts(user);
+  console.log('Fetching posts data...');
   const filelist = await getPostsData(posts);
   filelist.dirName = `${user}-reddit`;
   return filelist;

package/src/api/plain-curl.ts

@@ -1,4 +1,5 @@
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 
 export async function getPlainFileData(url: string): Promise<CoomerFileList> {
   const name = url.split('/').pop() as string;

package/src/cli/args-handler.ts

@@ -1,16 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-type ArgumentHandlerResult = {
-  url: string;
-  dir: string;
-  media: string;
-  include: string;
-  exclude: string;
-  skip: number;
-};
-
-export function argumentHander(): ArgumentHandlerResult {
+export function argumentHander() {
   return yargs(hideBin(process.argv))
     .option('url', {
       alias: 'u',
@@ -26,8 +17,7 @@ export function argumentHander(): ArgumentHandlerResult {
     })
     .option('media', {
       type: 'string',
-      choices: ['video', 'image', 'all'],
-      default: 'all',
+      choices: ['video', 'image'],
       description:
         "The type of media to download: 'video', 'image', or 'all'. 'all' is the default.",
     })
@@ -41,11 +31,26 @@ export function argumentHander(): ArgumentHandlerResult {
       default: '',
       description: 'Filter file names by a comma-separated list of keywords to exclude',
     })
+    .option('min-size', {
+      type: 'string',
+      default: '',
+      description: 'Minimum file size to download. Example: "1mb" or "500kb"',
+    })
+    .option('max-size', {
+      type: 'string',
+      default: '',
+      description: 'Maximum file size to download. Example: "1mb" or "500kb"',
+    })
     .option('skip', {
       type: 'number',
       default: 0,
       description: 'Skips the first N files in the download queue',
     })
+    .option('remove-dupilicates', {
+      type: 'boolean',
+      default: true,
+      description: 'removes duplicates by url and file hash',
+    })
     .help()
     .alias('help', 'h')
     .parseSync();

package/src/cli/ui/app.tsx

@@ -1,23 +1,24 @@
 import { Box } from 'ink';
 import React from 'react';
-import { CoomerFileList } from '../../services/file';
-import { FileBox, FileListStateBox, KeyboardControlsInfo, Loading, TitleBar } from './components';
+import { CoomerFileList } from '../../services/filelist';
+import {
+  FileBox,
+  FileListStateBox,
+  KeyboardControlsInfo,
+  Loading,
+  TitleBar,
+} from './components';
 import { useDownloaderHook } from './hooks/downloader';
 import { useInputHook } from './hooks/input';
-import { useInkStore } from './store';
 
 export function App() {
   useInputHook();
-  useDownloaderHook();
-
-  const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
-  const isFilelist = filelist instanceof CoomerFileList;
+  const filelist = useDownloaderHook();
 
   return (
     <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
       <TitleBar />
-      {!isFilelist ? (
+      {!(filelist instanceof CoomerFileList) ? (
         <Loading />
       ) : (
         <>
@@ -25,12 +26,12 @@ export function App() {
             <Box>
               <FileListStateBox filelist={filelist} />
             </Box>
-            <Box flexBasis={29}>
+            <Box flexBasis={30}>
               <KeyboardControlsInfo />
             </Box>
           </Box>
 
-          {filelist.getActiveFiles().map((file) => {
+          {filelist?.getActiveFiles().map((file) => {
             return <FileBox file={file} key={file.name} />;
           })}
         </>

package/src/cli/ui/components/preview.tsx

@@ -2,7 +2,7 @@ import { Box } from 'ink';
 import Image, { TerminalInfoProvider } from 'ink-picture';
 import React from 'react';
 import type { CoomerFile } from '../../../services/file';
-import { isImage } from '../../../utils/filters';
+import { isImage } from '../../../utils/mediatypes';
 import { useInkStore } from '../store';
 
 interface PreviewProps {

package/src/cli/ui/hooks/downloader.ts

@@ -1,21 +1,31 @@
-import { useEffect, useState } from 'react';
+import { useRef, useSyncExternalStore } from 'react';
 import { useInkStore } from '../store';
 
 export const useDownloaderHook = () => {
   const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
 
-  const [_, setHelper] = useState(0);
+  const versionRef = useRef(0);
 
-  useEffect(() => {
-    downloader?.subject.subscribe(({ type }) => {
-      if (
-        type === 'FILE_DOWNLOADING_START' ||
-        type === 'FILE_DOWNLOADING_END' ||
-        type === 'CHUNK_DOWNLOADING_UPDATE'
-      ) {
-        setHelper(Date.now());
-      }
-    });
-  });
+  useSyncExternalStore(
+    (onStoreChange) => {
+      if (!downloader) return () => {};
+
+      const sub = downloader.subject.subscribe(({ type }) => {
+        const targets = [
+          'FILE_DOWNLOADING_START',
+          'FILE_DOWNLOADING_END',
+          'CHUNK_DOWNLOADING_UPDATE',
+        ];
+
+        if (targets.includes(type)) {
+          versionRef.current++;
+          onStoreChange();
+        }
+      });
+      return () => sub.unsubscribe();
+    },
+    () => versionRef.current,
+  );
+
+  return downloader?.filelist;
 };

package/src/index.ts

@@ -6,27 +6,50 @@ import { argumentHander } from './cli/args-handler';
 import { createReactInk } from './cli/ui';
 import { useInkStore } from './cli/ui/store';
 import { Downloader } from './services/downloader';
+import { parseSizeValue } from './utils/filters';
 import { setGlobalHeaders } from './utils/requests';
 
 async function run() {
   createReactInk();
 
-  const { url, dir, media, include, exclude, skip } = argumentHander();
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
+    argumentHander();
 
   const filelist = await apiHandler(url);
 
-  filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
+  filelist
+    .setDirPath(dir)
+    .skip(skip)
+    .filterByText(include, exclude)
+    .filterByMediaType(media);
+
+  if (removeDupilicates) {
+    filelist.removeURLDuplicates();
+  }
+
+  const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
+  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
 
   await filelist.calculateFileSizes();
 
   setGlobalHeaders({ Referer: url });
 
-  const downloader = new Downloader(filelist);
+  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
   useInkStore.getState().setDownloader(downloader);
 
   await downloader.downloadFiles();
 
-  process.kill(process.pid, 'SIGINT');
+  if (removeDupilicates) {
+    await filelist.removeDuplicatesByHash();
+  }
 }
 
-run();
+(async () => {
+  try {
+    await run();
+    process.exit(0);
+  } catch (err) {
+    console.error('Fatal error:', err);
+    process.exit(1);
+  }
+})();

package/src/logger/index.ts

@@ -0,0 +1,15 @@
+import pino from 'pino';
+
+// tail -f debug.log | jq -C .
+const logger = pino(
+  {
+    level: 'debug',
+  },
+  pino.destination({
+    dest: './debug.log',
+    append: false,
+    sync: true,
+  }),
+);
+
+export default logger;

package/src/services/downloader.ts

@@ -3,18 +3,19 @@ import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
 import { tryFixCoomerUrl } from '../api/coomer-api';
-import type { DownloaderSubject } from '../types';
-import { getFileSize, mkdir } from '../utils/io';
+import type { AbortControllerSubject, DownloaderSubject } from '../types';
+import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
 import { fetchByteRange } from '../utils/requests';
 import { Timer } from '../utils/timer';
-import type { CoomerFile, CoomerFileList } from './file';
+import type { CoomerFile } from './file';
+import type { CoomerFileList } from './filelist';
 
 export class Downloader {
   public subject = new Subject<DownloaderSubject>();
 
   private abortController = new AbortController();
-  public abortControllerSubject = new Subject<string>();
+  public abortControllerSubject = new Subject<AbortControllerSubject>();
 
   setAbortControllerListener() {
     this.abortControllerSubject.subscribe((type) => {
@@ -25,6 +26,8 @@ export class Downloader {
 
   constructor(
     public filelist: CoomerFileList,
+    public minSize?: number,
+    public maxSize?: number,
     public chunkTimeout = 30_000,
     public chunkFetchRetries = 5,
     public fetchRetries = 7,
@@ -40,8 +43,10 @@ export class Downloader {
   ): Promise<void> {
     const signal = this.abortController.signal;
     const subject = this.subject;
-    const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
-    let i: NodeJS.Timeout | undefined;
+    const { timer } = Timer.withAbortController(
+      this.chunkTimeout,
+      this.abortControllerSubject,
+    );
 
     try {
       const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
@@ -74,7 +79,6 @@ export class Downloader {
     } finally {
       subject.next({ type: 'CHUNK_DOWNLOADING_END' });
       timer.stop();
-      clearInterval(i);
     }
   }
 
@@ -82,6 +86,20 @@ export class Downloader {
     this.abortControllerSubject.next('FILE_SKIP');
   }
 
+  private filterFileSize(file: CoomerFile) {
+    if (!file.size) return;
+    if (
+      (this.minSize && file.size < this.minSize) ||
+      (this.maxSize && file.size > this.maxSize)
+    ) {
+      try {
+        deleteFile(file.filepath);
+      } catch {}
+      this.skip();
+      return;
+    }
+  }
+
   async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
     const signal = this.abortController.signal;
     try {
@@ -100,6 +118,8 @@ export class Downloader {
       const restFileSize = parseInt(contentLength);
       file.size = restFileSize + file.downloaded;
 
+      this.filterFileSize(file);
+
       if (file.size > file.downloaded && response.body) {
         const stream = Readable.fromWeb(response.body);
         stream.setMaxListeners(20);
@@ -124,6 +144,7 @@ export class Downloader {
     mkdir(this.filelist.dirPath as string);
 
     this.subject.next({ type: 'FILES_DOWNLOADING_START' });
+
     for (const file of this.filelist.files) {
       file.active = true;
 

package/src/services/file.ts

@@ -1,31 +1,19 @@
-import os from 'node:os';
-import path from 'node:path';
-import type { MediaType } from '../types';
-import { filterString, testMediaType } from '../utils/filters';
 import { getFileSize } from '../utils/io';
 
-interface ICoomerFile {
-  name: string;
-  url: string;
-  filepath?: string;
-  size?: number;
-  downloaded?: number;
-  content?: string;
-}
-
 export class CoomerFile {
   public active = false;
+  public hash?: string;
 
   constructor(
     public name: string,
     public url: string,
-    public filepath?: string,
+    public filepath = '',
     public size?: number,
     public downloaded = 0,
     public content?: string,
   ) {}
 
-  public async getDownloadedSize() {
+  public async calcDownloadedSize() {
     this.downloaded = await getFileSize(this.filepath as string);
     return this;
   }
@@ -35,61 +23,7 @@ export class CoomerFile {
     return text;
   }
 
-  public static from(f: ICoomerFile) {
+  public static from(f: Pick<CoomerFile, 'name' | 'url'> & Partial<CoomerFile>) {
     return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
   }
 }
-
-export class CoomerFileList {
-  public dirPath?: string;
-  public dirName?: string;
-
-  constructor(public files: CoomerFile[] = []) {}
-
-  public setDirPath(dir: string, dirName?: string) {
-    dirName = dirName || (this.dirName as string);
-
-    if (dir === './') {
-      this.dirPath = path.resolve(dir, dirName);
-    } else {
-      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
-    }
-
-    this.files.forEach((file) => {
-      file.filepath = path.join(this.dirPath as string, file.name);
-    });
-
-    return this;
-  }
-
-  public filterByText(include: string, exclude: string) {
-    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
-    return this;
-  }
-
-  public filterByMediaType(media?: string) {
-    if (media) {
-      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
-    }
-    return this;
-  }
-
-  public skip(n: number) {
-    this.files = this.files.slice(n);
-    return this;
-  }
-
-  public async calculateFileSizes() {
-    for (const file of this.files) {
-      await file.getDownloadedSize();
-    }
-  }
-
-  public getActiveFiles() {
-    return this.files.filter((f) => f.active);
-  }
-
-  public getDownloaded() {
-    return this.files.filter((f) => f.size && f.size <= f.downloaded);
-  }
-}

package/src/services/filelist.ts

@@ -0,0 +1,86 @@
+import os from 'node:os';
+import path from 'node:path';
+import logger from '../logger';
+import type { MediaType } from '../types';
+import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
+import { filterString } from '../utils/filters';
+import { deleteFile, getFileHash, sanitizeFilename } from '../utils/io';
+import { testMediaType } from '../utils/mediatypes';
+import type { CoomerFile } from './file';
+
+export class CoomerFileList {
+  public dirPath?: string;
+  public dirName?: string;
+
+  constructor(public files: CoomerFile[] = []) {}
+
+  public setDirPath(dir: string, dirName?: string) {
+    dirName = dirName || (this.dirName as string);
+
+    if (dir === './') {
+      this.dirPath = path.resolve(dir, dirName);
+    } else {
+      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
+    }
+
+    this.files.forEach((file) => {
+      const safeName = sanitizeFilename(file.name) || file.name;
+      file.filepath = path.join(this.dirPath as string, safeName);
+    });
+
+    return this;
+  }
+
+  public filterByText(include: string, exclude: string) {
+    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
+    return this;
+  }
+
+  public filterByMediaType(media?: string) {
+    if (media) {
+      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
+    }
+    return this;
+  }
+
+  public skip(n: number) {
+    this.files = this.files.slice(n);
+    return this;
+  }
+
+  public async calculateFileSizes() {
+    for (const file of this.files) {
+      await file.calcDownloadedSize();
+    }
+    return this;
+  }
+
+  public getActiveFiles() {
+    return this.files.filter((f) => f.active);
+  }
+
+  public getDownloaded() {
+    return this.files.filter((f) => f.size && f.size <= f.downloaded);
+  }
+
+  public async removeDuplicatesByHash() {
+    for (const file of this.files) {
+      file.hash = await getFileHash(file.filepath);
+    }
+
+    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
+
+    // console.log({ duplicates });
+
+    // logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+
+    duplicates.forEach((f) => {
+      deleteFile(f.filepath);
+    });
+  }
+
+  public removeURLDuplicates() {
+    this.files = removeDuplicatesBy(this.files, 'url');
+    return this;
+  }
+}

package/src/types/index.ts

@@ -1,15 +1,16 @@
-export type MediaType = 'video' | 'image' | 'all';
+export type MediaType = 'video' | 'image';
 
 export type DownloaderSubjectSignal =
   | 'FILES_DOWNLOADING_START'
   | 'FILES_DOWNLOADING_END'
   | 'FILE_DOWNLOADING_START'
   | 'FILE_DOWNLOADING_END'
-  | 'FILE_SKIP'
   | 'CHUNK_DOWNLOADING_START'
   | 'CHUNK_DOWNLOADING_UPDATE'
   | 'CHUNK_DOWNLOADING_END';
 
+export type AbortControllerSubject = 'FILE_SKIP' | 'TIMEOUT';
+
 export type DownloaderSubject = {
   type: DownloaderSubjectSignal;
 };

package/src/utils/duplicates.ts

@@ -0,0 +1,23 @@
+export function collectUniquesAndDuplicatesBy<T extends {}, K extends keyof T>(
+  xs: T[],
+  k: K,
+): { uniques: T[]; duplicates: T[] } {
+  const seen = new Set<T[K]>();
+
+  return xs.reduce(
+    (acc, item) => {
+      if (seen.has(item[k])) {
+        acc.duplicates.push(item);
+      } else {
+        seen.add(item[k]);
+        acc.uniques.push(item);
+      }
+      return acc;
+    },
+    { uniques: [] as T[], duplicates: [] as T[] },
+  );
+}
+
+export function removeDuplicatesBy<T extends {}, K extends keyof T>(xs: T[], k: K) {
+  return [...new Map(xs.map((x) => [x[k], x])).values()];
+}

package/src/utils/filters.ts

@@ -1,17 +1,3 @@
-import type { MediaType } from '../types';
-
-export function isImage(name: string) {
-  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
-}
-
-export function isVideo(name: string) {
-  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
-}
-
-export function testMediaType(name: string, type: MediaType) {
-  return type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
-}
-
 function includesAllWords(str: string, words: string[]) {
   if (!words.length) return true;
   return words.every((w) => str.includes(w));
@@ -30,5 +16,19 @@ function parseQuery(query: string) {
 }
 
 export function filterString(text: string, include: string, exclude: string): boolean {
-  return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
+  return (
+    includesAllWords(text, parseQuery(include)) &&
+    includesNoWords(text, parseQuery(exclude))
+  );
+}
+
+export function parseSizeValue(s: string) {
+  if (!s) return NaN;
+  const m = s.match(/^([0-9]+(?:\.[0-9]+)?)(b|kb|mb|gb)?$/i);
+  if (!m) return NaN;
+  const val = parseFloat(m[1]);
+  const unit = (m[2] || 'b').toLowerCase();
+  const mult =
+    unit === 'kb' ? 1024 : unit === 'mb' ? 1024 ** 2 : unit === 'gb' ? 1024 ** 3 : 1;
+  return Math.floor(val * mult);
 }

package/src/utils/io.ts

@@ -1,4 +1,7 @@
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
+import { access, constants, unlink } from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
 
 export async function getFileSize(filepath: string) {
   let size = 0;
@@ -8,8 +11,30 @@ export async function getFileSize(filepath: string) {
   return size;
 }
 
+export async function getFileHash(filepath: string) {
+  const hash = createHash('sha256');
+  const filestream = fs.createReadStream(filepath);
+  await pipeline(filestream, hash);
+  return hash.digest('hex');
+}
+
 export function mkdir(filepath: string) {
   if (!fs.existsSync(filepath)) {
     fs.mkdirSync(filepath, { recursive: true });
   }
 }
+
+export async function deleteFile(path: string) {
+  await access(path, constants.F_OK);
+  await unlink(path);
+}
+
+export function sanitizeFilename(name: string) {
+  if (!name) return name;
+
+  return name
+    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '-') // Newlines (\r \n) are caught here
+    .replace(/\s+/g, ' ') // Turn tabs/multiple spaces into one space
+    .trim()
+    .replace(/[.]+$/, ''); // Remove trailing dots
+}