coomer-downloader 3.3.2 → 3.4.0

package/README.md

@@ -2,9 +2,12 @@
 
 [![NPM](https://nodei.co/npm/coomer-downloader.svg?color=lightgrey)](https://www.npmjs.com/package/coomer-downloader)
 
+<img src="https://github.com/smartacephale/coomer-downloader/blob/main/docs/images/Screenshot%2001.jpg?raw=true" width="500"></img>
 
 ### Features
-* script keeps track of downloaded files and resume downloading if it's crashed. 
+* Script keeps track of downloaded files and resume downloading if it's crashed.
+* Skip files, filter by text and media type
+* Preview images in console. For now partial downloads not supported.
 
 ### Install:
 ```
@@ -42,12 +45,20 @@ npx coomer-downloader --u https://random.com/file.mp4
 ### Options:
 ```
 --dir <directory>
---media <video|image|all>
+--media <video|image>
 --skip <number>
 
-# include/exclude filters files by name
+# Removes duplicates by url and file hash
+--remove-dupilicates <true|false>
+
+# Filter by file name
 --include <keyword1>
 --exclude <keyword2>
+
+# Filter by min/max file size. Example: "1mb" or "500kb"
+--min-size <keyword>
+--max-size <keyword>
+
 ```
 
 ```

package/biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/2.2.6/schema.json",
+  "$schema": "https://biomejs.dev/schemas/2.3.10/schema.json",
   "assist": {
     "actions": {
       "source": {
@@ -14,13 +14,14 @@
     "indentStyle": "space",
     "indentWidth": 2,
     "lineEnding": "lf",
-    "lineWidth": 100
+    "lineWidth": 90
   },
   "javascript": {
     "formatter": {
       "quoteStyle": "single",
       "semicolons": "always"
-    }
+    },
+    "jsxRuntime": "reactClassic"
   },
   "linter": {
     "enabled": true,
@@ -37,7 +38,8 @@
         "useNumberNamespace": "off"
       },
       "suspicious": {
-        "noRedundantUseStrict": "off"
+        "noRedundantUseStrict": "off",
+        "noControlCharactersInRegex": "info"
       }
     }
   }

package/dist/index.js

@@ -11,16 +11,41 @@ import { fetch } from "undici";
 import os from "node:os";
 import path from "node:path";
 
-// src/utils/filters.ts
-function isImage(name) {
-  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
-}
-function isVideo(name) {
-  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+// src/logger/index.ts
+import pino from "pino";
+var logger = pino(
+  {
+    level: "debug"
+  },
+  pino.destination({
+    dest: "./debug.log",
+    append: false,
+    sync: true
+  })
+);
+var logger_default = logger;
+
+// src/utils/duplicates.ts
+function collectUniquesAndDuplicatesBy(xs, k) {
+  const seen = /* @__PURE__ */ new Set();
+  return xs.reduce(
+    (acc, item) => {
+      if (seen.has(item[k])) {
+        acc.duplicates.push(item);
+      } else {
+        seen.add(item[k]);
+        acc.uniques.push(item);
+      }
+      return acc;
+    },
+    { uniques: [], duplicates: [] }
+  );
 }
-function testMediaType(name, type) {
-  return type === "all" ? true : type === "image" ? isImage(name) : isVideo(name);
+function removeDuplicatesBy(xs, k) {
+  return [...new Map(xs.map((x) => [x[k], x])).values()];
 }
+
+// src/utils/filters.ts
 function includesAllWords(str, words) {
   if (!words.length) return true;
   return words.every((w) => str.includes(w));
@@ -35,9 +60,21 @@ function parseQuery(query) {
 function filterString(text, include, exclude) {
   return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
 }
+function parseSizeValue(s) {
+  if (!s) return NaN;
+  const m = s.match(/^([0-9]+(?:\.[0-9]+)?)(b|kb|mb|gb)?$/i);
+  if (!m) return NaN;
+  const val = parseFloat(m[1]);
+  const unit = (m[2] || "b").toLowerCase();
+  const mult = unit === "kb" ? 1024 : unit === "mb" ? 1024 ** 2 : unit === "gb" ? 1024 ** 3 : 1;
+  return Math.floor(val * mult);
+}
 
 // src/utils/io.ts
+import { createHash } from "node:crypto";
 import fs from "node:fs";
+import { access, constants, unlink } from "node:fs/promises";
+import { pipeline } from "node:stream/promises";
 async function getFileSize(filepath) {
   let size = 0;
   if (fs.existsSync(filepath)) {
@@ -45,15 +82,40 @@ async function getFileSize(filepath) {
   }
   return size;
 }
+async function getFileHash(filepath) {
+  const hash = createHash("sha256");
+  const filestream = fs.createReadStream(filepath);
+  await pipeline(filestream, hash);
+  return hash.digest("hex");
+}
 function mkdir(filepath) {
   if (!fs.existsSync(filepath)) {
     fs.mkdirSync(filepath, { recursive: true });
   }
 }
+async function deleteFile(path2) {
+  await access(path2, constants.F_OK);
+  await unlink(path2);
+}
+function sanitizeFilename(name) {
+  if (!name) return name;
+  return name.replace(/[<>"/\\|?*\x00-\x1F]/g, "-").replace(/\s+/g, " ").trim().replace(/[.]+$/, "");
+}
+
+// src/utils/mediatypes.ts
+function isImage(name) {
+  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+}
+function isVideo(name) {
+  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+}
+function testMediaType(name, type) {
+  return type === "image" ? isImage(name) : isVideo(name);
+}
 
 // src/services/file.ts
 var CoomerFile = class _CoomerFile {
-  constructor(name, url, filepath, size, downloaded = 0, content) {
+  constructor(name, url, filepath = "", size, downloaded = 0, content) {
     this.name = name;
     this.url = url;
     this.filepath = filepath;
@@ -62,6 +124,7 @@ var CoomerFile = class _CoomerFile {
     this.content = content;
   }
   active = false;
+  hash;
   async getDownloadedSize() {
     this.downloaded = await getFileSize(this.filepath);
     return this;
@@ -88,7 +151,8 @@ var CoomerFileList = class {
       this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
     }
     this.files.forEach((file) => {
-      file.filepath = path.join(this.dirPath, file.name);
+      const safeName = sanitizeFilename(file.name) || file.name;
+      file.filepath = path.join(this.dirPath, safeName);
     });
     return this;
   }
@@ -110,6 +174,7 @@ var CoomerFileList = class {
     for (const file of this.files) {
       await file.getDownloadedSize();
     }
+    return this;
   }
   getActiveFiles() {
     return this.files.filter((f) => f.active);
@@ -117,6 +182,21 @@ var CoomerFileList = class {
   getDownloaded() {
     return this.files.filter((f) => f.size && f.size <= f.downloaded);
   }
+  async removeDuplicatesByHash() {
+    for (const file of this.files) {
+      file.hash = await getFileHash(file.filepath);
+    }
+    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, "hash");
+    console.log({ duplicates });
+    logger_default.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+    duplicates.forEach((f) => {
+      deleteFile(f.filepath);
+    });
+  }
+  removeURLDuplicates() {
+    this.files = removeDuplicatesBy(this.files, "url");
+    return this;
+  }
 };
 
 // src/api/bunkr.ts
@@ -225,10 +305,10 @@ async function getUserPostsAPI(user, offset) {
 async function getUserFiles(user) {
   const userPosts = [];
   const offset = 50;
-  for (let i = 0; i < 1e3; i++) {
+  for (let i = 0; i < 1e4; i++) {
     const posts = await getUserPostsAPI(user, i * offset);
     userPosts.push(...posts);
-    if (posts.length < 50) break;
+    if (posts.length < offset) break;
   }
   const filelist = new CoomerFileList();
   for (const p of userPosts) {
@@ -239,13 +319,23 @@ async function getUserFiles(user) {
     const postFiles = [...p.attachments, p.file].filter((f) => f.path).map((f, i) => {
       const ext = f.name.split(".").pop();
       const name = `${datentitle} ${i + 1}.${ext}`;
-      const url = `${user.domain}/${f.path}`;
+      const url = getUrl(f, user);
       return CoomerFile.from({ name, url, content });
     });
     filelist.files.push(...postFiles);
   }
   return filelist;
 }
+function getUrl(f, user) {
+  const normalizedPath = f.path.replace(/^\/+/, "/");
+  let url = "";
+  try {
+    url = new URL(normalizedPath, user.domain).toString();
+  } catch (_) {
+    url = `${user.domain}/${normalizedPath.replace(/^\//, "")}`;
+  }
+  return url;
+}
 async function parseUser(url) {
   const [_, domain, service, id] = url.match(
     /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/
@@ -320,7 +410,6 @@ async function getUserPage(user, offset) {
   return fetch4(url).then((r) => r.text());
 }
 async function getUserPosts(user) {
-  console.log("Fetching user posts...");
   const posts = [];
   for (let i = 1; i < 1e5; i++) {
     const page = await getUserPage(user, i);
@@ -332,7 +421,6 @@ async function getUserPosts(user) {
   return posts;
 }
 async function getPostsData(posts) {
-  console.log("Fetching posts data...");
   const filelist = new CoomerFileList();
   for (const post of posts) {
     const page = await fetch4(post).then((r) => r.text());
@@ -349,7 +437,9 @@ async function getPostsData(posts) {
 }
 async function getRedditData(url) {
   const user = url.match(/u\/(\w+)/)?.[1];
+  console.log("Fetching user posts...");
   const posts = await getUserPosts(user);
+  console.log("Fetching posts data...");
   const filelist = await getPostsData(posts);
   filelist.dirName = `${user}-reddit`;
   return filelist;
@@ -400,8 +490,7 @@ function argumentHander() {
     default: "./"
   }).option("media", {
     type: "string",
-    choices: ["video", "image", "all"],
-    default: "all",
+    choices: ["video", "image"],
     description: "The type of media to download: 'video', 'image', or 'all'. 'all' is the default."
   }).option("include", {
     type: "string",
@@ -411,10 +500,22 @@ function argumentHander() {
     type: "string",
     default: "",
     description: "Filter file names by a comma-separated list of keywords to exclude"
+  }).option("min-size", {
+    type: "string",
+    default: "",
+    description: 'Minimum file size to download. Example: "1mb" or "500kb"'
+  }).option("max-size", {
+    type: "string",
+    default: "",
+    description: 'Maximum file size to download. Example: "1mb" or "500kb"'
   }).option("skip", {
     type: "number",
     default: 0,
     description: "Skips the first N files in the download queue"
+  }).option("remove-dupilicates", {
+    type: "boolean",
+    default: true,
+    description: "removes duplicates by url and file hash"
   }).help().alias("help", "h").parseSync();
 }
 
@@ -558,7 +659,7 @@ import { Box as Box6, Spacer as Spacer2, Text as Text6 } from "ink";
 import React7 from "react";
 
 // package.json
-var version = "3.3.2";
+var version = "3.4.0";
 
 // src/cli/ui/components/titlebar.tsx
 function TitleBar() {
@@ -602,7 +703,7 @@ function App() {
   const downloader = useInkStore((state) => state.downloader);
   const filelist = downloader?.filelist;
   const isFilelist = filelist instanceof CoomerFileList;
-  return /* @__PURE__ */ React8.createElement(Box7, { borderStyle: "single", flexDirection: "column", borderColor: "blue", width: 80 }, /* @__PURE__ */ React8.createElement(TitleBar, null), !isFilelist ? /* @__PURE__ */ React8.createElement(Loading, null) : /* @__PURE__ */ React8.createElement(React8.Fragment, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(FileListStateBox, { filelist })), /* @__PURE__ */ React8.createElement(Box7, { flexBasis: 29 }, /* @__PURE__ */ React8.createElement(KeyboardControlsInfo, null))), filelist.getActiveFiles().map((file) => {
+  return /* @__PURE__ */ React8.createElement(Box7, { borderStyle: "single", flexDirection: "column", borderColor: "blue", width: 80 }, /* @__PURE__ */ React8.createElement(TitleBar, null), !isFilelist ? /* @__PURE__ */ React8.createElement(Loading, null) : /* @__PURE__ */ React8.createElement(React8.Fragment, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(FileListStateBox, { filelist })), /* @__PURE__ */ React8.createElement(Box7, { flexBasis: 30 }, /* @__PURE__ */ React8.createElement(KeyboardControlsInfo, null))), filelist.getActiveFiles().map((file) => {
     return /* @__PURE__ */ React8.createElement(FileBox, { file, key: file.name });
   })));
 }
@@ -615,7 +716,7 @@ function createReactInk() {
 // src/services/downloader.ts
 import fs2 from "node:fs";
 import { Readable, Transform } from "node:stream";
-import { pipeline } from "node:stream/promises";
+import { pipeline as pipeline2 } from "node:stream/promises";
 import { Subject } from "rxjs";
 
 // src/utils/promise.ts
@@ -650,7 +751,7 @@ var Timer = class _Timer {
     this.start();
     return this;
   }
-  static withAbortController(timeout, abortControllerSubject, message = "Timeout") {
+  static withAbortController(timeout, abortControllerSubject, message = "TIMEOUT") {
     const callback = () => {
       abortControllerSubject.next(message);
     };
@@ -661,8 +762,10 @@ var Timer = class _Timer {
 
 // src/services/downloader.ts
 var Downloader = class {
-  constructor(filelist, chunkTimeout = 3e4, chunkFetchRetries = 5, fetchRetries = 7) {
+  constructor(filelist, minSize, maxSize, chunkTimeout = 3e4, chunkFetchRetries = 5, fetchRetries = 7) {
     this.filelist = filelist;
+    this.minSize = minSize;
+    this.maxSize = maxSize;
     this.chunkTimeout = chunkTimeout;
     this.chunkFetchRetries = chunkFetchRetries;
     this.fetchRetries = fetchRetries;
@@ -680,8 +783,10 @@ var Downloader = class {
   async fetchStream(file, stream, sizeOld = 0, retries = this.chunkFetchRetries) {
     const signal = this.abortController.signal;
     const subject = this.subject;
-    const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
-    let i;
+    const { timer } = Timer.withAbortController(
+      this.chunkTimeout,
+      this.abortControllerSubject
+    );
     try {
       const fileStream = fs2.createWriteStream(file.filepath, { flags: "a" });
       const progressStream = new Transform({
@@ -694,7 +799,7 @@ var Downloader = class {
         }
       });
       subject.next({ type: "CHUNK_DOWNLOADING_START" });
-      await pipeline(stream, progressStream, fileStream, { signal });
+      await pipeline2(stream, progressStream, fileStream, { signal });
     } catch (error) {
       if (signal.aborted) {
         if (signal.reason === "FILE_SKIP") return;
@@ -711,12 +816,22 @@ var Downloader = class {
     } finally {
       subject.next({ type: "CHUNK_DOWNLOADING_END" });
       timer.stop();
-      clearInterval(i);
     }
   }
   skip() {
     this.abortControllerSubject.next("FILE_SKIP");
   }
+  filterFileSize(file) {
+    if (!file.size) return;
+    if (this.minSize && file.size < this.minSize || this.maxSize && file.size > this.maxSize) {
+      try {
+        deleteFile(file.filepath);
+      } catch {
+      }
+      this.skip();
+      return;
+    }
+  }
   async downloadFile(file, retries = this.fetchRetries) {
     const signal = this.abortController.signal;
     try {
@@ -729,6 +844,7 @@ var Downloader = class {
       if (!contentLength && file.downloaded > 0) return;
       const restFileSize = parseInt(contentLength);
       file.size = restFileSize + file.downloaded;
+      this.filterFileSize(file);
       if (file.size > file.downloaded && response.body) {
         const stream = Readable.fromWeb(response.body);
         stream.setMaxListeners(20);
@@ -765,14 +881,29 @@ var Downloader = class {
 // src/index.ts
 async function run() {
   createReactInk();
-  const { url, dir, media, include, exclude, skip } = argumentHander();
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } = argumentHander();
   const filelist = await apiHandler(url);
   filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
+  if (removeDupilicates) {
+    filelist.removeURLDuplicates();
+  }
+  const minSizeBytes = minSize ? parseSizeValue(minSize) : void 0;
+  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : void 0;
   await filelist.calculateFileSizes();
   setGlobalHeaders({ Referer: url });
-  const downloader = new Downloader(filelist);
+  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
   useInkStore.getState().setDownloader(downloader);
   await downloader.downloadFiles();
-  process2.kill(process2.pid, "SIGINT");
+  if (removeDupilicates) {
+    await filelist.removeDuplicatesByHash();
+  }
 }
-run();
+(async () => {
+  try {
+    await run();
+    process2.exit(0);
+  } catch (err) {
+    console.error("Fatal error:", err);
+    process2.exit(1);
+  }
+})();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.3.2",
+  "version": "3.4.0",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",
@@ -58,6 +58,7 @@
     "eslint-config-xo-react": "^0.27.0",
     "eslint-plugin-react": "^7.32.2",
     "eslint-plugin-react-hooks": "^4.6.0",
+    "pino": "^10.1.0",
     "tsx": "^4.20.6"
   }
 }

package/src/api/coomer-api.ts

@@ -1,5 +1,5 @@
 import { CoomerFile, CoomerFileList } from '../services/file';
-import { isImage } from '../utils/filters';
+import { isImage } from '../utils/mediatypes';
 import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
 
 type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
@@ -34,7 +34,10 @@ async function getUserProfileData(user: CoomerAPIUser): Promise<CoomerAPIUserDat
   return result as CoomerAPIUserData;
 }
 
-async function getUserPostsAPI(user: CoomerAPIUser, offset: number): Promise<CoomerAPIPost[]> {
+async function getUserPostsAPI(
+  user: CoomerAPIUser,
+  offset: number,
+): Promise<CoomerAPIPost[]> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
   const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
   return posts as CoomerAPIPost[];
@@ -44,10 +47,10 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
   const userPosts = [];
 
   const offset = 50;
-  for (let i = 0; i < 1000; i++) {
+  for (let i = 0; i < 10_000; i++) {
     const posts = await getUserPostsAPI(user, i * offset);
     userPosts.push(...posts);
-    if (posts.length < 50) break;
+    if (posts.length < offset) break;
   }
 
   const filelist = new CoomerFileList();
@@ -63,7 +66,7 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
       .map((f, i) => {
         const ext = f.name.split('.').pop();
         const name = `${datentitle} ${i + 1}.${ext}`;
-        const url = `${user.domain}/${f.path}`;
+        const url = getUrl(f, user);
         return CoomerFile.from({ name, url, content });
       });
 
@@ -73,6 +76,19 @@ export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList>
   return filelist;
 }
 
+function getUrl(f: CoomerAPIFile, user: CoomerAPIUser) {
+  // Normalize f.path to avoid protocol-relative or multiple-leading-slash paths
+  const normalizedPath = f.path.replace(/^\/+/, '/');
+  let url = '';
+  try {
+    url = new URL(normalizedPath, user.domain).toString();
+  } catch (_) {
+    // Fallback: join with a single slash
+    url = `${user.domain}/${normalizedPath.replace(/^\//, '')}`;
+  }
+  return url;
+}
+
 async function parseUser(url: string): Promise<CoomerAPIUser> {
   const [_, domain, service, id] = url.match(
     /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/,

package/src/api/nsfw.xxx.ts

@@ -8,7 +8,6 @@ async function getUserPage(user: string, offset: number) {
 }
 
 async function getUserPosts(user: string): Promise<string[]> {
-  console.log('Fetching user posts...');
   const posts = [];
   for (let i = 1; i < 100000; i++) {
     const page = await getUserPage(user, i);
@@ -26,7 +25,6 @@ async function getUserPosts(user: string): Promise<string[]> {
 }
 
 async function getPostsData(posts: string[]): Promise<CoomerFileList> {
-  console.log('Fetching posts data...');
   const filelist = new CoomerFileList();
   for (const post of posts) {
     const page = await fetch(post).then((r) => r.text());
@@ -53,7 +51,9 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
 
 export async function getRedditData(url: string): Promise<CoomerFileList> {
   const user = url.match(/u\/(\w+)/)?.[1] as string;
+  console.log('Fetching user posts...');
   const posts = await getUserPosts(user);
+  console.log('Fetching posts data...');
   const filelist = await getPostsData(posts);
   filelist.dirName = `${user}-reddit`;
   return filelist;

package/src/cli/args-handler.ts

@@ -1,16 +1,7 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-type ArgumentHandlerResult = {
-  url: string;
-  dir: string;
-  media: string;
-  include: string;
-  exclude: string;
-  skip: number;
-};
-
-export function argumentHander(): ArgumentHandlerResult {
+export function argumentHander() {
   return yargs(hideBin(process.argv))
     .option('url', {
       alias: 'u',
@@ -26,8 +17,7 @@ export function argumentHander(): ArgumentHandlerResult {
     })
     .option('media', {
       type: 'string',
-      choices: ['video', 'image', 'all'],
-      default: 'all',
+      choices: ['video', 'image'],
       description:
         "The type of media to download: 'video', 'image', or 'all'. 'all' is the default.",
     })
@@ -41,11 +31,26 @@ export function argumentHander(): ArgumentHandlerResult {
       default: '',
       description: 'Filter file names by a comma-separated list of keywords to exclude',
     })
+    .option('min-size', {
+      type: 'string',
+      default: '',
+      description: 'Minimum file size to download. Example: "1mb" or "500kb"',
+    })
+    .option('max-size', {
+      type: 'string',
+      default: '',
+      description: 'Maximum file size to download. Example: "1mb" or "500kb"',
+    })
     .option('skip', {
       type: 'number',
       default: 0,
       description: 'Skips the first N files in the download queue',
     })
+    .option('remove-dupilicates', {
+      type: 'boolean',
+      default: true,
+      description: 'removes duplicates by url and file hash',
+    })
     .help()
     .alias('help', 'h')
     .parseSync();

package/src/cli/ui/app.tsx

@@ -25,7 +25,7 @@ export function App() {
             <Box>
               <FileListStateBox filelist={filelist} />
             </Box>
-            <Box flexBasis={29}>
+            <Box flexBasis={30}>
               <KeyboardControlsInfo />
             </Box>
           </Box>

package/src/cli/ui/components/preview.tsx

@@ -2,7 +2,7 @@ import { Box } from 'ink';
 import Image, { TerminalInfoProvider } from 'ink-picture';
 import React from 'react';
 import type { CoomerFile } from '../../../services/file';
-import { isImage } from '../../../utils/filters';
+import { isImage } from '../../../utils/mediatypes';
 import { useInkStore } from '../store';
 
 interface PreviewProps {

package/src/index.ts

@@ -6,27 +6,50 @@ import { argumentHander } from './cli/args-handler';
 import { createReactInk } from './cli/ui';
 import { useInkStore } from './cli/ui/store';
 import { Downloader } from './services/downloader';
+import { parseSizeValue } from './utils/filters';
 import { setGlobalHeaders } from './utils/requests';
 
 async function run() {
   createReactInk();
 
-  const { url, dir, media, include, exclude, skip } = argumentHander();
+  const { url, dir, media, include, exclude, minSize, maxSize, skip, removeDupilicates } =
+    argumentHander();
 
   const filelist = await apiHandler(url);
 
-  filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
+  filelist
+    .setDirPath(dir)
+    .skip(skip)
+    .filterByText(include, exclude)
+    .filterByMediaType(media);
+
+  if (removeDupilicates) {
+    filelist.removeURLDuplicates();
+  }
+
+  const minSizeBytes = minSize ? parseSizeValue(minSize) : undefined;
+  const maxSizeBytes = maxSize ? parseSizeValue(maxSize) : undefined;
 
   await filelist.calculateFileSizes();
 
   setGlobalHeaders({ Referer: url });
 
-  const downloader = new Downloader(filelist);
+  const downloader = new Downloader(filelist, minSizeBytes, maxSizeBytes);
   useInkStore.getState().setDownloader(downloader);
 
   await downloader.downloadFiles();
 
-  process.kill(process.pid, 'SIGINT');
+  if (removeDupilicates) {
+    await filelist.removeDuplicatesByHash();
+  }
 }
 
-run();
+(async () => {
+  try {
+    await run();
+    process.exit(0);
+  } catch (err) {
+    console.error('Fatal error:', err);
+    process.exit(1);
+  }
+})();

package/src/logger/index.ts

@@ -0,0 +1,15 @@
+import pino from 'pino';
+
+// tail -f debug.log | jq -C .
+const logger = pino(
+  {
+    level: 'debug',
+  },
+  pino.destination({
+    dest: './debug.log',
+    append: false,
+    sync: true,
+  }),
+);
+
+export default logger;

package/src/services/downloader.ts

@@ -3,8 +3,8 @@ import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
 import { tryFixCoomerUrl } from '../api/coomer-api';
-import type { DownloaderSubject } from '../types';
-import { getFileSize, mkdir } from '../utils/io';
+import type { AbortControllerSubject, DownloaderSubject } from '../types';
+import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
 import { fetchByteRange } from '../utils/requests';
 import { Timer } from '../utils/timer';
@@ -14,7 +14,7 @@ export class Downloader {
   public subject = new Subject<DownloaderSubject>();
 
   private abortController = new AbortController();
-  public abortControllerSubject = new Subject<string>();
+  public abortControllerSubject = new Subject<AbortControllerSubject>();
 
   setAbortControllerListener() {
     this.abortControllerSubject.subscribe((type) => {
@@ -25,6 +25,8 @@ export class Downloader {
 
   constructor(
     public filelist: CoomerFileList,
+    public minSize?: number,
+    public maxSize?: number,
     public chunkTimeout = 30_000,
     public chunkFetchRetries = 5,
     public fetchRetries = 7,
@@ -40,8 +42,10 @@ export class Downloader {
   ): Promise<void> {
     const signal = this.abortController.signal;
     const subject = this.subject;
-    const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
-    let i: NodeJS.Timeout | undefined;
+    const { timer } = Timer.withAbortController(
+      this.chunkTimeout,
+      this.abortControllerSubject,
+    );
 
     try {
       const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
@@ -74,7 +78,6 @@ export class Downloader {
     } finally {
       subject.next({ type: 'CHUNK_DOWNLOADING_END' });
       timer.stop();
-      clearInterval(i);
     }
   }
 
@@ -82,6 +85,20 @@ export class Downloader {
     this.abortControllerSubject.next('FILE_SKIP');
   }
 
+  private filterFileSize(file: CoomerFile) {
+    if (!file.size) return;
+    if (
+      (this.minSize && file.size < this.minSize) ||
+      (this.maxSize && file.size > this.maxSize)
+    ) {
+      try {
+        deleteFile(file.filepath);
+      } catch {}
+      this.skip();
+      return;
+    }
+  }
+
   async downloadFile(file: CoomerFile, retries = this.fetchRetries): Promise<void> {
     const signal = this.abortController.signal;
     try {
@@ -100,6 +117,8 @@ export class Downloader {
       const restFileSize = parseInt(contentLength);
       file.size = restFileSize + file.downloaded;
 
+      this.filterFileSize(file);
+
       if (file.size > file.downloaded && response.body) {
         const stream = Readable.fromWeb(response.body);
         stream.setMaxListeners(20);
@@ -124,6 +143,7 @@ export class Downloader {
     mkdir(this.filelist.dirPath as string);
 
     this.subject.next({ type: 'FILES_DOWNLOADING_START' });
+
     for (const file of this.filelist.files) {
       file.active = true;
 

package/src/services/file.ts

@@ -1,25 +1,20 @@
 import os from 'node:os';
 import path from 'node:path';
+import logger from '../logger';
 import type { MediaType } from '../types';
-import { filterString, testMediaType } from '../utils/filters';
-import { getFileSize } from '../utils/io';
-
-interface ICoomerFile {
-  name: string;
-  url: string;
-  filepath?: string;
-  size?: number;
-  downloaded?: number;
-  content?: string;
-}
+import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
+import { filterString } from '../utils/filters';
+import { deleteFile, getFileHash, getFileSize, sanitizeFilename } from '../utils/io';
+import { testMediaType } from '../utils/mediatypes';
 
 export class CoomerFile {
   public active = false;
+  public hash?: string;
 
   constructor(
     public name: string,
     public url: string,
-    public filepath?: string,
+    public filepath = '',
     public size?: number,
     public downloaded = 0,
     public content?: string,
@@ -35,7 +30,7 @@ export class CoomerFile {
     return text;
   }
 
-  public static from(f: ICoomerFile) {
+  public static from(f: Pick<CoomerFile, 'name' | 'url'> & Partial<CoomerFile>) {
     return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
   }
 }
@@ -56,7 +51,8 @@ export class CoomerFileList {
     }
 
     this.files.forEach((file) => {
-      file.filepath = path.join(this.dirPath as string, file.name);
+      const safeName = sanitizeFilename(file.name) || file.name;
+      file.filepath = path.join(this.dirPath as string, safeName);
     });
 
     return this;
@@ -83,6 +79,7 @@ export class CoomerFileList {
     for (const file of this.files) {
       await file.getDownloadedSize();
     }
+    return this;
   }
 
   public getActiveFiles() {
@@ -92,4 +89,25 @@ export class CoomerFileList {
   public getDownloaded() {
     return this.files.filter((f) => f.size && f.size <= f.downloaded);
   }
+
+  public async removeDuplicatesByHash() {
+    for (const file of this.files) {
+      file.hash = await getFileHash(file.filepath);
+    }
+
+    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
+
+    console.log({ duplicates });
+
+    logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+
+    duplicates.forEach((f) => {
+      deleteFile(f.filepath);
+    });
+  }
+
+  public removeURLDuplicates() {
+    this.files = removeDuplicatesBy(this.files, 'url');
+    return this;
+  }
 }

package/src/types/index.ts

@@ -1,15 +1,16 @@
-export type MediaType = 'video' | 'image' | 'all';
+export type MediaType = 'video' | 'image';
 
 export type DownloaderSubjectSignal =
   | 'FILES_DOWNLOADING_START'
   | 'FILES_DOWNLOADING_END'
   | 'FILE_DOWNLOADING_START'
   | 'FILE_DOWNLOADING_END'
-  | 'FILE_SKIP'
   | 'CHUNK_DOWNLOADING_START'
   | 'CHUNK_DOWNLOADING_UPDATE'
   | 'CHUNK_DOWNLOADING_END';
 
+export type AbortControllerSubject = 'FILE_SKIP' | 'TIMEOUT';
+
 export type DownloaderSubject = {
   type: DownloaderSubjectSignal;
 };

package/src/utils/duplicates.ts

@@ -0,0 +1,23 @@
+export function collectUniquesAndDuplicatesBy<T extends {}, K extends keyof T>(
+  xs: T[],
+  k: K,
+): { uniques: T[]; duplicates: T[] } {
+  const seen = new Set<T[K]>();
+
+  return xs.reduce(
+    (acc, item) => {
+      if (seen.has(item[k])) {
+        acc.duplicates.push(item);
+      } else {
+        seen.add(item[k]);
+        acc.uniques.push(item);
+      }
+      return acc;
+    },
+    { uniques: [] as T[], duplicates: [] as T[] },
+  );
+}
+
+export function removeDuplicatesBy<T extends {}, K extends keyof T>(xs: T[], k: K) {
+  return [...new Map(xs.map((x) => [x[k], x])).values()];
+}

package/src/utils/filters.ts

@@ -1,17 +1,3 @@
-import type { MediaType } from '../types';
-
-export function isImage(name: string) {
-  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
-}
-
-export function isVideo(name: string) {
-  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
-}
-
-export function testMediaType(name: string, type: MediaType) {
-  return type === 'all' ? true : type === 'image' ? isImage(name) : isVideo(name);
-}
-
 function includesAllWords(str: string, words: string[]) {
   if (!words.length) return true;
   return words.every((w) => str.includes(w));
@@ -30,5 +16,19 @@ function parseQuery(query: string) {
 }
 
 export function filterString(text: string, include: string, exclude: string): boolean {
-  return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
+  return (
+    includesAllWords(text, parseQuery(include)) &&
+    includesNoWords(text, parseQuery(exclude))
+  );
+}
+
+export function parseSizeValue(s: string) {
+  if (!s) return NaN;
+  const m = s.match(/^([0-9]+(?:\.[0-9]+)?)(b|kb|mb|gb)?$/i);
+  if (!m) return NaN;
+  const val = parseFloat(m[1]);
+  const unit = (m[2] || 'b').toLowerCase();
+  const mult =
+    unit === 'kb' ? 1024 : unit === 'mb' ? 1024 ** 2 : unit === 'gb' ? 1024 ** 3 : 1;
+  return Math.floor(val * mult);
 }

package/src/utils/io.ts

@@ -1,4 +1,7 @@
+import { createHash } from 'node:crypto';
 import fs from 'node:fs';
+import { access, constants, unlink } from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
 
 export async function getFileSize(filepath: string) {
   let size = 0;
@@ -8,8 +11,30 @@ export async function getFileSize(filepath: string) {
   return size;
 }
 
+export async function getFileHash(filepath: string) {
+  const hash = createHash('sha256');
+  const filestream = fs.createReadStream(filepath);
+  await pipeline(filestream, hash);
+  return hash.digest('hex');
+}
+
 export function mkdir(filepath: string) {
   if (!fs.existsSync(filepath)) {
     fs.mkdirSync(filepath, { recursive: true });
   }
 }
+
+export async function deleteFile(path: string) {
+  await access(path, constants.F_OK);
+  await unlink(path);
+}
+
+export function sanitizeFilename(name: string) {
+  if (!name) return name;
+
+  return name
+    .replace(/[<>:"/\\|?*\x00-\x1F]/g, '-') // Newlines (\r \n) are caught here
+    .replace(/\s+/g, ' ') // Turn tabs/multiple spaces into one space
+    .trim()
+    .replace(/[.]+$/, ''); // Remove trailing dots
+}

package/src/utils/mediatypes.ts

@@ -0,0 +1,13 @@
+import type { MediaType } from '../types';
+
+export function isImage(name: string) {
+  return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
+}
+
+export function isVideo(name: string) {
+  return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
+}
+
+export function testMediaType(name: string, type: MediaType) {
+  return type === 'image' ? isImage(name) : isVideo(name);
+}

package/src/utils/timer.ts

@@ -1,4 +1,5 @@
 import type { Subject } from 'rxjs';
+import type { AbortControllerSubject } from '../types';
 
 export class Timer {
   private timer: NodeJS.Timeout | undefined;
@@ -34,8 +35,8 @@ export class Timer {
 
   static withAbortController(
     timeout: number,
-    abortControllerSubject: Subject<string>,
-    message: string = 'Timeout',
+    abortControllerSubject: Subject<AbortControllerSubject>,
+    message: AbortControllerSubject = 'TIMEOUT',
   ) {
     const callback = () => {
       abortControllerSubject.next(message);