coomer-downloader 3.4.0 → 3.4.1

package/dist/index.js

@@ -7,24 +7,67 @@ import process2 from "node:process";
 import * as cheerio from "cheerio";
 import { fetch } from "undici";
 
+// src/utils/io.ts
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import { access, constants, unlink } from "node:fs/promises";
+import { pipeline } from "node:stream/promises";
+async function getFileSize(filepath) {
+  let size = 0;
+  if (fs.existsSync(filepath)) {
+    size = (await fs.promises.stat(filepath)).size || 0;
+  }
+  return size;
+}
+async function getFileHash(filepath) {
+  const hash = createHash("sha256");
+  const filestream = fs.createReadStream(filepath);
+  await pipeline(filestream, hash);
+  return hash.digest("hex");
+}
+function mkdir(filepath) {
+  if (!fs.existsSync(filepath)) {
+    fs.mkdirSync(filepath, { recursive: true });
+  }
+}
+async function deleteFile(path2) {
+  await access(path2, constants.F_OK);
+  await unlink(path2);
+}
+function sanitizeFilename(name) {
+  if (!name) return name;
+  return name.replace(/[<>:"/\\|?*\x00-\x1F]/g, "-").replace(/\s+/g, " ").trim().replace(/[.]+$/, "");
+}
+
 // src/services/file.ts
+var CoomerFile = class _CoomerFile {
+  constructor(name, url, filepath = "", size, downloaded = 0, content) {
+    this.name = name;
+    this.url = url;
+    this.filepath = filepath;
+    this.size = size;
+    this.downloaded = downloaded;
+    this.content = content;
+  }
+  active = false;
+  hash;
+  async calcDownloadedSize() {
+    this.downloaded = await getFileSize(this.filepath);
+    return this;
+  }
+  get textContent() {
+    const text = `${this.name || ""} ${this.content || ""}`.toLowerCase();
+    return text;
+  }
+  static from(f) {
+    return new _CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
+  }
+};
+
+// src/services/filelist.ts
 import os from "node:os";
 import path from "node:path";
 
-// src/logger/index.ts
-import pino from "pino";
-var logger = pino(
-  {
-    level: "debug"
-  },
-  pino.destination({
-    dest: "./debug.log",
-    append: false,
-    sync: true
-  })
-);
-var logger_default = logger;
-
 // src/utils/duplicates.ts
 function collectUniquesAndDuplicatesBy(xs, k) {
   const seen = /* @__PURE__ */ new Set();
@@ -70,38 +113,6 @@ function parseSizeValue(s) {
   return Math.floor(val * mult);
 }
 
-// src/utils/io.ts
-import { createHash } from "node:crypto";
-import fs from "node:fs";
-import { access, constants, unlink } from "node:fs/promises";
-import { pipeline } from "node:stream/promises";
-async function getFileSize(filepath) {
-  let size = 0;
-  if (fs.existsSync(filepath)) {
-    size = (await fs.promises.stat(filepath)).size || 0;
-  }
-  return size;
-}
-async function getFileHash(filepath) {
-  const hash = createHash("sha256");
-  const filestream = fs.createReadStream(filepath);
-  await pipeline(filestream, hash);
-  return hash.digest("hex");
-}
-function mkdir(filepath) {
-  if (!fs.existsSync(filepath)) {
-    fs.mkdirSync(filepath, { recursive: true });
-  }
-}
-async function deleteFile(path2) {
-  await access(path2, constants.F_OK);
-  await unlink(path2);
-}
-function sanitizeFilename(name) {
-  if (!name) return name;
-  return name.replace(/[<>"/\\|?*\x00-\x1F]/g, "-").replace(/\s+/g, " ").trim().replace(/[.]+$/, "");
-}
-
 // src/utils/mediatypes.ts
 function isImage(name) {
   return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
@@ -113,30 +124,7 @@ function testMediaType(name, type) {
   return type === "image" ? isImage(name) : isVideo(name);
 }
 
-// src/services/file.ts
-var CoomerFile = class _CoomerFile {
-  constructor(name, url, filepath = "", size, downloaded = 0, content) {
-    this.name = name;
-    this.url = url;
-    this.filepath = filepath;
-    this.size = size;
-    this.downloaded = downloaded;
-    this.content = content;
-  }
-  active = false;
-  hash;
-  async getDownloadedSize() {
-    this.downloaded = await getFileSize(this.filepath);
-    return this;
-  }
-  get textContent() {
-    const text = `${this.name || ""} ${this.content || ""}`.toLowerCase();
-    return text;
-  }
-  static from(f) {
-    return new _CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
-  }
-};
+// src/services/filelist.ts
 var CoomerFileList = class {
   constructor(files = []) {
     this.files = files;
@@ -172,7 +160,7 @@ var CoomerFileList = class {
   }
   async calculateFileSizes() {
     for (const file of this.files) {
-      await file.getDownloadedSize();
+      await file.calcDownloadedSize();
     }
     return this;
   }
@@ -187,8 +175,6 @@ var CoomerFileList = class {
       file.hash = await getFileHash(file.filepath);
     }
     const { duplicates } = collectUniquesAndDuplicatesBy(this.files, "hash");
-    console.log({ duplicates });
-    logger_default.debug(`duplicates: ${JSON.stringify(duplicates)}`);
     duplicates.forEach((f) => {
       deleteFile(f.filepath);
     });
@@ -212,7 +198,9 @@ function decryptEncryptedUrl(encryptionData) {
   const secretKey = `SECRET_KEY_${Math.floor(encryptionData.timestamp / 3600)}`;
   const encryptedUrlBuffer = Buffer.from(encryptionData.url, "base64");
   const secretKeyBuffer = Buffer.from(secretKey, "utf-8");
-  return Array.from(encryptedUrlBuffer).map((byte, i) => String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length])).join("");
+  return Array.from(encryptedUrlBuffer).map(
+    (byte, i) => String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length])
+  ).join("");
 }
 async function getFileData(url, name) {
   const slug = url.split("/").pop();
@@ -659,7 +647,7 @@ import { Box as Box6, Spacer as Spacer2, Text as Text6 } from "ink";
 import React7 from "react";
 
 // package.json
-var version = "3.4.0";
+var version = "3.4.1";
 
 // src/cli/ui/components/titlebar.tsx
 function TitleBar() {
@@ -667,18 +655,30 @@ function TitleBar() {
 }
 
 // src/cli/ui/hooks/downloader.ts
-import { useEffect as useEffect2, useState as useState2 } from "react";
+import { useRef, useSyncExternalStore } from "react";
 var useDownloaderHook = () => {
   const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
-  const [_, setHelper] = useState2(0);
-  useEffect2(() => {
-    downloader?.subject.subscribe(({ type }) => {
-      if (type === "FILE_DOWNLOADING_START" || type === "FILE_DOWNLOADING_END" || type === "CHUNK_DOWNLOADING_UPDATE") {
-        setHelper(Date.now());
-      }
-    });
-  });
+  const versionRef = useRef(0);
+  useSyncExternalStore(
+    (onStoreChange) => {
+      if (!downloader) return () => {
+      };
+      const sub = downloader.subject.subscribe(({ type }) => {
+        const targets = [
+          "FILE_DOWNLOADING_START",
+          "FILE_DOWNLOADING_END",
+          "CHUNK_DOWNLOADING_UPDATE"
+        ];
+        if (targets.includes(type)) {
+          versionRef.current++;
+          onStoreChange();
+        }
+      });
+      return () => sub.unsubscribe();
+    },
+    () => versionRef.current
+  );
+  return downloader?.filelist;
 };
 
 // src/cli/ui/hooks/input.ts
@@ -699,11 +699,8 @@ var useInputHook = () => {
 // src/cli/ui/app.tsx
 function App() {
   useInputHook();
-  useDownloaderHook();
-  const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
-  const isFilelist = filelist instanceof CoomerFileList;
-  return /* @__PURE__ */ React8.createElement(Box7, { borderStyle: "single", flexDirection: "column", borderColor: "blue", width: 80 }, /* @__PURE__ */ React8.createElement(TitleBar, null), !isFilelist ? /* @__PURE__ */ React8.createElement(Loading, null) : /* @__PURE__ */ React8.createElement(React8.Fragment, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(FileListStateBox, { filelist })), /* @__PURE__ */ React8.createElement(Box7, { flexBasis: 30 }, /* @__PURE__ */ React8.createElement(KeyboardControlsInfo, null))), filelist.getActiveFiles().map((file) => {
+  const filelist = useDownloaderHook();
+  return /* @__PURE__ */ React8.createElement(Box7, { borderStyle: "single", flexDirection: "column", borderColor: "blue", width: 80 }, /* @__PURE__ */ React8.createElement(TitleBar, null), !(filelist instanceof CoomerFileList) ? /* @__PURE__ */ React8.createElement(Loading, null) : /* @__PURE__ */ React8.createElement(React8.Fragment, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(FileListStateBox, { filelist })), /* @__PURE__ */ React8.createElement(Box7, { flexBasis: 30 }, /* @__PURE__ */ React8.createElement(KeyboardControlsInfo, null))), filelist?.getActiveFiles().map((file) => {
     return /* @__PURE__ */ React8.createElement(FileBox, { file, key: file.name });
   })));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.4.0",
+  "version": "3.4.1",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",

package/src/api/bunkr.ts

@@ -1,6 +1,7 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 
 type EncData = { url: string; timestamp: number };
 
@@ -18,7 +19,9 @@ function decryptEncryptedUrl(encryptionData: EncData) {
   const encryptedUrlBuffer = Buffer.from(encryptionData.url, 'base64');
   const secretKeyBuffer = Buffer.from(secretKey, 'utf-8');
   return Array.from(encryptedUrlBuffer)
-    .map((byte, i) => String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]))
+    .map((byte, i) =>
+      String.fromCharCode(byte ^ secretKeyBuffer[i % secretKeyBuffer.length]),
+    )
     .join('');
 }
 

package/src/api/coomer-api.ts

@@ -1,4 +1,5 @@
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 import { isImage } from '../utils/mediatypes';
 import { fetchWithGlobalHeader, setGlobalHeaders } from '../utils/requests';
 

package/src/api/gofile.ts

@@ -1,5 +1,6 @@
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 import { setGlobalHeaders } from '../utils/requests';
 
 type GoFileAPIToken = { status: string; data: { token: string } };

package/src/api/index.ts

@@ -1,4 +1,4 @@
-import type { CoomerFileList } from '../services/file';
+import type { CoomerFileList } from '../services/filelist';
 import { getBunkrData } from './bunkr';
 import { getCoomerData } from './coomer-api';
 import { getGofileData } from './gofile';

package/src/api/nsfw.xxx.ts

@@ -1,6 +1,7 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -38,7 +39,8 @@ async function getPostsData(posts: string[]): Promise<CoomerFileList> {
     if (!src) continue;
 
     const slug = post.split('post/')[1].split('?')[0];
-    const date = $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
+    const date =
+      $('.sh-section .sh-section__passed').first().text().replace(/ /g, '-') || '';
 
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;

package/src/api/plain-curl.ts

@@ -1,4 +1,5 @@
-import { CoomerFile, CoomerFileList } from '../services/file';
+import { CoomerFile } from '../services/file';
+import { CoomerFileList } from '../services/filelist';
 
 export async function getPlainFileData(url: string): Promise<CoomerFileList> {
   const name = url.split('/').pop() as string;

package/src/cli/ui/app.tsx

@@ -1,23 +1,24 @@
 import { Box } from 'ink';
 import React from 'react';
-import { CoomerFileList } from '../../services/file';
-import { FileBox, FileListStateBox, KeyboardControlsInfo, Loading, TitleBar } from './components';
+import { CoomerFileList } from '../../services/filelist';
+import {
+  FileBox,
+  FileListStateBox,
+  KeyboardControlsInfo,
+  Loading,
+  TitleBar,
+} from './components';
 import { useDownloaderHook } from './hooks/downloader';
 import { useInputHook } from './hooks/input';
-import { useInkStore } from './store';
 
 export function App() {
   useInputHook();
-  useDownloaderHook();
-
-  const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
-  const isFilelist = filelist instanceof CoomerFileList;
+  const filelist = useDownloaderHook();
 
   return (
     <Box borderStyle="single" flexDirection="column" borderColor="blue" width={80}>
       <TitleBar />
-      {!isFilelist ? (
+      {!(filelist instanceof CoomerFileList) ? (
         <Loading />
       ) : (
         <>
@@ -30,7 +31,7 @@ export function App() {
             </Box>
           </Box>
 
-          {filelist.getActiveFiles().map((file) => {
+          {filelist?.getActiveFiles().map((file) => {
             return <FileBox file={file} key={file.name} />;
           })}
         </>

package/src/cli/ui/hooks/downloader.ts

@@ -1,21 +1,31 @@
-import { useEffect, useState } from 'react';
+import { useRef, useSyncExternalStore } from 'react';
 import { useInkStore } from '../store';
 
 export const useDownloaderHook = () => {
   const downloader = useInkStore((state) => state.downloader);
-  const filelist = downloader?.filelist;
 
-  const [_, setHelper] = useState(0);
+  const versionRef = useRef(0);
 
-  useEffect(() => {
-    downloader?.subject.subscribe(({ type }) => {
-      if (
-        type === 'FILE_DOWNLOADING_START' ||
-        type === 'FILE_DOWNLOADING_END' ||
-        type === 'CHUNK_DOWNLOADING_UPDATE'
-      ) {
-        setHelper(Date.now());
-      }
-    });
-  });
+  useSyncExternalStore(
+    (onStoreChange) => {
+      if (!downloader) return () => {};
+
+      const sub = downloader.subject.subscribe(({ type }) => {
+        const targets = [
+          'FILE_DOWNLOADING_START',
+          'FILE_DOWNLOADING_END',
+          'CHUNK_DOWNLOADING_UPDATE',
+        ];
+
+        if (targets.includes(type)) {
+          versionRef.current++;
+          onStoreChange();
+        }
+      });
+      return () => sub.unsubscribe();
+    },
+    () => versionRef.current,
+  );
+
+  return downloader?.filelist;
 };

package/src/services/downloader.ts

@@ -8,7 +8,8 @@ import { deleteFile, getFileSize, mkdir } from '../utils/io';
 import { sleep } from '../utils/promise';
 import { fetchByteRange } from '../utils/requests';
 import { Timer } from '../utils/timer';
-import type { CoomerFile, CoomerFileList } from './file';
+import type { CoomerFile } from './file';
+import type { CoomerFileList } from './filelist';
 
 export class Downloader {
   public subject = new Subject<DownloaderSubject>();

package/src/services/file.ts

@@ -1,11 +1,4 @@
-import os from 'node:os';
-import path from 'node:path';
-import logger from '../logger';
-import type { MediaType } from '../types';
-import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
-import { filterString } from '../utils/filters';
-import { deleteFile, getFileHash, getFileSize, sanitizeFilename } from '../utils/io';
-import { testMediaType } from '../utils/mediatypes';
+import { getFileSize } from '../utils/io';
 
 export class CoomerFile {
   public active = false;
@@ -20,7 +13,7 @@ export class CoomerFile {
     public content?: string,
   ) {}
 
-  public async getDownloadedSize() {
+  public async calcDownloadedSize() {
     this.downloaded = await getFileSize(this.filepath as string);
     return this;
   }
@@ -34,80 +27,3 @@ export class CoomerFile {
     return new CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
   }
 }
-
-export class CoomerFileList {
-  public dirPath?: string;
-  public dirName?: string;
-
-  constructor(public files: CoomerFile[] = []) {}
-
-  public setDirPath(dir: string, dirName?: string) {
-    dirName = dirName || (this.dirName as string);
-
-    if (dir === './') {
-      this.dirPath = path.resolve(dir, dirName);
-    } else {
-      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
-    }
-
-    this.files.forEach((file) => {
-      const safeName = sanitizeFilename(file.name) || file.name;
-      file.filepath = path.join(this.dirPath as string, safeName);
-    });
-
-    return this;
-  }
-
-  public filterByText(include: string, exclude: string) {
-    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
-    return this;
-  }
-
-  public filterByMediaType(media?: string) {
-    if (media) {
-      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
-    }
-    return this;
-  }
-
-  public skip(n: number) {
-    this.files = this.files.slice(n);
-    return this;
-  }
-
-  public async calculateFileSizes() {
-    for (const file of this.files) {
-      await file.getDownloadedSize();
-    }
-    return this;
-  }
-
-  public getActiveFiles() {
-    return this.files.filter((f) => f.active);
-  }
-
-  public getDownloaded() {
-    return this.files.filter((f) => f.size && f.size <= f.downloaded);
-  }
-
-  public async removeDuplicatesByHash() {
-    for (const file of this.files) {
-      file.hash = await getFileHash(file.filepath);
-    }
-
-    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
-
-    console.log({ duplicates });
-
-    logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
-
-    duplicates.forEach((f) => {
-      deleteFile(f.filepath);
-    });
-  }
-
-  public removeURLDuplicates() {
-    this.files = removeDuplicatesBy(this.files, 'url');
-    return this;
-  }
-}

package/src/services/filelist.ts

@@ -0,0 +1,86 @@
+import os from 'node:os';
+import path from 'node:path';
+import logger from '../logger';
+import type { MediaType } from '../types';
+import { collectUniquesAndDuplicatesBy, removeDuplicatesBy } from '../utils/duplicates';
+import { filterString } from '../utils/filters';
+import { deleteFile, getFileHash, sanitizeFilename } from '../utils/io';
+import { testMediaType } from '../utils/mediatypes';
+import type { CoomerFile } from './file';
+
+export class CoomerFileList {
+  public dirPath?: string;
+  public dirName?: string;
+
+  constructor(public files: CoomerFile[] = []) {}
+
+  public setDirPath(dir: string, dirName?: string) {
+    dirName = dirName || (this.dirName as string);
+
+    if (dir === './') {
+      this.dirPath = path.resolve(dir, dirName);
+    } else {
+      this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
+    }
+
+    this.files.forEach((file) => {
+      const safeName = sanitizeFilename(file.name) || file.name;
+      file.filepath = path.join(this.dirPath as string, safeName);
+    });
+
+    return this;
+  }
+
+  public filterByText(include: string, exclude: string) {
+    this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
+    return this;
+  }
+
+  public filterByMediaType(media?: string) {
+    if (media) {
+      this.files = this.files.filter((f) => testMediaType(f.name, media as MediaType));
+    }
+    return this;
+  }
+
+  public skip(n: number) {
+    this.files = this.files.slice(n);
+    return this;
+  }
+
+  public async calculateFileSizes() {
+    for (const file of this.files) {
+      await file.calcDownloadedSize();
+    }
+    return this;
+  }
+
+  public getActiveFiles() {
+    return this.files.filter((f) => f.active);
+  }
+
+  public getDownloaded() {
+    return this.files.filter((f) => f.size && f.size <= f.downloaded);
+  }
+
+  public async removeDuplicatesByHash() {
+    for (const file of this.files) {
+      file.hash = await getFileHash(file.filepath);
+    }
+
+    const { duplicates } = collectUniquesAndDuplicatesBy(this.files, 'hash');
+
+    // console.log({ duplicates });
+
+    // logger.debug(`duplicates: ${JSON.stringify(duplicates)}`);
+
+    duplicates.forEach((f) => {
+      deleteFile(f.filepath);
+    });
+  }
+
+  public removeURLDuplicates() {
+    this.files = removeDuplicatesBy(this.files, 'url');
+    return this;
+  }
+}