coomer-downloader 3.0.10 → 3.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coomer-downloader",
-  "version": "3.0.10",
+  "version": "3.2.0",
   "author": "smartacephal",
   "license": "MIT",
   "description": "Downloads images/videos from Coomer/Kemono, Bunkr, GoFile, Reddit-NSFW user posts",

package/src/api/bunkr.ts

@@ -1,7 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import type { ApiResult, MediaType } from '../types/index.js';
-import { testMediaType } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../utils/file';
 
 type EncData = { url: string; timestamp: number };
 
@@ -23,46 +22,46 @@ function decryptEncryptedUrl(encryptionData: EncData) {
     .join('');
 }
 
-async function getFileData(url: string, name: string) {
+async function getFileData(url: string, name: string): Promise<CoomerFile> {
   const slug = url.split('/').pop() as string;
   const encryptionData = await getEncryptionData(slug);
   const src = decryptEncryptedUrl(encryptionData);
-  return { name, url: src };
+  return CoomerFile.from({ name, url: src });
 }
 
-async function getGalleryFiles(url: string, mediaType: MediaType) {
-  const data = [];
+async function getGalleryFiles(url: string): Promise<CoomerFileList> {
+  const filelist = new CoomerFileList();
   const page = await fetch(url).then((r) => r.text());
   const $ = cheerio.load(page);
-  const title = $('title').text();
+  const dirName = $('title').text();
+  filelist.dirName = `${dirName.split('|')[0].trim()}-bunkr`;
   const url_ = new URL(url);
 
   if (url_.pathname.startsWith('/f/')) {
     const fileName = $('h1').text();
     const singleFile = await getFileData(url, fileName);
-    data.push(singleFile);
-    return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
+    filelist.files.push(singleFile);
+    return filelist;
   }
 
   const fileNames = Array.from($('div[title]').map((_, e) => $(e).attr('title')));
 
-  const files = Array.from($('a').map((_, e) => $(e).attr('href')))
+  const data = Array.from($('a').map((_, e) => $(e).attr('href')))
     .filter((a) => /\/f\/\w+/.test(a))
     .map((a, i) => ({
       url: `${url_.origin}${a}`,
       name: fileNames[i] || (url.split('/').pop() as string),
     }));
 
-  for (const { name, url } of files) {
+  for (const { name, url } of data) {
     const res = await getFileData(url, name);
-    data.push(res);
+    filelist.files.push(res);
   }
 
-  return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
+  return filelist;
 }
 
-export async function getBunkrData(url: string, mediaType: MediaType): Promise<ApiResult> {
-  const { files, title } = await getGalleryFiles(url, mediaType);
-  const dirName = `${title.split('|')[0].trim()}-bunkr`;
-  return { dirName, files };
+export async function getBunkrData(url: string): Promise<CoomerFileList> {
+  const filelist = await getGalleryFiles(url);
+  return filelist;
 }

package/src/api/coomer-api.ts

@@ -1,15 +1,15 @@
-import type { ApiResult, File, MediaType } from '../types/index.js';
-import { fetch_, isImage, setGlobalHeaders, testMediaType } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../utils/file.js';
+import { fetchWithGlobalHeader, isImage, setGlobalHeaders } from '../utils/index.js';
 
-type CoomerUser = { domain: string; service: string; id: string; name?: string };
-type CoomerUserApi = { name: string };
-type CoomerFile = { path: string; name: string };
-type CoomerPost = {
+type CoomerAPIUser = { domain: string; service: string; id: string; name?: string };
+type CoomerAPIUserData = { name: string };
+type CoomerAPIFile = { path: string; name: string };
+type CoomerAPIPost = {
   title: string;
   content: string;
   published: string;
-  attachments: CoomerFile[];
-  file: CoomerFile;
+  attachments: CoomerAPIFile[];
+  file: CoomerAPIFile;
 };
 
 const SERVERS = ['n1', 'n2', 'n3', 'n4'];
@@ -27,19 +27,19 @@ export function tryFixCoomerUrl(url: string, attempts: number) {
   return url;
 }
 
-async function getUserProfileAPI(user: CoomerUser): Promise<CoomerUserApi> {
+async function getUserProfileData(user: CoomerAPIUser): Promise<CoomerAPIUserData> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
-  const result = await fetch_(url).then((r) => r.json());
-  return result as CoomerUserApi;
+  const result = await fetchWithGlobalHeader(url).then((r) => r.json());
+  return result as CoomerAPIUserData;
 }
 
-async function getUserPostsAPI(user: CoomerUser, offset: number): Promise<CoomerPost[]> {
+async function getUserPostsAPI(user: CoomerAPIUser, offset: number): Promise<CoomerAPIPost[]> {
   const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
-  const posts = await fetch_(url).then((r) => r.json());
-  return posts as CoomerPost[];
+  const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
+  return posts as CoomerAPIPost[];
 }
 
-export async function getUserFiles(user: CoomerUser, mediaType: MediaType): Promise<File[]> {
+export async function getUserFiles(user: CoomerAPIUser): Promise<CoomerFileList> {
   const userPosts = [];
 
   const offset = 50;
@@ -49,7 +49,7 @@ export async function getUserFiles(user: CoomerUser, mediaType: MediaType): Prom
     if (posts.length < 50) break;
   }
 
-  const files: File[] = [];
+  const filelist = new CoomerFileList();
 
   for (const p of userPosts) {
     const title = p.title.match(/\w+/g)?.join(' ') || '';
@@ -59,35 +59,34 @@ export async function getUserFiles(user: CoomerUser, mediaType: MediaType): Prom
 
     const postFiles = [...p.attachments, p.file]
       .filter((f) => f.path)
-      .filter((f) => testMediaType(f.name, mediaType))
       .map((f, i) => {
         const ext = f.name.split('.').pop();
         const name = `${datentitle} ${i + 1}.${ext}`;
         const url = `${user.domain}/${f.path}`;
-        return { name, url, content };
+        return CoomerFile.from({ name, url, content });
       });
 
-    files.push(...postFiles);
+    filelist.files.push(...postFiles);
   }
 
-  return files;
+  return filelist;
 }
 
-async function parseUser(url: string): Promise<CoomerUser> {
+async function parseUser(url: string): Promise<CoomerAPIUser> {
   const [_, domain, service, id] = url.match(
     /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/,
   ) as RegExpMatchArray;
   if (!domain || !service || !id) console.error('Invalid URL', url);
 
-  const { name } = await getUserProfileAPI({ domain, service, id });
+  const { name } = await getUserProfileData({ domain, service, id });
 
   return { domain, service, id, name };
 }
 
-export async function getCoomerData(url: string, mediaType: MediaType): Promise<ApiResult> {
+export async function getCoomerData(url: string): Promise<CoomerFileList> {
   setGlobalHeaders({ accept: 'text/css' });
   const user = await parseUser(url);
-  const dirName = `${user.name}-${user.service}`;
-  const files = await getUserFiles(user, mediaType);
-  return { dirName, files };
+  const filelist = await getUserFiles(user);
+  filelist.dirName = `${user.name}-${user.service}`;
+  return filelist;
 }

package/src/api/gofile.ts

@@ -1,6 +1,6 @@
 import { fetch } from 'undici';
-import type { ApiResult, File, MediaType } from '../types/index.js';
-import { setGlobalHeaders, testMediaType } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../utils/file.js';
+import { setGlobalHeaders } from '../utils/index.js';
 
 type GoFileAPIToken = { status: string; data: { token: string } };
 type GoFileAPIFilelist = { data: { children: { link: string; name: string }[] } };
@@ -26,7 +26,11 @@ async function getWebsiteToken() {
   throw new Error('cannot get wt');
 }
 
-async function getFolderFiles(id: string, token: string, websiteToken: string): Promise<File[]> {
+async function getFolderFiles(
+  id: string,
+  token: string,
+  websiteToken: string,
+): Promise<CoomerFileList> {
   const url = `https://api.gofile.io/contents/${id}?wt=${websiteToken}&cache=true}`;
   const response = await fetch(url, {
     headers: {
@@ -39,26 +43,26 @@ async function getFolderFiles(id: string, token: string, websiteToken: string):
   }
 
   const data = (await response.json()) as GoFileAPIFilelist;
-  const files = Object.values(data.data.children).map((f) => ({
-    url: f.link,
-    name: f.name,
-  }));
+  const files = Object.values(data.data.children).map((f) =>
+    CoomerFile.from({
+      url: f.link,
+      name: f.name,
+    }),
+  );
 
-  return files;
+  return new CoomerFileList(files);
 }
 
-export async function getGofileData(url: string, mediaType: MediaType): Promise<ApiResult> {
+export async function getGofileData(url: string): Promise<CoomerFileList> {
   const id = url.match(/gofile.io\/d\/(\w+)/)?.[1] as string;
-  const dirName = `gofile-${id}`;
 
   const token = await getToken();
   const websiteToken = await getWebsiteToken();
 
-  const files = (await getFolderFiles(id, token, websiteToken)).filter((f) =>
-    testMediaType(f.name, mediaType),
-  );
+  const filelist = await getFolderFiles(id, token, websiteToken);
+  filelist.dirName = `gofile-${id}`;
 
   setGlobalHeaders({ Cookie: `accountToken=${token}` });
 
-  return { dirName, files };
+  return filelist;
 }

package/src/api/index.ts

@@ -1,28 +1,28 @@
-import type { ApiResult, MediaType } from '../types';
+import type { CoomerFileList } from '../utils/file';
 import { getBunkrData } from './bunkr';
 import { getCoomerData } from './coomer-api';
 import { getGofileData } from './gofile';
 import { getRedditData } from './nsfw.xxx';
 import { getPlainFileData } from './plain-curl';
 
-export async function apiHandler(
-  url: string,
-  mediaType: MediaType,
-): Promise<ApiResult | undefined> {
-  if (/^u\/\w+$/.test(url.trim())) {
-    return getRedditData(url, mediaType);
+export async function apiHandler(url_: string): Promise<CoomerFileList> {
+  const url = new URL(url_);
+
+  if (/^u\/\w+$/.test(url.origin)) {
+    return getRedditData(url.href);
   }
-  if (/coomer|kemono/.test(url)) {
-    return getCoomerData(url, mediaType);
+  if (/coomer|kemono/.test(url.origin)) {
+    return getCoomerData(url.href);
   }
-  if (/bunkr/.test(url)) {
-    return getBunkrData(url, mediaType);
+  if (/bunkr/.test(url.origin)) {
+    return getBunkrData(url.href);
   }
-  if (/gofile\.io/.test(url)) {
-    return getGofileData(url, mediaType);
+  if (/gofile\.io/.test(url.origin)) {
+    return getGofileData(url.href);
   }
-  if (/\.\w+/.test(url.split('/').pop() as string)) {
-    return getPlainFileData(url);
+  if (/\.\w+/.test(url.pathname)) {
+    return getPlainFileData(url.href);
   }
-  console.error('Wrong URL.');
+
+  throw Error('Invalid URL');
 }

package/src/api/nsfw.xxx.ts

@@ -1,7 +1,6 @@
 import * as cheerio from 'cheerio';
 import { fetch } from 'undici';
-import type { ApiResult, File, MediaType } from '../types/index.js';
-import { testMediaType } from '../utils/index.js';
+import { CoomerFile, CoomerFileList } from '../utils/file';
 
 async function getUserPage(user: string, offset: number) {
   const url = `https://nsfw.xxx/page/${offset}?nsfw[]=0&types[]=image&types[]=video&types[]=gallery&slider=1&jsload=1&user=${user}&_=${Date.now()}`;
@@ -26,9 +25,9 @@ async function getUserPosts(user: string): Promise<string[]> {
   return posts;
 }
 
-async function getPostsData(posts: string[], mediaType: MediaType): Promise<File[]> {
+async function getPostsData(posts: string[]): Promise<CoomerFileList> {
   console.log('Fetching posts data...');
-  const data = [];
+  const filelist = new CoomerFileList();
   for (const post of posts) {
     const page = await fetch(post).then((r) => r.text());
     const $ = cheerio.load(page);
@@ -46,16 +45,16 @@ async function getPostsData(posts: string[], mediaType: MediaType): Promise<File
     const ext = src.split('.').pop();
     const name = `${slug}-${date}.${ext}`;
 
-    data.push({ name, url: src });
+    filelist.files.push(CoomerFile.from({ name, url: src }));
   }
 
-  return data.filter((f) => testMediaType(f.name, mediaType));
+  return filelist;
 }
 
-export async function getRedditData(url: string, mediaType: MediaType): Promise<ApiResult> {
+export async function getRedditData(url: string): Promise<CoomerFileList> {
   const user = url.match(/u\/(\w+)/)?.[1] as string;
   const posts = await getUserPosts(user);
-  const files = await getPostsData(posts, mediaType);
-  const dirName = `${user}-reddit`;
-  return { dirName, files };
+  const filelist = await getPostsData(posts);
+  filelist.dirName = `${user}-reddit`;
+  return filelist;
 }

package/src/api/plain-curl.ts

@@ -1,13 +1,9 @@
-import type { ApiResult } from '../types';
+import { CoomerFile, CoomerFileList } from '../utils/file';
 
-export async function getPlainFileData(url: string): Promise<ApiResult> {
-  return {
-    dirName: '',
-    files: [
-      {
-        name: url.split('/').pop() as string,
-        url,
-      },
-    ],
-  };
+export async function getPlainFileData(url: string): Promise<CoomerFileList> {
+  const name = url.split('/').pop() as string;
+  const file = CoomerFile.from({ name, url });
+  const filelist = new CoomerFileList([file]);
+  filelist.dirName = '';
+  return filelist;
 }

package/src/args-handler.ts

@@ -1,16 +1,13 @@
 import yargs from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-export type ArgumentHandlerResult = {
-  [x: string]: unknown;
+type ArgumentHandlerResult = {
   url: string;
   dir: string;
   media: string;
   include: string;
   exclude: string;
   skip: number;
-  _: (string | number)[];
-  $0: string;
 };
 
 export function argumentHander(): ArgumentHandlerResult {

package/src/index.ts

@@ -1,35 +1,34 @@
 #!/usr/bin/env node
-import os from 'node:os';
-import path from 'node:path';
 import process from 'node:process';
 import { apiHandler } from './api';
 import { argumentHander } from './args-handler';
-import type { ApiResult, MediaType } from './types';
-import { createMultibar, downloadFiles, filterKeywords, setGlobalHeaders } from './utils';
+import { createMultibar, Downloader, setGlobalHeaders } from './utils';
 
 async function run() {
   const { url, dir, media, include, exclude, skip } = argumentHander();
 
-  const { dirName, files } = (await apiHandler(url, media as MediaType)) as ApiResult;
+  const filelist = await apiHandler(url);
 
-  const downloadDir =
-    dir === './' ? path.resolve(dir, dirName) : path.join(os.homedir(), path.join(dir, dirName));
-
-  const filteredFiles = filterKeywords(files, include, exclude).slice(skip);
+  const found = filelist.files.length;
+  filelist.setDirPath(dir);
+  filelist.skip(skip);
+  filelist.filterByText(include, exclude);
+  filelist.filterByMediaType(media);
 
   console.table([
     {
-      found: files.length,
+      found,
       skip,
-      filtered: files.length - filteredFiles.length - skip,
-      folder: downloadDir,
+      filtered: found - filelist.files.length,
+      folder: filelist.dirPath,
     },
   ]);
 
   setGlobalHeaders({ Referer: url });
 
-  createMultibar();
-  await downloadFiles(filteredFiles, downloadDir);
+  const downloader = new Downloader();
+  createMultibar(downloader);
+  await downloader.downloadFiles(filelist);
 
   process.kill(process.pid, 'SIGINT');
 }

package/src/types/index.ts

@@ -1,23 +1 @@
-export type File = {
-  name: string;
-  url: string;
-  filepath?: string;
-  size?: number;
-  downloaded?: number;
-  content?: string;
-};
-
-export type DownloaderSubject = {
-  type: string;
-  file?: File;
-  index?: number;
-  filesCount?: number;
-};
-
-export type ApiResult = {
-  files: File[];
-  // should merge into filepaths?
-  dirName: string;
-};
-
 export type MediaType = 'video' | 'image' | 'all';

package/src/utils/downloader.ts

@@ -1,60 +1,59 @@
 import fs from 'node:fs';
-import path from 'node:path';
 import { Readable, Transform } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
 import { Subject } from 'rxjs';
 import { tryFixCoomerUrl } from '../api/coomer-api';
-import type { DownloaderSubject, File } from '../types';
-import { getFileSize, mkdir } from './files';
+import type { CoomerFile, CoomerFileList } from './file';
+import { getFileSize, mkdir } from './io';
+import { PromiseRetry } from './promise';
 import { fetchByteRange } from './requests';
 import { Timer } from './timer';
 
-export const subject = new Subject<DownloaderSubject>();
-
-const CHUNK_TIMEOUT = 30_000;
-const DOWNLOAD_ATTEMPTS = 7;
-
-async function downloadStream(file: File, stream: Readable): Promise<void> {
-  subject.next({ type: 'CHUNK_DOWNLOADING_STARTED', file });
-
-  const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
-
-  const controller = new AbortController();
-  const timer = new Timer(CHUNK_TIMEOUT, () => {
-    controller.abort('Stream is stuck.');
-  }).start();
-
-  const progressStream = new Transform({
-    transform(chunk, _encoding, callback) {
-      this.push(chunk);
-      file.downloaded += chunk.length;
-      timer.reset();
-      subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE', file });
-      callback();
-    },
-  });
-
-  await pipeline(stream, progressStream, fileStream, { signal: controller.signal });
-  timer.stop();
-  subject.next({ type: 'FILE_DOWNLOADING_FINISHED' });
-}
-
-function handleFetchError(error: Error, file: File, attempts: number): void {
-  const url = file?.url as string;
-  if (/coomer|kemono/.test(url)) {
-    file.url = tryFixCoomerUrl(url, attempts);
+type DownloaderSubject = {
+  type: string;
+  file?: CoomerFile;
+  filesCount?: number;
+};
+
+export class Downloader {
+  public subject = new Subject<DownloaderSubject>();
+
+  constructor(
+    public chunkTimeout = 30_000,
+    public chunkFetchRetries = 5,
+    public fetchRetries = 7,
+  ) {}
+
+  async fetchStream(file: CoomerFile, stream: Readable): Promise<void> {
+    const { subject, chunkTimeout } = this;
+    const { timer, signal } = Timer.withSignal(chunkTimeout, 'chunkTimeout');
+
+    const fileStream = fs.createWriteStream(file.filepath as string, { flags: 'a' });
+
+    const progressStream = new Transform({
+      transform(chunk, _encoding, callback) {
+        this.push(chunk);
+        file.downloaded += chunk.length;
+        timer.reset();
+        subject.next({ type: 'CHUNK_DOWNLOADING_UPDATE', file });
+        callback();
+      },
+    });
+
+    try {
+      subject.next({ type: 'CHUNK_DOWNLOADING_START', file });
+      await pipeline(stream, progressStream, fileStream, { signal });
+    } catch (error) {
+      console.error((error as Error).name === 'AbortError' ? signal.reason : error);
+    } finally {
+      subject.next({ type: 'CHUNK_DOWNLOADING_END', file });
+    }
   }
-  throw error;
-}
 
-async function downloadFile(file: File, attempts = DOWNLOAD_ATTEMPTS): Promise<void> {
-  const downloadedOld = file.downloaded || 0;
-  try {
+  async downloadFile(file: CoomerFile): Promise<void> {
     file.downloaded = await getFileSize(file.filepath as string);
 
-    const response = await fetchByteRange(file.url, file.downloaded).catch((error) =>
-      handleFetchError(error, file, --attempts),
-    );
+    const response = await fetchByteRange(file.url, file.downloaded);
 
     if (!response?.ok && response?.status !== 416) {
       throw new Error(`HTTP error! status: ${response?.status}`);
@@ -62,40 +61,48 @@ async function downloadFile(file: File, attempts = DOWNLOAD_ATTEMPTS): Promise<v
 
     const contentLength = response.headers.get('Content-Length') as string;
 
-    if (!contentLength && file.downloaded > 0) {
-      return;
-    }
+    if (!contentLength && file.downloaded > 0) return;
 
     const restFileSize = parseInt(contentLength);
     file.size = restFileSize + file.downloaded;
 
     if (file.size > file.downloaded && response.body) {
       const stream = Readable.fromWeb(response.body);
-      await downloadStream(file, stream);
-    }
-  } catch (error) {
-    if (downloadedOld < (file.downloaded || 0)) {
-      attempts = DOWNLOAD_ATTEMPTS;
-    }
-    if (attempts < 1) {
-      console.error(file.url);
-      console.error(error);
-    } else {
-      await downloadFile(file, attempts - 1);
+      const sizeOld = file.downloaded;
+
+      await PromiseRetry.create({
+        retries: this.chunkFetchRetries,
+        callback: () => {
+          if (sizeOld !== file.downloaded) {
+            return { newRetries: 5 };
+          }
+        },
+      }).execute(async () => await this.fetchStream(file, stream));
     }
+
+    this.subject.next({ type: 'FILE_DOWNLOADING_END' });
   }
-}
 
-export async function downloadFiles(data: File[], downloadDir: string): Promise<void> {
-  mkdir(downloadDir);
+  async downloadFiles(filelist: CoomerFileList): Promise<void> {
+    mkdir(filelist.dirPath as string);
 
-  subject.next({ type: 'FILES_DOWNLOADING_STARTED', filesCount: data.length });
+    this.subject.next({ type: 'FILES_DOWNLOADING_START', filesCount: filelist.files.length });
 
-  for (const [index, file] of data.entries()) {
-    file.filepath = path.join(downloadDir, file.name);
-    subject.next({ type: 'FILE_DOWNLOADING_STARTED', index });
-    await downloadFile(file);
-  }
+    for (const file of filelist.files) {
+      this.subject.next({ type: 'FILE_DOWNLOADING_START' });
+
+      await PromiseRetry.create({
+        retries: this.fetchRetries,
+        callback: (retries) => {
+          if (/coomer|kemono/.test(file.url)) {
+            file.url = tryFixCoomerUrl(file.url, retries);
+          }
+        },
+      }).execute(async () => await this.downloadFile(file));
 
-  subject.next({ type: 'FILES_DOWNLOADING_STARTED' });
+      this.subject.next({ type: 'FILE_DOWNLOADING_END' });
+    }
+
+    this.subject.next({ type: 'FILES_DOWNLOADING_END' });
+  }
 }