coomer-downloader 3.1.0 → 3.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,86 +1,42 @@
1
- #!/usr/bin/env node
1
+ #!/usr/bin/env -S node --no-warnings=ExperimentalWarning
2
2
 
3
3
  // src/index.ts
4
- import os from "node:os";
5
- import path2 from "node:path";
6
4
  import process2 from "node:process";
7
5
 
8
6
  // src/api/bunkr.ts
9
7
  import * as cheerio from "cheerio";
10
- import { fetch as fetch2 } from "undici";
8
+ import { fetch } from "undici";
11
9
 
12
- // src/utils/downloader.ts
13
- import fs2 from "node:fs";
10
+ // src/services/file.ts
11
+ import os from "node:os";
14
12
  import path from "node:path";
15
- import { Readable, Transform } from "node:stream";
16
- import { pipeline } from "node:stream/promises";
17
- import { Subject } from "rxjs";
18
13
 
19
- // src/api/coomer-api.ts
20
- var SERVERS = ["n1", "n2", "n3", "n4"];
21
- function tryFixCoomerUrl(url, attempts) {
22
- if (attempts < 2 && isImage(url)) {
23
- return url.replace(/\/data\//, "/thumbnail/data/").replace(/n\d\./, "img.");
24
- }
25
- const server = url.match(/n\d\./)?.[0].slice(0, 2);
26
- const i = SERVERS.indexOf(server);
27
- if (i !== -1) {
28
- const newServer = SERVERS[(i + 1) % SERVERS.length];
29
- return url.replace(/n\d./, `${newServer}.`);
30
- }
31
- return url;
14
+ // src/utils/filters.ts
15
+ function isImage(name) {
16
+ return /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
32
17
  }
33
- async function getUserProfileAPI(user) {
34
- const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
35
- const result = await fetchWithGlobalHeader(url).then((r) => r.json());
36
- return result;
18
+ function isVideo(name) {
19
+ return /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
37
20
  }
38
- async function getUserPostsAPI(user, offset) {
39
- const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
40
- const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
41
- return posts;
21
+ function testMediaType(name, type) {
22
+ return type === "all" ? true : type === "image" ? isImage(name) : isVideo(name);
42
23
  }
43
- async function getUserFiles(user, mediaType) {
44
- const userPosts = [];
45
- const offset = 50;
46
- for (let i = 0; i < 1e3; i++) {
47
- const posts = await getUserPostsAPI(user, i * offset);
48
- userPosts.push(...posts);
49
- if (posts.length < 50) break;
50
- }
51
- const files = [];
52
- for (const p of userPosts) {
53
- const title = p.title.match(/\w+/g)?.join(" ") || "";
54
- const content = p.content;
55
- const date = p.published.replace(/T/, " ");
56
- const datentitle = `${date} ${title}`.trim();
57
- const postFiles = [...p.attachments, p.file].filter((f) => f.path).filter((f) => testMediaType(f.name, mediaType)).map((f, i) => {
58
- const ext = f.name.split(".").pop();
59
- const name = `${datentitle} ${i + 1}.${ext}`;
60
- const url = `${user.domain}/${f.path}`;
61
- return { name, url, content };
62
- });
63
- files.push(...postFiles);
64
- }
65
- return files;
24
+ function includesAllWords(str, words) {
25
+ if (!words.length) return true;
26
+ return words.every((w) => str.includes(w));
66
27
  }
67
- async function parseUser(url) {
68
- const [_, domain, service, id] = url.match(
69
- /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/
70
- );
71
- if (!domain || !service || !id) console.error("Invalid URL", url);
72
- const { name } = await getUserProfileAPI({ domain, service, id });
73
- return { domain, service, id, name };
28
+ function includesNoWords(str, words) {
29
+ if (!words.length) return true;
30
+ return words.every((w) => !str.includes(w));
74
31
  }
75
- async function getCoomerData(url, mediaType) {
76
- setGlobalHeaders({ accept: "text/css" });
77
- const user = await parseUser(url);
78
- const dirName = `${user.name}-${user.service}`;
79
- const files = await getUserFiles(user, mediaType);
80
- return { dirName, files };
32
+ function parseQuery(query) {
33
+ return query.split(",").map((x) => x.toLowerCase().trim()).filter((_) => _);
34
+ }
35
+ function filterString(text, include, exclude) {
36
+ return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
81
37
  }
82
38
 
83
- // src/utils/files.ts
39
+ // src/utils/io.ts
84
40
  import fs from "node:fs";
85
41
  async function getFileSize(filepath) {
86
42
  let size = 0;
@@ -95,281 +51,77 @@ function mkdir(filepath) {
95
51
  }
96
52
  }
97
53
 
98
- // src/utils/promise.ts
99
- async function sleep(time) {
100
- return new Promise((resolve) => setTimeout(resolve, time));
101
- }
102
- var PromiseRetry = class _PromiseRetry {
103
- retries;
104
- delay;
105
- callback;
106
- constructor(options) {
107
- this.retries = options.retries || 3;
108
- this.delay = options.delay || 1e3;
109
- this.callback = options.callback;
110
- }
111
- async execute(fn) {
112
- let retries = this.retries;
113
- while (true) {
114
- try {
115
- return await fn();
116
- } catch (error) {
117
- if (retries <= 0) {
118
- throw error;
119
- }
120
- if (this.callback) {
121
- const res = this.callback(retries, error);
122
- if (res) {
123
- const { newRetries } = res;
124
- if (newRetries === 0) throw error;
125
- this.retries = newRetries || retries;
126
- }
127
- }
128
- await sleep(this.delay);
129
- retries--;
130
- }
131
- }
54
+ // src/services/file.ts
55
+ var CoomerFile = class _CoomerFile {
56
+ constructor(name, url, filepath, size, downloaded = 0, content) {
57
+ this.name = name;
58
+ this.url = url;
59
+ this.filepath = filepath;
60
+ this.size = size;
61
+ this.downloaded = downloaded;
62
+ this.content = content;
63
+ }
64
+ active = false;
65
+ async getDownloadedSize() {
66
+ this.downloaded = await getFileSize(this.filepath);
67
+ return this;
68
+ }
69
+ get textContent() {
70
+ const text = `${this.name || ""} ${this.content || ""}`.toLowerCase();
71
+ return text;
132
72
  }
133
- static create(options) {
134
- return new _PromiseRetry(options);
73
+ static from(f) {
74
+ return new _CoomerFile(f.name, f.url, f.filepath, f.size, f.downloaded, f.content);
135
75
  }
136
76
  };
137
-
138
- // src/utils/requests.ts
139
- import { CookieAgent } from "http-cookie-agent/undici";
140
- import { CookieJar } from "tough-cookie";
141
- import { fetch, interceptors, setGlobalDispatcher } from "undici";
142
- function setCookieJarDispatcher() {
143
- const jar = new CookieJar();
144
- const agent = new CookieAgent({ cookies: { jar } }).compose(interceptors.retry()).compose(interceptors.redirect({ maxRedirections: 3 }));
145
- setGlobalDispatcher(agent);
146
- }
147
- setCookieJarDispatcher();
148
- var HeadersDefault = new Headers({
149
- accept: "application/json, text/css",
150
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
151
- });
152
- function setGlobalHeaders(headers) {
153
- Object.keys(headers).forEach((k) => {
154
- HeadersDefault.set(k, headers[k]);
155
- });
156
- }
157
- function fetchWithGlobalHeader(url) {
158
- const requestHeaders = new Headers(HeadersDefault);
159
- return fetch(url, { headers: requestHeaders });
160
- }
161
- function fetchByteRange(url, downloadedSize) {
162
- const requestHeaders = new Headers(HeadersDefault);
163
- requestHeaders.set("Range", `bytes=${downloadedSize}-`);
164
- return fetch(url, { headers: requestHeaders });
165
- }
166
-
167
- // src/utils/timer.ts
168
- var Timer = class _Timer {
169
- constructor(timeout = 1e4, timeoutCallback) {
170
- this.timeout = timeout;
171
- this.timeoutCallback = timeoutCallback;
172
- this.timeout = timeout;
77
+ var CoomerFileList = class {
78
+ constructor(files = []) {
79
+ this.files = files;
173
80
  }
174
- timer = void 0;
175
- start() {
176
- this.timer = setTimeout(() => {
177
- this.stop();
178
- this.timeoutCallback();
179
- }, this.timeout);
81
+ dirPath;
82
+ dirName;
83
+ setDirPath(dir, dirName) {
84
+ dirName = dirName || this.dirName;
85
+ if (dir === "./") {
86
+ this.dirPath = path.resolve(dir, dirName);
87
+ } else {
88
+ this.dirPath = path.join(os.homedir(), path.join(dir, dirName));
89
+ }
90
+ this.files.forEach((file) => {
91
+ file.filepath = path.join(this.dirPath, file.name);
92
+ });
180
93
  return this;
181
94
  }
182
- stop() {
183
- if (this.timer) {
184
- clearTimeout(this.timer);
185
- this.timer = void 0;
186
- }
95
+ filterByText(include, exclude) {
96
+ this.files = this.files.filter((f) => filterString(f.textContent, include, exclude));
187
97
  return this;
188
98
  }
189
- reset() {
190
- this.stop();
191
- this.start();
99
+ filterByMediaType(media) {
100
+ if (media) {
101
+ this.files = this.files.filter((f) => testMediaType(f.name, media));
102
+ }
192
103
  return this;
193
104
  }
194
- static withSignal(timeout, message) {
195
- const controller = new AbortController();
196
- const callback = () => {
197
- controller.abort(message);
198
- };
199
- const timer = new _Timer(timeout, callback).start();
200
- return {
201
- timer,
202
- signal: controller.signal
203
- };
105
+ skip(n) {
106
+ this.files = this.files.slice(n);
107
+ return this;
204
108
  }
205
- };
206
-
207
- // src/utils/downloader.ts
208
- var subject = new Subject();
209
- var CHUNK_TIMEOUT = 3e4;
210
- var CHUNK_FETCH_RETRIES = 5;
211
- var FETCH_RETRIES = 7;
212
- async function fetchStream(file, stream) {
213
- const { timer, signal } = Timer.withSignal(CHUNK_TIMEOUT, "CHUNK_TIMEOUT");
214
- const fileStream = fs2.createWriteStream(file.filepath, { flags: "a" });
215
- const progressStream = new Transform({
216
- transform(chunk, _encoding, callback) {
217
- this.push(chunk);
218
- file.downloaded += chunk.length;
219
- timer.reset();
220
- subject.next({ type: "CHUNK_DOWNLOADING_UPDATE", file });
221
- callback();
109
+ async calculateFileSizes() {
110
+ for (const file of this.files) {
111
+ await file.getDownloadedSize();
222
112
  }
223
- });
224
- try {
225
- subject.next({ type: "CHUNK_DOWNLOADING_START", file });
226
- await pipeline(stream, progressStream, fileStream, { signal });
227
- } catch (error) {
228
- console.error(error.name === "AbortError" ? signal.reason : error);
229
- } finally {
230
- subject.next({ type: "CHUNK_DOWNLOADING_END", file });
231
- }
232
- }
233
- async function downloadFile(file) {
234
- file.downloaded = await getFileSize(file.filepath);
235
- const response = await fetchByteRange(file.url, file.downloaded);
236
- if (!response?.ok && response?.status !== 416) {
237
- throw new Error(`HTTP error! status: ${response?.status}`);
238
- }
239
- const contentLength = response.headers.get("Content-Length");
240
- if (!contentLength && file.downloaded > 0) {
241
- return;
242
- }
243
- const restFileSize = parseInt(contentLength);
244
- file.size = restFileSize + file.downloaded;
245
- if (file.size > file.downloaded && response.body) {
246
- const stream = Readable.fromWeb(response.body);
247
- const sizeOld = file.downloaded;
248
- await PromiseRetry.create({
249
- retries: CHUNK_FETCH_RETRIES,
250
- callback: () => {
251
- if (sizeOld !== file.downloaded) {
252
- return { newRetries: 5 };
253
- }
254
- }
255
- }).execute(async () => await fetchStream(file, stream));
256
- }
257
- subject.next({ type: "FILE_DOWNLOADING_END" });
258
- }
259
- async function downloadFiles(data, downloadDir) {
260
- mkdir(downloadDir);
261
- subject.next({ type: "FILES_DOWNLOADING_START", filesCount: data.length });
262
- for (const [_, file] of data.entries()) {
263
- file.filepath = path.join(downloadDir, file.name);
264
- subject.next({ type: "FILE_DOWNLOADING_START" });
265
- await PromiseRetry.create({
266
- retries: FETCH_RETRIES,
267
- callback: (retries) => {
268
- if (/coomer|kemono/.test(file.url)) {
269
- file.url = tryFixCoomerUrl(file.url, retries);
270
- }
271
- }
272
- }).execute(async () => await downloadFile(file));
273
- subject.next({ type: "FILE_DOWNLOADING_END" });
274
113
  }
275
- subject.next({ type: "FILES_DOWNLOADING_END" });
276
- }
277
-
278
- // src/utils/filters.ts
279
- var isImage = (name) => /\.(jpg|jpeg|png|gif|bmp|tiff|webp|avif)$/i.test(name);
280
- var isVideo = (name) => /\.(mp4|m4v|avi|mov|mkv|webm|flv|wmv|mpeg|mpg|3gp)$/i.test(name);
281
- var testMediaType = (name, type) => type === "all" ? true : type === "image" ? isImage(name) : isVideo(name);
282
- function includesAllWords(str, words) {
283
- if (!words.length) return true;
284
- return words.every((w) => str.includes(w));
285
- }
286
- function includesNoWords(str, words) {
287
- if (!words.length) return true;
288
- return words.every((w) => !str.includes(w));
289
- }
290
- function parseQuery(query) {
291
- return query.split(",").map((x) => x.toLowerCase().trim()).filter((_) => _);
292
- }
293
- function filterString(text, include, exclude) {
294
- return includesAllWords(text, parseQuery(include)) && includesNoWords(text, parseQuery(exclude));
295
- }
296
- function filterKeywords(files, include, exclude) {
297
- return files.filter((f) => {
298
- const text = `${f.name || ""} ${f.content || ""}`.toLowerCase();
299
- return filterString(text, include, exclude);
300
- });
301
- }
302
-
303
- // src/utils/multibar.ts
304
- import { MultiBar } from "cli-progress";
305
-
306
- // src/utils/strings.ts
307
- function b2mb(bytes) {
308
- return Number.parseFloat((bytes / 1048576).toFixed(2));
309
- }
310
- function formatNameStdout(pathname) {
311
- const name = pathname.split("/").pop() || "";
312
- const consoleWidth = process.stdout.columns;
313
- const width = Math.max(consoleWidth / 2 | 0, 40);
314
- if (name.length < width) return name.trim();
315
- const result = `${name.slice(0, width - 15)} ... ${name.slice(-10)}`.replace(/ +/g, " ");
316
- return result;
317
- }
318
-
319
- // src/utils/multibar.ts
320
- var config = {
321
- clearOnComplete: true,
322
- gracefulExit: true,
323
- autopadding: true,
324
- hideCursor: true,
325
- format: "{percentage}% | {filename} | {value}/{total}{size}"
114
+ getActiveFiles() {
115
+ return this.files.filter((f) => f.active);
116
+ }
117
+ getDownloaded() {
118
+ return this.files.filter((f) => f.size && f.size <= f.downloaded);
119
+ }
326
120
  };
327
- function createMultibar() {
328
- const multibar = new MultiBar(config);
329
- let bar;
330
- let minibar;
331
- let filename;
332
- let index = 0;
333
- subject.subscribe({
334
- next: ({ type, filesCount, file }) => {
335
- switch (type) {
336
- case "FILES_DOWNLOADING_START":
337
- bar?.stop();
338
- bar = multibar.create(filesCount, 0);
339
- break;
340
- case "FILES_DOWNLOADING_END":
341
- bar?.stop();
342
- break;
343
- case "FILE_DOWNLOADING_START":
344
- bar?.update(++index, { filename: "Downloaded files", size: "" });
345
- break;
346
- case "FILE_DOWNLOADING_END":
347
- multibar.remove(minibar);
348
- break;
349
- case "CHUNK_DOWNLOADING_START":
350
- multibar?.remove(minibar);
351
- filename = formatNameStdout(file?.filepath);
352
- minibar = multibar.create(b2mb(file?.size), b2mb(file?.downloaded));
353
- break;
354
- case "CHUNK_DOWNLOADING_UPDATE":
355
- minibar?.update(b2mb(file?.downloaded), {
356
- filename,
357
- size: "mb"
358
- });
359
- break;
360
- case "CHUNK_DOWNLOADING_END":
361
- multibar?.remove(minibar);
362
- break;
363
- default:
364
- break;
365
- }
366
- }
367
- });
368
- }
369
121
 
370
122
  // src/api/bunkr.ts
371
123
  async function getEncryptionData(slug) {
372
- const response = await fetch2("https://bunkr.cr/api/vs", {
124
+ const response = await fetch("https://bunkr.cr/api/vs", {
373
125
  method: "POST",
374
126
  headers: { "Content-Type": "application/json" },
375
127
  body: JSON.stringify({ slug })
@@ -386,35 +138,128 @@ async function getFileData(url, name) {
386
138
  const slug = url.split("/").pop();
387
139
  const encryptionData = await getEncryptionData(slug);
388
140
  const src = decryptEncryptedUrl(encryptionData);
389
- return { name, url: src };
141
+ return CoomerFile.from({ name, url: src });
390
142
  }
391
- async function getGalleryFiles(url, mediaType) {
392
- const data = [];
393
- const page = await fetch2(url).then((r) => r.text());
143
+ async function getGalleryFiles(url) {
144
+ const filelist = new CoomerFileList();
145
+ const page = await fetch(url).then((r) => r.text());
394
146
  const $ = cheerio.load(page);
395
- const title = $("title").text();
147
+ const dirName = $("title").text();
148
+ filelist.dirName = `${dirName.split("|")[0].trim()}-bunkr`;
396
149
  const url_ = new URL(url);
397
150
  if (url_.pathname.startsWith("/f/")) {
398
151
  const fileName = $("h1").text();
399
152
  const singleFile = await getFileData(url, fileName);
400
- data.push(singleFile);
401
- return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
153
+ filelist.files.push(singleFile);
154
+ return filelist;
402
155
  }
403
156
  const fileNames = Array.from($("div[title]").map((_, e) => $(e).attr("title")));
404
- const files = Array.from($("a").map((_, e) => $(e).attr("href"))).filter((a) => /\/f\/\w+/.test(a)).map((a, i) => ({
157
+ const data = Array.from($("a").map((_, e) => $(e).attr("href"))).filter((a) => /\/f\/\w+/.test(a)).map((a, i) => ({
405
158
  url: `${url_.origin}${a}`,
406
159
  name: fileNames[i] || url.split("/").pop()
407
160
  }));
408
- for (const { name, url: url2 } of files) {
161
+ for (const { name, url: url2 } of data) {
409
162
  const res = await getFileData(url2, name);
410
- data.push(res);
163
+ filelist.files.push(res);
164
+ }
165
+ return filelist;
166
+ }
167
+ async function getBunkrData(url) {
168
+ const filelist = await getGalleryFiles(url);
169
+ return filelist;
170
+ }
171
+
172
+ // src/utils/requests.ts
173
+ import { CookieAgent } from "http-cookie-agent/undici";
174
+ import { CookieJar } from "tough-cookie";
175
+ import { fetch as fetch2, interceptors, setGlobalDispatcher } from "undici";
176
+ function setCookieJarDispatcher() {
177
+ const jar = new CookieJar();
178
+ const agent = new CookieAgent({ cookies: { jar } }).compose(interceptors.retry()).compose(interceptors.redirect({ maxRedirections: 3 }));
179
+ setGlobalDispatcher(agent);
180
+ }
181
+ setCookieJarDispatcher();
182
+ var HeadersDefault = new Headers({
183
+ accept: "application/json, text/css",
184
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
185
+ });
186
+ function setGlobalHeaders(headers) {
187
+ Object.keys(headers).forEach((k) => {
188
+ HeadersDefault.set(k, headers[k]);
189
+ });
190
+ }
191
+ function fetchWithGlobalHeader(url) {
192
+ const requestHeaders = new Headers(HeadersDefault);
193
+ return fetch2(url, { headers: requestHeaders });
194
+ }
195
+ function fetchByteRange(url, downloadedSize, signal) {
196
+ const requestHeaders = new Headers(HeadersDefault);
197
+ requestHeaders.set("Range", `bytes=${downloadedSize}-`);
198
+ return fetch2(url, { headers: requestHeaders, signal });
199
+ }
200
+
201
+ // src/api/coomer-api.ts
202
+ var SERVERS = ["n1", "n2", "n3", "n4"];
203
+ function tryFixCoomerUrl(url, attempts) {
204
+ if (attempts < 2 && isImage(url)) {
205
+ return url.replace(/\/data\//, "/thumbnail/data/").replace(/n\d\./, "img.");
206
+ }
207
+ const server = url.match(/n\d\./)?.[0].slice(0, 2);
208
+ const i = SERVERS.indexOf(server);
209
+ if (i !== -1) {
210
+ const newServer = SERVERS[(i + 1) % SERVERS.length];
211
+ return url.replace(/n\d./, `${newServer}.`);
411
212
  }
412
- return { title, files: data.filter((f) => testMediaType(f.name, mediaType)) };
213
+ return url;
413
214
  }
414
- async function getBunkrData(url, mediaType) {
415
- const { files, title } = await getGalleryFiles(url, mediaType);
416
- const dirName = `${title.split("|")[0].trim()}-bunkr`;
417
- return { dirName, files };
215
+ async function getUserProfileData(user) {
216
+ const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/profile`;
217
+ const result = await fetchWithGlobalHeader(url).then((r) => r.json());
218
+ return result;
219
+ }
220
+ async function getUserPostsAPI(user, offset) {
221
+ const url = `${user.domain}/api/v1/${user.service}/user/${user.id}/posts?o=${offset}`;
222
+ const posts = await fetchWithGlobalHeader(url).then((r) => r.json());
223
+ return posts;
224
+ }
225
+ async function getUserFiles(user) {
226
+ const userPosts = [];
227
+ const offset = 50;
228
+ for (let i = 0; i < 1e3; i++) {
229
+ const posts = await getUserPostsAPI(user, i * offset);
230
+ userPosts.push(...posts);
231
+ if (posts.length < 50) break;
232
+ }
233
+ const filelist = new CoomerFileList();
234
+ for (const p of userPosts) {
235
+ const title = p.title.match(/\w+/g)?.join(" ") || "";
236
+ const content = p.content;
237
+ const date = p.published.replace(/T/, " ");
238
+ const datentitle = `${date} ${title}`.trim();
239
+ const postFiles = [...p.attachments, p.file].filter((f) => f.path).map((f, i) => {
240
+ const ext = f.name.split(".").pop();
241
+ const name = `${datentitle} ${i + 1}.${ext}`;
242
+ const url = `${user.domain}/${f.path}`;
243
+ return CoomerFile.from({ name, url, content });
244
+ });
245
+ filelist.files.push(...postFiles);
246
+ }
247
+ return filelist;
248
+ }
249
+ async function parseUser(url) {
250
+ const [_, domain, service, id] = url.match(
251
+ /(https:\/\/\w+\.\w+)\/(\w+)\/user\/([\w|.|-]+)/
252
+ );
253
+ if (!domain || !service || !id) console.error("Invalid URL", url);
254
+ const { name } = await getUserProfileData({ domain, service, id });
255
+ return { domain, service, id, name };
256
+ }
257
+ async function getCoomerData(url) {
258
+ setGlobalHeaders({ accept: "text/css" });
259
+ const user = await parseUser(url);
260
+ const filelist = await getUserFiles(user);
261
+ filelist.dirName = `${user.name}-${user.service}`;
262
+ return filelist;
418
263
  }
419
264
 
420
265
  // src/api/gofile.ts
@@ -449,22 +294,22 @@ async function getFolderFiles(id, token, websiteToken) {
449
294
  throw new Error(`HTTP error! status: ${response.status}`);
450
295
  }
451
296
  const data = await response.json();
452
- const files = Object.values(data.data.children).map((f) => ({
453
- url: f.link,
454
- name: f.name
455
- }));
456
- return files;
297
+ const files = Object.values(data.data.children).map(
298
+ (f) => CoomerFile.from({
299
+ url: f.link,
300
+ name: f.name
301
+ })
302
+ );
303
+ return new CoomerFileList(files);
457
304
  }
458
- async function getGofileData(url, mediaType) {
305
+ async function getGofileData(url) {
459
306
  const id = url.match(/gofile.io\/d\/(\w+)/)?.[1];
460
- const dirName = `gofile-${id}`;
461
307
  const token = await getToken();
462
308
  const websiteToken = await getWebsiteToken();
463
- const files = (await getFolderFiles(id, token, websiteToken)).filter(
464
- (f) => testMediaType(f.name, mediaType)
465
- );
309
+ const filelist = await getFolderFiles(id, token, websiteToken);
310
+ filelist.dirName = `gofile-${id}`;
466
311
  setGlobalHeaders({ Cookie: `accountToken=${token}` });
467
- return { dirName, files };
312
+ return filelist;
468
313
  }
469
314
 
470
315
  // src/api/nsfw.xxx.ts
@@ -486,9 +331,9 @@ async function getUserPosts(user) {
486
331
  }
487
332
  return posts;
488
333
  }
489
- async function getPostsData(posts, mediaType) {
334
+ async function getPostsData(posts) {
490
335
  console.log("Fetching posts data...");
491
- const data = [];
336
+ const filelist = new CoomerFileList();
492
337
  for (const post of posts) {
493
338
  const page = await fetch4(post).then((r) => r.text());
494
339
  const $ = cheerio2.load(page);
@@ -498,52 +343,49 @@ async function getPostsData(posts, mediaType) {
498
343
  const date = $(".sh-section .sh-section__passed").first().text().replace(/ /g, "-") || "";
499
344
  const ext = src.split(".").pop();
500
345
  const name = `${slug}-${date}.${ext}`;
501
- data.push({ name, url: src });
346
+ filelist.files.push(CoomerFile.from({ name, url: src }));
502
347
  }
503
- return data.filter((f) => testMediaType(f.name, mediaType));
348
+ return filelist;
504
349
  }
505
- async function getRedditData(url, mediaType) {
350
+ async function getRedditData(url) {
506
351
  const user = url.match(/u\/(\w+)/)?.[1];
507
352
  const posts = await getUserPosts(user);
508
- const files = await getPostsData(posts, mediaType);
509
- const dirName = `${user}-reddit`;
510
- return { dirName, files };
353
+ const filelist = await getPostsData(posts);
354
+ filelist.dirName = `${user}-reddit`;
355
+ return filelist;
511
356
  }
512
357
 
513
358
  // src/api/plain-curl.ts
514
359
  async function getPlainFileData(url) {
515
- return {
516
- dirName: "",
517
- files: [
518
- {
519
- name: url.split("/").pop(),
520
- url
521
- }
522
- ]
523
- };
360
+ const name = url.split("/").pop();
361
+ const file = CoomerFile.from({ name, url });
362
+ const filelist = new CoomerFileList([file]);
363
+ filelist.dirName = "";
364
+ return filelist;
524
365
  }
525
366
 
526
367
  // src/api/index.ts
527
- async function apiHandler(url, mediaType) {
528
- if (/^u\/\w+$/.test(url.trim())) {
529
- return getRedditData(url, mediaType);
368
+ async function apiHandler(url_) {
369
+ const url = new URL(url_);
370
+ if (/^u\/\w+$/.test(url.origin)) {
371
+ return getRedditData(url.href);
530
372
  }
531
- if (/coomer|kemono/.test(url)) {
532
- return getCoomerData(url, mediaType);
373
+ if (/coomer|kemono/.test(url.origin)) {
374
+ return getCoomerData(url.href);
533
375
  }
534
- if (/bunkr/.test(url)) {
535
- return getBunkrData(url, mediaType);
376
+ if (/bunkr/.test(url.origin)) {
377
+ return getBunkrData(url.href);
536
378
  }
537
- if (/gofile\.io/.test(url)) {
538
- return getGofileData(url, mediaType);
379
+ if (/gofile\.io/.test(url.origin)) {
380
+ return getGofileData(url.href);
539
381
  }
540
- if (/\.\w+/.test(url.split("/").pop())) {
541
- return getPlainFileData(url);
382
+ if (/\.\w+/.test(url.pathname)) {
383
+ return getPlainFileData(url.href);
542
384
  }
543
- console.error("Wrong URL.");
385
+ throw Error("Invalid URL");
544
386
  }
545
387
 
546
- // src/args-handler.ts
388
+ // src/cli/args-handler.ts
547
389
  import yargs from "yargs";
548
390
  import { hideBin } from "yargs/helpers";
549
391
  function argumentHander() {
@@ -576,23 +418,361 @@ function argumentHander() {
576
418
  }).help().alias("help", "h").parseSync();
577
419
  }
578
420
 
421
+ // src/cli/ui/index.tsx
422
+ import { render } from "ink";
423
+ import React9 from "react";
424
+
425
+ // src/cli/ui/app.tsx
426
+ import { Box as Box7 } from "ink";
427
+ import React8 from "react";
428
+
429
+ // src/cli/ui/components/file.tsx
430
+ import { Box as Box2, Spacer, Text as Text2 } from "ink";
431
+ import React3 from "react";
432
+
433
+ // src/utils/strings.ts
434
+ function b2mb(bytes) {
435
+ return (bytes / 1048576).toFixed(2);
436
+ }
437
+
438
+ // src/cli/ui/components/preview.tsx
439
+ import { Box } from "ink";
440
+ import Image, { TerminalInfoProvider } from "ink-picture";
441
+ import React from "react";
442
+
443
+ // src/cli/ui/store/index.ts
444
+ import { create } from "zustand";
445
+ var useInkStore = create((set) => ({
446
+ preview: false,
447
+ switchPreview: () => set((state) => ({
448
+ preview: !state.preview
449
+ })),
450
+ downloader: void 0,
451
+ setDownloader: (downloader) => set({ downloader })
452
+ }));
453
+
454
+ // src/cli/ui/components/preview.tsx
455
+ function Preview({ file }) {
456
+ const previewEnabled = useInkStore((state) => state.preview);
457
+ const bigEnough = file.downloaded > 50 * 1024;
458
+ const shouldShow = previewEnabled && bigEnough && isImage(file.filepath);
459
+ const imgInfo = `
460
+ can't read partial images yet...
461
+ actual size: ${file.size}}
462
+ downloaded: ${file.downloaded}}
463
+ `;
464
+ return shouldShow && /* @__PURE__ */ React.createElement(Box, { paddingX: 1 }, /* @__PURE__ */ React.createElement(TerminalInfoProvider, null, /* @__PURE__ */ React.createElement(Box, { width: 30, height: 15 }, /* @__PURE__ */ React.createElement(Image, { src: file.filepath, alt: imgInfo }))));
465
+ }
466
+
467
+ // src/cli/ui/components/spinner.tsx
468
+ import spinners from "cli-spinners";
469
+ import { Text } from "ink";
470
+ import React2, { useEffect, useState } from "react";
471
+ function Spinner({ type = "dots" }) {
472
+ const spinner = spinners[type];
473
+ const randomFrame = spinner.frames.length * Math.random() | 0;
474
+ const [frame, setFrame] = useState(randomFrame);
475
+ useEffect(() => {
476
+ const timer = setInterval(() => {
477
+ setFrame((previousFrame) => {
478
+ return (previousFrame + 1) % spinner.frames.length;
479
+ });
480
+ }, spinner.interval);
481
+ return () => {
482
+ clearInterval(timer);
483
+ };
484
+ }, [spinner]);
485
+ return /* @__PURE__ */ React2.createElement(Text, null, spinner.frames[frame]);
486
+ }
487
+
488
+ // src/cli/ui/components/file.tsx
489
+ function FileBox({ file }) {
490
+ const percentage = Number(file.downloaded / file.size * 100).toFixed(2);
491
+ return /* @__PURE__ */ React3.createElement(React3.Fragment, null, /* @__PURE__ */ React3.createElement(
492
+ Box2,
493
+ {
494
+ borderStyle: "single",
495
+ borderColor: "magentaBright",
496
+ borderDimColor: true,
497
+ paddingX: 1,
498
+ flexDirection: "column"
499
+ },
500
+ /* @__PURE__ */ React3.createElement(Box2, null, /* @__PURE__ */ React3.createElement(Text2, { color: "blue", dimColor: true, wrap: "truncate-middle" }, file.name)),
501
+ /* @__PURE__ */ React3.createElement(Box2, { flexDirection: "row-reverse" }, /* @__PURE__ */ React3.createElement(Text2, { color: "cyan", dimColor: true }, b2mb(file.downloaded), "/", file.size ? b2mb(file.size) : "\u221E", " MB"), /* @__PURE__ */ React3.createElement(Text2, { color: "redBright", dimColor: true }, file.size ? ` ${percentage}% ` : ""), /* @__PURE__ */ React3.createElement(Spacer, null), /* @__PURE__ */ React3.createElement(Text2, { color: "green", dimColor: true }, /* @__PURE__ */ React3.createElement(Spinner, null)))
502
+ ), /* @__PURE__ */ React3.createElement(Preview, { file }));
503
+ }
504
+
505
+ // src/cli/ui/components/filelist.tsx
506
+ import { Box as Box3, Text as Text3 } from "ink";
507
+ import React4 from "react";
508
+ function FileListStateBox({ filelist }) {
509
+ return /* @__PURE__ */ React4.createElement(
510
+ Box3,
511
+ {
512
+ paddingX: 1,
513
+ flexDirection: "column",
514
+ borderStyle: "single",
515
+ borderColor: "magenta",
516
+ borderDimColor: true
517
+ },
518
+ /* @__PURE__ */ React4.createElement(Box3, null, /* @__PURE__ */ React4.createElement(Box3, { marginRight: 1 }, /* @__PURE__ */ React4.createElement(Text3, { color: "cyanBright", dimColor: true }, "Found:")), /* @__PURE__ */ React4.createElement(Text3, { color: "blue", dimColor: true, wrap: "wrap" }, filelist.files.length)),
519
+ /* @__PURE__ */ React4.createElement(Box3, null, /* @__PURE__ */ React4.createElement(Box3, { marginRight: 1 }, /* @__PURE__ */ React4.createElement(Text3, { color: "cyanBright", dimColor: true }, "Downloaded:")), /* @__PURE__ */ React4.createElement(Text3, { color: "blue", dimColor: true, wrap: "wrap" }, filelist.getDownloaded().length)),
520
+ /* @__PURE__ */ React4.createElement(Box3, null, /* @__PURE__ */ React4.createElement(Box3, { width: 9 }, /* @__PURE__ */ React4.createElement(Text3, { color: "cyanBright", dimColor: true }, "Folder:")), /* @__PURE__ */ React4.createElement(Box3, { flexGrow: 1 }, /* @__PURE__ */ React4.createElement(Text3, { color: "blue", dimColor: true, wrap: "truncate-middle" }, filelist.dirPath)))
521
+ );
522
+ }
523
+
524
+ // src/cli/ui/components/keyboardinfo.tsx
525
+ import { Box as Box4, Text as Text4 } from "ink";
526
+ import React5 from "react";
527
+ var info = {
528
+ "s ": "skip current file",
529
+ p: "on/off image preview"
530
+ };
531
+ function KeyboardControlsInfo() {
532
+ const infoRender = Object.entries(info).map(([key, value]) => {
533
+ return /* @__PURE__ */ React5.createElement(Box4, { key }, /* @__PURE__ */ React5.createElement(Box4, { marginRight: 2 }, /* @__PURE__ */ React5.createElement(Text4, { color: "red", dimColor: true, bold: true }, key)), /* @__PURE__ */ React5.createElement(Text4, { dimColor: true, bold: false }, value));
534
+ });
535
+ return /* @__PURE__ */ React5.createElement(
536
+ Box4,
537
+ {
538
+ flexDirection: "column",
539
+ paddingX: 1,
540
+ borderStyle: "single",
541
+ borderColor: "gray",
542
+ borderDimColor: true
543
+ },
544
+ /* @__PURE__ */ React5.createElement(Box4, null, /* @__PURE__ */ React5.createElement(Text4, { color: "red", dimColor: true, bold: true }, "Keyboard controls:")),
545
+ infoRender
546
+ );
547
+ }
548
+
549
+ // src/cli/ui/components/loading.tsx
550
+ import { Box as Box5, Text as Text5 } from "ink";
551
+ import React6 from "react";
552
+ function Loading() {
553
+ return /* @__PURE__ */ React6.createElement(Box5, { paddingX: 1, borderDimColor: true, flexDirection: "column" }, /* @__PURE__ */ React6.createElement(Box5, { alignSelf: "center" }, /* @__PURE__ */ React6.createElement(Text5, { dimColor: true, color: "redBright" }, "Fetching Data")), /* @__PURE__ */ React6.createElement(Box5, { alignSelf: "center" }, /* @__PURE__ */ React6.createElement(Text5, { color: "blueBright", dimColor: true }, /* @__PURE__ */ React6.createElement(Spinner, { type: "grenade" }))));
554
+ }
555
+
556
+ // src/cli/ui/components/titlebar.tsx
557
+ import { Box as Box6, Spacer as Spacer2, Text as Text6 } from "ink";
558
+ import React7 from "react";
559
+
560
+ // package.json
561
+ var version = "3.3.2";
562
+
563
+ // src/cli/ui/components/titlebar.tsx
564
+ function TitleBar() {
565
+ return /* @__PURE__ */ React7.createElement(Box6, null, /* @__PURE__ */ React7.createElement(Spacer2, null), /* @__PURE__ */ React7.createElement(Box6, { borderColor: "magenta", borderStyle: "arrow" }, /* @__PURE__ */ React7.createElement(Text6, { color: "cyanBright" }, "Coomer-Downloader ", version)), /* @__PURE__ */ React7.createElement(Spacer2, null));
566
+ }
567
+
568
+ // src/cli/ui/hooks/downloader.ts
569
+ import { useEffect as useEffect2, useState as useState2 } from "react";
570
+ var useDownloaderHook = () => {
571
+ const downloader = useInkStore((state) => state.downloader);
572
+ const filelist = downloader?.filelist;
573
+ const [_, setHelper] = useState2(0);
574
+ useEffect2(() => {
575
+ downloader?.subject.subscribe(({ type }) => {
576
+ if (type === "FILE_DOWNLOADING_START" || type === "FILE_DOWNLOADING_END" || type === "CHUNK_DOWNLOADING_UPDATE") {
577
+ setHelper(Date.now());
578
+ }
579
+ });
580
+ });
581
+ };
582
+
583
+ // src/cli/ui/hooks/input.ts
584
+ import { useInput } from "ink";
585
+ var useInputHook = () => {
586
+ const downloader = useInkStore((state) => state.downloader);
587
+ const switchPreview = useInkStore((state) => state.switchPreview);
588
+ useInput((input) => {
589
+ if (input === "s") {
590
+ downloader?.skip();
591
+ }
592
+ if (input === "p") {
593
+ switchPreview();
594
+ }
595
+ });
596
+ };
597
+
598
+ // src/cli/ui/app.tsx
599
+ function App() {
600
+ useInputHook();
601
+ useDownloaderHook();
602
+ const downloader = useInkStore((state) => state.downloader);
603
+ const filelist = downloader?.filelist;
604
+ const isFilelist = filelist instanceof CoomerFileList;
605
+ return /* @__PURE__ */ React8.createElement(Box7, { borderStyle: "single", flexDirection: "column", borderColor: "blue", width: 80 }, /* @__PURE__ */ React8.createElement(TitleBar, null), !isFilelist ? /* @__PURE__ */ React8.createElement(Loading, null) : /* @__PURE__ */ React8.createElement(React8.Fragment, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(Box7, null, /* @__PURE__ */ React8.createElement(FileListStateBox, { filelist })), /* @__PURE__ */ React8.createElement(Box7, { flexBasis: 29 }, /* @__PURE__ */ React8.createElement(KeyboardControlsInfo, null))), filelist.getActiveFiles().map((file) => {
606
+ return /* @__PURE__ */ React8.createElement(FileBox, { file, key: file.name });
607
+ })));
608
+ }
609
+
610
+ // src/cli/ui/index.tsx
611
+ function createReactInk() {
612
+ return render(/* @__PURE__ */ React9.createElement(App, null));
613
+ }
614
+
615
+ // src/services/downloader.ts
616
+ import fs2 from "node:fs";
617
+ import { Readable, Transform } from "node:stream";
618
+ import { pipeline } from "node:stream/promises";
619
+ import { Subject } from "rxjs";
620
+
621
+ // src/utils/promise.ts
622
+ async function sleep(time) {
623
+ return new Promise((resolve) => setTimeout(resolve, time));
624
+ }
625
+
626
+ // src/utils/timer.ts
627
+ var Timer = class _Timer {
628
+ constructor(timeout = 1e4, timeoutCallback) {
629
+ this.timeout = timeout;
630
+ this.timeoutCallback = timeoutCallback;
631
+ this.timeout = timeout;
632
+ }
633
+ timer;
634
+ start() {
635
+ this.timer = setTimeout(() => {
636
+ this.stop();
637
+ this.timeoutCallback();
638
+ }, this.timeout);
639
+ return this;
640
+ }
641
+ stop() {
642
+ if (this.timer) {
643
+ clearTimeout(this.timer);
644
+ this.timer = void 0;
645
+ }
646
+ return this;
647
+ }
648
+ reset() {
649
+ this.stop();
650
+ this.start();
651
+ return this;
652
+ }
653
+ static withAbortController(timeout, abortControllerSubject, message = "Timeout") {
654
+ const callback = () => {
655
+ abortControllerSubject.next(message);
656
+ };
657
+ const timer = new _Timer(timeout, callback).start();
658
+ return { timer };
659
+ }
660
+ };
661
+
662
+ // src/services/downloader.ts
663
+ var Downloader = class {
664
+ constructor(filelist, chunkTimeout = 3e4, chunkFetchRetries = 5, fetchRetries = 7) {
665
+ this.filelist = filelist;
666
+ this.chunkTimeout = chunkTimeout;
667
+ this.chunkFetchRetries = chunkFetchRetries;
668
+ this.fetchRetries = fetchRetries;
669
+ this.setAbortControllerListener();
670
+ }
671
+ subject = new Subject();
672
+ abortController = new AbortController();
673
+ abortControllerSubject = new Subject();
674
+ setAbortControllerListener() {
675
+ this.abortControllerSubject.subscribe((type) => {
676
+ this.abortController.abort(type);
677
+ this.abortController = new AbortController();
678
+ });
679
+ }
680
+ async fetchStream(file, stream, sizeOld = 0, retries = this.chunkFetchRetries) {
681
+ const signal = this.abortController.signal;
682
+ const subject = this.subject;
683
+ const { timer } = Timer.withAbortController(this.chunkTimeout, this.abortControllerSubject);
684
+ let i;
685
+ try {
686
+ const fileStream = fs2.createWriteStream(file.filepath, { flags: "a" });
687
+ const progressStream = new Transform({
688
+ transform(chunk, _encoding, callback) {
689
+ this.push(chunk);
690
+ file.downloaded += chunk.length;
691
+ timer.reset();
692
+ subject.next({ type: "CHUNK_DOWNLOADING_UPDATE" });
693
+ callback();
694
+ }
695
+ });
696
+ subject.next({ type: "CHUNK_DOWNLOADING_START" });
697
+ await pipeline(stream, progressStream, fileStream, { signal });
698
+ } catch (error) {
699
+ if (signal.aborted) {
700
+ if (signal.reason === "FILE_SKIP") return;
701
+ if (signal.reason === "TIMEOUT") {
702
+ if (retries === 0 && sizeOld < file.downloaded) {
703
+ retries += this.chunkFetchRetries;
704
+ sizeOld = file.downloaded;
705
+ }
706
+ if (retries === 0) return;
707
+ return await this.fetchStream(file, stream, sizeOld, retries - 1);
708
+ }
709
+ }
710
+ throw error;
711
+ } finally {
712
+ subject.next({ type: "CHUNK_DOWNLOADING_END" });
713
+ timer.stop();
714
+ clearInterval(i);
715
+ }
716
+ }
717
+ skip() {
718
+ this.abortControllerSubject.next("FILE_SKIP");
719
+ }
720
+ async downloadFile(file, retries = this.fetchRetries) {
721
+ const signal = this.abortController.signal;
722
+ try {
723
+ file.downloaded = await getFileSize(file.filepath);
724
+ const response = await fetchByteRange(file.url, file.downloaded, signal);
725
+ if (!response?.ok && response?.status !== 416) {
726
+ throw new Error(`HTTP error! status: ${response?.status}`);
727
+ }
728
+ const contentLength = response.headers.get("Content-Length");
729
+ if (!contentLength && file.downloaded > 0) return;
730
+ const restFileSize = parseInt(contentLength);
731
+ file.size = restFileSize + file.downloaded;
732
+ if (file.size > file.downloaded && response.body) {
733
+ const stream = Readable.fromWeb(response.body);
734
+ stream.setMaxListeners(20);
735
+ await this.fetchStream(file, stream, file.downloaded);
736
+ }
737
+ } catch (error) {
738
+ if (signal.aborted) {
739
+ if (signal.reason === "FILE_SKIP") return;
740
+ }
741
+ if (retries > 0) {
742
+ if (/coomer|kemono/.test(file.url)) {
743
+ file.url = tryFixCoomerUrl(file.url, retries);
744
+ }
745
+ await sleep(1e3);
746
+ return await this.downloadFile(file, retries - 1);
747
+ }
748
+ throw error;
749
+ }
750
+ }
751
+ async downloadFiles() {
752
+ mkdir(this.filelist.dirPath);
753
+ this.subject.next({ type: "FILES_DOWNLOADING_START" });
754
+ for (const file of this.filelist.files) {
755
+ file.active = true;
756
+ this.subject.next({ type: "FILE_DOWNLOADING_START" });
757
+ await this.downloadFile(file);
758
+ file.active = false;
759
+ this.subject.next({ type: "FILE_DOWNLOADING_END" });
760
+ }
761
+ this.subject.next({ type: "FILES_DOWNLOADING_END" });
762
+ }
763
+ };
764
+
579
765
  // src/index.ts
580
766
  async function run() {
767
+ createReactInk();
581
768
  const { url, dir, media, include, exclude, skip } = argumentHander();
582
- const { dirName, files } = await apiHandler(url, media);
583
- const downloadDir = dir === "./" ? path2.resolve(dir, dirName) : path2.join(os.homedir(), path2.join(dir, dirName));
584
- const filteredFiles = filterKeywords(files.slice(skip), include, exclude);
585
- console.table([
586
- {
587
- found: files.length,
588
- skip,
589
- filtered: files.length - filteredFiles.length - skip,
590
- folder: downloadDir
591
- }
592
- ]);
769
+ const filelist = await apiHandler(url);
770
+ filelist.setDirPath(dir).skip(skip).filterByText(include, exclude).filterByMediaType(media);
771
+ await filelist.calculateFileSizes();
593
772
  setGlobalHeaders({ Referer: url });
594
- createMultibar();
595
- await downloadFiles(filteredFiles, downloadDir);
773
+ const downloader = new Downloader(filelist);
774
+ useInkStore.getState().setDownloader(downloader);
775
+ await downloader.downloadFiles();
596
776
  process2.kill(process2.pid, "SIGINT");
597
777
  }
598
778
  run();