animepic-utils 0.0.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/image-utils.d.ts +8 -0
- package/dist/image-utils.js +183 -5
- package/dist/pics.js +3 -1
- package/image-utils.ts +138 -0
- package/package.json +7 -2
- package/pics.ts +3 -1
- package/readme.md +22 -4
- package/title.png +0 -0
package/dist/image-utils.d.ts
CHANGED
|
@@ -71,5 +71,13 @@ export default class ImageUtils {
|
|
|
71
71
|
* Adds the _p, _s, _g, or _c qualifier to images
|
|
72
72
|
*/
|
|
73
73
|
static changeQualifiers: (folder: string, qualifier?: "p" | "s" | "g" | "c") => void;
|
|
74
|
+
/**
|
|
75
|
+
* Reverse searches the image to find danbooru post.
|
|
76
|
+
*/
|
|
77
|
+
static reverseDanbooruSearch: (filepath: string, minSimilarity?: number) => Promise<any[]>;
|
|
78
|
+
/**
|
|
79
|
+
* Attempts to recover arbitrarily named posts from danbooru, if they exist.
|
|
80
|
+
*/
|
|
81
|
+
static recoverFromDanbooru: (folder: string) => Promise<void>;
|
|
74
82
|
}
|
|
75
83
|
export {};
|
package/dist/image-utils.js
CHANGED
|
@@ -1,4 +1,37 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
2
35
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
37
|
};
|
|
@@ -7,12 +40,14 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
7
40
|
const path_1 = __importDefault(require("path"));
|
|
8
41
|
const sharp_1 = __importDefault(require("sharp"));
|
|
9
42
|
const waifu2x_1 = __importDefault(require("waifu2x"));
|
|
43
|
+
const cheerio = __importStar(require("cheerio"));
|
|
10
44
|
class ImageUtils {
|
|
11
45
|
/**
|
|
12
46
|
* Fixes incorrect image extensions. It must be correct to preview on mac.
|
|
13
47
|
*/
|
|
14
48
|
static fixFileExtensions = async (folder) => {
|
|
15
|
-
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
49
|
+
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
50
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
16
51
|
for (const file of files) {
|
|
17
52
|
let filepath = path_1.default.join(folder, file);
|
|
18
53
|
if (fs_1.default.lstatSync(filepath).isDirectory())
|
|
@@ -29,7 +64,8 @@ class ImageUtils {
|
|
|
29
64
|
* Copies images to the destination (unchanged)
|
|
30
65
|
*/
|
|
31
66
|
static copyImages = (sourceFolder, destFolder) => {
|
|
32
|
-
const files = fs_1.default.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
67
|
+
const files = fs_1.default.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
68
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
33
69
|
for (const file of files) {
|
|
34
70
|
let src = path_1.default.join(sourceFolder, file);
|
|
35
71
|
if (fs_1.default.lstatSync(src).isDirectory())
|
|
@@ -42,7 +78,8 @@ class ImageUtils {
|
|
|
42
78
|
* Moves images to the destination
|
|
43
79
|
*/
|
|
44
80
|
static moveImages = (sourceFolder, destFolder) => {
|
|
45
|
-
const files = fs_1.default.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
81
|
+
const files = fs_1.default.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
82
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
46
83
|
for (const file of files) {
|
|
47
84
|
let src = path_1.default.join(sourceFolder, file);
|
|
48
85
|
if (fs_1.default.lstatSync(src).isDirectory())
|
|
@@ -163,7 +200,8 @@ class ImageUtils {
|
|
|
163
200
|
* Processes an image folder with a custom chain of operations.
|
|
164
201
|
*/
|
|
165
202
|
static processImages = async (folder, ...operations) => {
|
|
166
|
-
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
203
|
+
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
204
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
167
205
|
let i = 1;
|
|
168
206
|
for (const file of files) {
|
|
169
207
|
console.log(`${i}/${files.length} -> ${file}`);
|
|
@@ -239,7 +277,8 @@ class ImageUtils {
|
|
|
239
277
|
* Adds the _p, _s, _g, or _c qualifier to images
|
|
240
278
|
*/
|
|
241
279
|
static changeQualifiers = (folder, qualifier = "p") => {
|
|
242
|
-
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
280
|
+
const files = fs_1.default.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
281
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
243
282
|
for (const file of files) {
|
|
244
283
|
const { name, ext } = path_1.default.parse(file);
|
|
245
284
|
const match = name.match(/(_s|_p|_g|_c!?)(\d+)?$/);
|
|
@@ -253,5 +292,144 @@ class ImageUtils {
|
|
|
253
292
|
fs_1.default.renameSync(src, dest);
|
|
254
293
|
}
|
|
255
294
|
};
|
|
295
|
+
/**
|
|
296
|
+
* Reverse searches the image to find danbooru post.
|
|
297
|
+
*/
|
|
298
|
+
static reverseDanbooruSearch = async (filepath, minSimilarity = 75) => {
|
|
299
|
+
const buffer = new Uint8Array(fs_1.default.readFileSync(filepath)).buffer;
|
|
300
|
+
const form = new FormData();
|
|
301
|
+
form.append("file", new Blob([buffer], { type: "image/png" }));
|
|
302
|
+
const html = await fetch("https://iqdb.org/", { method: "POST", body: form }).then((r) => r.text());
|
|
303
|
+
const $ = cheerio.load(html);
|
|
304
|
+
let result = [];
|
|
305
|
+
let downloadLinks = [];
|
|
306
|
+
let promises = [];
|
|
307
|
+
const appendDanbooru = async (link) => {
|
|
308
|
+
const json = await fetch(`${link}.json`).then((r) => r.json());
|
|
309
|
+
result.push(json);
|
|
310
|
+
};
|
|
311
|
+
const appendZerochanDownload = async (link) => {
|
|
312
|
+
const json = await fetch(`${link}?json`).then((r) => r.json());
|
|
313
|
+
downloadLinks.push(json.full);
|
|
314
|
+
};
|
|
315
|
+
const appendGelbooruDownload = async (link) => {
|
|
316
|
+
let baseURL = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&json=1&id=`;
|
|
317
|
+
const result = await fetch(`${baseURL}${link.match(/\d+/)?.[0]}`).then((r) => r.json());
|
|
318
|
+
downloadLinks.push(result.post[0]?.file_url);
|
|
319
|
+
};
|
|
320
|
+
const appendSafebooruDownload = async (link) => {
|
|
321
|
+
let baseURL = `https://safebooru.org//index.php?page=dapi&s=post&q=index&json=1&id=`;
|
|
322
|
+
const result = await fetch(`${baseURL}${link.match(/\d+/)?.[0]}`).then((r) => r.json());
|
|
323
|
+
downloadLinks.push(result[0]?.file_url);
|
|
324
|
+
};
|
|
325
|
+
const appendYandereDownload = async (link) => {
|
|
326
|
+
console.log(link.match(/\d+/)?.[0]);
|
|
327
|
+
const result = await fetch(`https://yande.re/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json());
|
|
328
|
+
downloadLinks.push(result[0]?.file_url);
|
|
329
|
+
};
|
|
330
|
+
const appendKonachanDownload = async (link) => {
|
|
331
|
+
const result = await fetch(`https://konachan.com/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json());
|
|
332
|
+
downloadLinks.push(result[0]?.file_url);
|
|
333
|
+
};
|
|
334
|
+
$("#pages > div").each((i, el) => {
|
|
335
|
+
let link = ($(el).find("a").first().attr("href") || "").replace(/^\/\//, "http://").replace("http://", "https://");
|
|
336
|
+
let link2 = ($(el).find("a").last().attr("href") || "").replace(/^\/\//, "http://").replace("http://", "https://");
|
|
337
|
+
const textTds = $(el).find("td").filter((_, td) => $(td).children("img").length === 0)
|
|
338
|
+
.map((_, td) => $(td).text().trim()).get();
|
|
339
|
+
const similarity = parseFloat(textTds.find(text => /% similarity$/.test(text)) || "");
|
|
340
|
+
if (similarity > minSimilarity) {
|
|
341
|
+
if (link.includes("danbooru.donmai.us"))
|
|
342
|
+
promises.push(appendDanbooru(link));
|
|
343
|
+
if (link.includes("zerochan.net"))
|
|
344
|
+
promises.push(appendZerochanDownload(link));
|
|
345
|
+
if (link2.includes("gelbooru.com"))
|
|
346
|
+
promises.push(appendGelbooruDownload(link));
|
|
347
|
+
if (link2.includes("safebooru.org"))
|
|
348
|
+
promises.push(appendSafebooruDownload(link));
|
|
349
|
+
if (link.includes("yande.re"))
|
|
350
|
+
promises.push(appendYandereDownload(link));
|
|
351
|
+
if (link.includes("konachan.com"))
|
|
352
|
+
promises.push(appendKonachanDownload(link));
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
await Promise.allSettled(promises);
|
|
356
|
+
if (result.length === 1) {
|
|
357
|
+
if (!result[0].file_url)
|
|
358
|
+
result[0].file_url = downloadLinks[0];
|
|
359
|
+
}
|
|
360
|
+
return result;
|
|
361
|
+
};
|
|
362
|
+
/**
|
|
363
|
+
* Attempts to recover arbitrarily named posts from danbooru, if they exist.
|
|
364
|
+
*/
|
|
365
|
+
static recoverFromDanbooru = async (folder) => {
|
|
366
|
+
const original = path_1.default.join(folder, "original");
|
|
367
|
+
const pixiv = path_1.default.join(folder, "pixiv");
|
|
368
|
+
const twitter = path_1.default.join(folder, "twitter");
|
|
369
|
+
const comic = path_1.default.join(folder, "comic");
|
|
370
|
+
const unrecoverable = path_1.default.join(folder, "unrecoverable");
|
|
371
|
+
if (!fs_1.default.existsSync(original))
|
|
372
|
+
fs_1.default.mkdirSync(original);
|
|
373
|
+
this.moveImages(folder, original);
|
|
374
|
+
const files = fs_1.default.readdirSync(original).filter((f) => f !== ".DS_Store")
|
|
375
|
+
.sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
|
|
376
|
+
for (const file of files) {
|
|
377
|
+
const pixivID = file.match(/\d+/)?.[0];
|
|
378
|
+
let result = [];
|
|
379
|
+
if (pixivID)
|
|
380
|
+
result = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json());
|
|
381
|
+
if (!result.length)
|
|
382
|
+
result = await this.reverseDanbooruSearch(path_1.default.join(original, file));
|
|
383
|
+
if (result.length) {
|
|
384
|
+
let isComic = false;
|
|
385
|
+
for (const json of result) {
|
|
386
|
+
if (json.tag_string.includes("comic"))
|
|
387
|
+
isComic = true;
|
|
388
|
+
}
|
|
389
|
+
for (const json of result) {
|
|
390
|
+
let filename = path_1.default.basename(json.source);
|
|
391
|
+
if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
|
|
392
|
+
filename = `twitter_${filename}.${json.file_ext}`;
|
|
393
|
+
}
|
|
394
|
+
const downloadLink = json.file_url;
|
|
395
|
+
if (!downloadLink) {
|
|
396
|
+
if (!fs_1.default.existsSync(unrecoverable))
|
|
397
|
+
fs_1.default.mkdirSync(unrecoverable);
|
|
398
|
+
let src = path_1.default.join(original, file);
|
|
399
|
+
let dest = path_1.default.join(unrecoverable, file);
|
|
400
|
+
fs_1.default.copyFileSync(src, dest);
|
|
401
|
+
}
|
|
402
|
+
else {
|
|
403
|
+
const buffer = await fetch(downloadLink).then((r) => r.arrayBuffer());
|
|
404
|
+
if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
|
|
405
|
+
if (!fs_1.default.existsSync(twitter))
|
|
406
|
+
fs_1.default.mkdirSync(twitter);
|
|
407
|
+
let dest = path_1.default.join(twitter, filename);
|
|
408
|
+
fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
|
|
409
|
+
}
|
|
410
|
+
else if (isComic) {
|
|
411
|
+
if (!fs_1.default.existsSync(comic))
|
|
412
|
+
fs_1.default.mkdirSync(comic);
|
|
413
|
+
let dest = path_1.default.join(comic, filename);
|
|
414
|
+
fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
|
|
415
|
+
}
|
|
416
|
+
else {
|
|
417
|
+
if (!fs_1.default.existsSync(pixiv))
|
|
418
|
+
fs_1.default.mkdirSync(pixiv);
|
|
419
|
+
let dest = path_1.default.join(pixiv, filename);
|
|
420
|
+
fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
else {
|
|
426
|
+
if (!fs_1.default.existsSync(unrecoverable))
|
|
427
|
+
fs_1.default.mkdirSync(unrecoverable);
|
|
428
|
+
let src = path_1.default.join(original, file);
|
|
429
|
+
let dest = path_1.default.join(unrecoverable, file);
|
|
430
|
+
fs_1.default.copyFileSync(src, dest);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
};
|
|
256
434
|
}
|
|
257
435
|
exports.default = ImageUtils;
|
package/dist/pics.js
CHANGED
|
@@ -6,6 +6,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
require("dotenv/config");
|
|
7
7
|
const image_utils_1 = __importDefault(require("./image-utils"));
|
|
8
8
|
const start = async () => {
|
|
9
|
-
|
|
9
|
+
// await imageUtils.recoverFromDanbooru(process.env.FOLDER!)
|
|
10
|
+
await image_utils_1.default.moepicsProcess(process.env.FOLDER);
|
|
11
|
+
// imageUtils.changeQualifiers(process.env.FOLDER!, "c")
|
|
10
12
|
};
|
|
11
13
|
start();
|
package/image-utils.ts
CHANGED
|
@@ -2,6 +2,7 @@ import fs from "fs"
|
|
|
2
2
|
import path from "path"
|
|
3
3
|
import sharp from "sharp"
|
|
4
4
|
import waifu2x, {Waifu2xOptions} from "waifu2x"
|
|
5
|
+
import * as cheerio from "cheerio"
|
|
5
6
|
|
|
6
7
|
type Formats = "jpg" | "png" | "webp" | "avif" | "jxl"
|
|
7
8
|
|
|
@@ -19,6 +20,7 @@ export default class ImageUtils {
|
|
|
19
20
|
*/
|
|
20
21
|
public static fixFileExtensions = async (folder: string) => {
|
|
21
22
|
const files = fs.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
23
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
22
24
|
for (const file of files) {
|
|
23
25
|
let filepath = path.join(folder, file)
|
|
24
26
|
if (fs.lstatSync(filepath).isDirectory()) continue
|
|
@@ -36,6 +38,7 @@ export default class ImageUtils {
|
|
|
36
38
|
*/
|
|
37
39
|
public static copyImages = (sourceFolder: string, destFolder: string) => {
|
|
38
40
|
const files = fs.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
41
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
39
42
|
for (const file of files) {
|
|
40
43
|
let src = path.join(sourceFolder, file)
|
|
41
44
|
if (fs.lstatSync(src).isDirectory()) continue
|
|
@@ -49,6 +52,7 @@ export default class ImageUtils {
|
|
|
49
52
|
*/
|
|
50
53
|
public static moveImages = (sourceFolder: string, destFolder: string) => {
|
|
51
54
|
const files = fs.readdirSync(sourceFolder).filter((f) => f !== ".DS_Store")
|
|
55
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
52
56
|
for (const file of files) {
|
|
53
57
|
let src = path.join(sourceFolder, file)
|
|
54
58
|
if (fs.lstatSync(src).isDirectory()) continue
|
|
@@ -178,6 +182,7 @@ export default class ImageUtils {
|
|
|
178
182
|
public static processImages = async (folder: string,
|
|
179
183
|
...operations: Array<(file: string) => Promise<string>>) => {
|
|
180
184
|
const files = fs.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
185
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
181
186
|
let i = 1
|
|
182
187
|
for (const file of files) {
|
|
183
188
|
console.log(`${i}/${files.length} -> ${file}`)
|
|
@@ -274,6 +279,7 @@ export default class ImageUtils {
|
|
|
274
279
|
*/
|
|
275
280
|
public static changeQualifiers = (folder: string, qualifier: "p" | "s" | "g" | "c" = "p") => {
|
|
276
281
|
const files = fs.readdirSync(folder).filter((f) => f !== ".DS_Store")
|
|
282
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
277
283
|
for (const file of files) {
|
|
278
284
|
const {name, ext} = path.parse(file)
|
|
279
285
|
|
|
@@ -290,4 +296,136 @@ export default class ImageUtils {
|
|
|
290
296
|
fs.renameSync(src, dest)
|
|
291
297
|
}
|
|
292
298
|
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Reverse searches the image to find danbooru post.
|
|
302
|
+
*/
|
|
303
|
+
public static reverseDanbooruSearch = async (filepath: string, minSimilarity = 75) => {
|
|
304
|
+
const buffer = new Uint8Array(fs.readFileSync(filepath)).buffer
|
|
305
|
+
|
|
306
|
+
const form = new FormData()
|
|
307
|
+
form.append("file", new Blob([buffer], {type: "image/png"}))
|
|
308
|
+
|
|
309
|
+
const html = await fetch("https://iqdb.org/", {method: "POST", body: form}).then((r) => r.text())
|
|
310
|
+
const $ = cheerio.load(html)
|
|
311
|
+
|
|
312
|
+
let result = [] as any[]
|
|
313
|
+
let downloadLinks = [] as string[]
|
|
314
|
+
let promises = [] as Promise<void>[]
|
|
315
|
+
|
|
316
|
+
const appendDanbooru = async (link: string) => {
|
|
317
|
+
const json = await fetch(`${link}.json`).then((r) => r.json())
|
|
318
|
+
result.push(json)
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const appendZerochanDownload = async (link: string) => {
|
|
322
|
+
const json = await fetch(`${link}?json`).then((r) => r.json())
|
|
323
|
+
downloadLinks.push(json.full)
|
|
324
|
+
}
|
|
325
|
+
const appendGelbooruDownload = async (link: string) => {
|
|
326
|
+
let baseURL = `https://gelbooru.com/index.php?page=dapi&s=post&q=index&json=1&id=`
|
|
327
|
+
const result = await fetch(`${baseURL}${link.match(/\d+/)?.[0]}`).then((r) => r.json())
|
|
328
|
+
downloadLinks.push(result.post[0]?.file_url)
|
|
329
|
+
}
|
|
330
|
+
const appendSafebooruDownload = async (link: string) => {
|
|
331
|
+
let baseURL = `https://safebooru.org//index.php?page=dapi&s=post&q=index&json=1&id=`
|
|
332
|
+
const result = await fetch(`${baseURL}${link.match(/\d+/)?.[0]}`).then((r) => r.json())
|
|
333
|
+
downloadLinks.push(result[0]?.file_url)
|
|
334
|
+
}
|
|
335
|
+
const appendYandereDownload = async (link: string) => {
|
|
336
|
+
console.log(link.match(/\d+/)?.[0])
|
|
337
|
+
const result = await fetch(`https://yande.re/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json())
|
|
338
|
+
downloadLinks.push(result[0]?.file_url)
|
|
339
|
+
}
|
|
340
|
+
const appendKonachanDownload = async (link: string) => {
|
|
341
|
+
const result = await fetch(`https://konachan.com/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json())
|
|
342
|
+
downloadLinks.push(result[0]?.file_url)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
$("#pages > div").each((i, el) => {
|
|
346
|
+
let link = ($(el).find("a").first().attr("href") || "").replace(/^\/\//, "http://").replace("http://", "https://")
|
|
347
|
+
let link2 = ($(el).find("a").last().attr("href") || "").replace(/^\/\//, "http://").replace("http://", "https://")
|
|
348
|
+
const textTds = $(el).find("td").filter((_, td) => $(td).children("img").length === 0)
|
|
349
|
+
.map((_, td) => $(td).text().trim()).get()
|
|
350
|
+
const similarity = parseFloat(textTds.find(text => /% similarity$/.test(text)) || "")
|
|
351
|
+
|
|
352
|
+
if (similarity > minSimilarity) {
|
|
353
|
+
if (link.includes("danbooru.donmai.us")) promises.push(appendDanbooru(link))
|
|
354
|
+
|
|
355
|
+
if (link.includes("zerochan.net")) promises.push(appendZerochanDownload(link))
|
|
356
|
+
if (link2.includes("gelbooru.com")) promises.push(appendGelbooruDownload(link))
|
|
357
|
+
if (link2.includes("safebooru.org")) promises.push(appendSafebooruDownload(link))
|
|
358
|
+
if (link.includes("yande.re")) promises.push(appendYandereDownload(link))
|
|
359
|
+
if (link.includes("konachan.com")) promises.push(appendKonachanDownload(link))
|
|
360
|
+
}
|
|
361
|
+
})
|
|
362
|
+
|
|
363
|
+
await Promise.allSettled(promises)
|
|
364
|
+
if (result.length === 1) {
|
|
365
|
+
if (!result[0].file_url) result[0].file_url = downloadLinks[0]
|
|
366
|
+
}
|
|
367
|
+
return result
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
/**
|
|
371
|
+
* Attempts to recover arbitrarily named posts from danbooru, if they exist.
|
|
372
|
+
*/
|
|
373
|
+
public static recoverFromDanbooru = async (folder: string) => {
|
|
374
|
+
const original = path.join(folder, "original")
|
|
375
|
+
const pixiv = path.join(folder, "pixiv")
|
|
376
|
+
const twitter = path.join(folder, "twitter")
|
|
377
|
+
const comic = path.join(folder, "comic")
|
|
378
|
+
const unrecoverable = path.join(folder, "unrecoverable")
|
|
379
|
+
if (!fs.existsSync(original)) fs.mkdirSync(original)
|
|
380
|
+
|
|
381
|
+
this.moveImages(folder, original)
|
|
382
|
+
|
|
383
|
+
const files = fs.readdirSync(original).filter((f) => f !== ".DS_Store")
|
|
384
|
+
.sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
|
|
385
|
+
for (const file of files) {
|
|
386
|
+
const pixivID = file.match(/\d+/)?.[0]
|
|
387
|
+
let result = [] as any[]
|
|
388
|
+
if (pixivID) result = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json())
|
|
389
|
+
if (!result.length) result = await this.reverseDanbooruSearch(path.join(original, file))
|
|
390
|
+
if (result.length) {
|
|
391
|
+
let isComic = false
|
|
392
|
+
for (const json of result) {
|
|
393
|
+
if (json.tag_string.includes("comic")) isComic = true
|
|
394
|
+
}
|
|
395
|
+
for (const json of result) {
|
|
396
|
+
let filename = path.basename(json.source)
|
|
397
|
+
if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
|
|
398
|
+
filename = `twitter_${filename}.${json.file_ext}`
|
|
399
|
+
}
|
|
400
|
+
const downloadLink = json.file_url
|
|
401
|
+
if (!downloadLink) {
|
|
402
|
+
if (!fs.existsSync(unrecoverable)) fs.mkdirSync(unrecoverable)
|
|
403
|
+
let src = path.join(original, file)
|
|
404
|
+
let dest = path.join(unrecoverable, file)
|
|
405
|
+
fs.copyFileSync(src, dest)
|
|
406
|
+
} else {
|
|
407
|
+
const buffer = await fetch(downloadLink).then((r) => r.arrayBuffer())
|
|
408
|
+
if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
|
|
409
|
+
if (!fs.existsSync(twitter)) fs.mkdirSync(twitter)
|
|
410
|
+
let dest = path.join(twitter, filename)
|
|
411
|
+
fs.writeFileSync(dest, new Uint8Array(buffer))
|
|
412
|
+
} else if (isComic) {
|
|
413
|
+
if (!fs.existsSync(comic)) fs.mkdirSync(comic)
|
|
414
|
+
let dest = path.join(comic, filename)
|
|
415
|
+
fs.writeFileSync(dest, new Uint8Array(buffer))
|
|
416
|
+
} else {
|
|
417
|
+
if (!fs.existsSync(pixiv)) fs.mkdirSync(pixiv)
|
|
418
|
+
let dest = path.join(pixiv, filename)
|
|
419
|
+
fs.writeFileSync(dest, new Uint8Array(buffer))
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
} else {
|
|
424
|
+
if (!fs.existsSync(unrecoverable)) fs.mkdirSync(unrecoverable)
|
|
425
|
+
let src = path.join(original, file)
|
|
426
|
+
let dest = path.join(unrecoverable, file)
|
|
427
|
+
fs.copyFileSync(src, dest)
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
293
431
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "animepic-utils",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.4",
|
|
4
4
|
"description": "Node.js utils for processing anime images",
|
|
5
5
|
"main": "dist/image-utils.js",
|
|
6
6
|
"types": "dist/image-utils.d.ts",
|
|
@@ -9,7 +9,11 @@
|
|
|
9
9
|
"clean": "del-cli ./dist",
|
|
10
10
|
"prepublishOnly": "tsc"
|
|
11
11
|
},
|
|
12
|
-
"keywords": [
|
|
12
|
+
"keywords": [
|
|
13
|
+
"anime",
|
|
14
|
+
"image",
|
|
15
|
+
"utils"
|
|
16
|
+
],
|
|
13
17
|
"author": "Moebytes",
|
|
14
18
|
"license": "MIT",
|
|
15
19
|
"devDependencies": {
|
|
@@ -19,6 +23,7 @@
|
|
|
19
23
|
"typescript": "^5.9.3"
|
|
20
24
|
},
|
|
21
25
|
"dependencies": {
|
|
26
|
+
"cheerio": "^1.1.2",
|
|
22
27
|
"sharp": "^0.34.4",
|
|
23
28
|
"waifu2x": "^1.5.1"
|
|
24
29
|
}
|
package/pics.ts
CHANGED
|
@@ -2,7 +2,9 @@ import "dotenv/config"
|
|
|
2
2
|
import imageUtils from "./image-utils"
|
|
3
3
|
|
|
4
4
|
const start = async () => {
|
|
5
|
-
|
|
5
|
+
// await imageUtils.recoverFromDanbooru(process.env.FOLDER!)
|
|
6
|
+
await imageUtils.moepicsProcess(process.env.FOLDER!)
|
|
7
|
+
// imageUtils.changeQualifiers(process.env.FOLDER!, "c")
|
|
6
8
|
}
|
|
7
9
|
|
|
8
10
|
start()
|
package/readme.md
CHANGED
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
<div align="left">
|
|
2
|
+
<p>
|
|
3
|
+
<img src="https://github.com/Moebytes/animepic-utils/blob/main/title.png?raw=true" width="700" />
|
|
4
|
+
</p>
|
|
5
|
+
<p>
|
|
6
|
+
<a href="https://nodei.co/npm/animepic-utils/"><img src="https://nodei.co/npm/animepic-utils.png" /></a>
|
|
7
|
+
</p>
|
|
8
|
+
</div>
|
|
2
9
|
|
|
3
10
|
Some utilities for processing anime images. (However I guess it'll work for any images).
|
|
4
11
|
|
|
@@ -28,7 +35,17 @@ import imageUtils from "animepic-utils"
|
|
|
28
35
|
await imageUtils.fixFileExtensions(folder)
|
|
29
36
|
```
|
|
30
37
|
|
|
31
|
-
###
|
|
38
|
+
### Anime specific
|
|
39
|
+
|
|
40
|
+
The function `recoverFromDanbooru` takes a folder of arbitrarily named images and attempts to recover
|
|
41
|
+
the original pixiv/twitter files from the largest mirror site, danbooru. Files which are unrecoverable
|
|
42
|
+
will be put in an "unrecoverable" folder, usually because it wasn't found on danbooru.
|
|
43
|
+
|
|
44
|
+
```ts
|
|
45
|
+
import imageUtils from "animepic-utils"
|
|
46
|
+
|
|
47
|
+
await imageUtils.recoverFromDanbooru(folder)
|
|
48
|
+
```
|
|
32
49
|
|
|
33
50
|
The function `moepicsProcess` takes a folder of anime images and will generate the compressed
|
|
34
51
|
and upscaled versions that are suitable to upload to my website, https://moepictures.moe.
|
|
@@ -54,8 +71,9 @@ await imageUtils.changeQualifiers(folder, "g")
|
|
|
54
71
|
|
|
55
72
|
These are the full list of qualifiers:
|
|
56
73
|
|
|
57
|
-
- `_s` or none - Uploads images seperately
|
|
74
|
+
- `_s` or none - Uploads images seperately
|
|
58
75
|
- `_p` - Joins images together into one post as variations
|
|
59
76
|
- `_g` - Uploads images seperately but adds them to the same group
|
|
60
|
-
- `
|
|
77
|
+
- `_g!` - Adds the image as a *variation* to the previous group post
|
|
78
|
+
- `_c` - Adds images as child posts to the first in the set (`_c0`)
|
|
61
79
|
- `_c!` - Adds the image as a *variation* to the previous child post
|
package/title.png
ADDED
|
Binary file
|