animepic-utils 0.0.9 → 0.1.0

package/dist/download-utils.d.ts

@@ -7,4 +7,17 @@ export default class DownloadUtils {
      * Download any missing pixiv images.
      */
     static downloadMissingPixiv: (folder: string, pixivRefreshToken?: string) => Promise<void>;
+    /**
+     * Filter out duplicates.
+     */
+    static filterDuplicates: (folder: string) => Promise<void>;
+    private static normalizeFilename;
+    /**
+     * Process emojis.
+     */
+    static emojiProcessing: (folder: string) => Promise<void>;
+    /**
+     * Move images that do not have a very square aspect ratio.
+     */
+    static moveNonSquare: (folder: string, tolerance?: number) => Promise<void>;
 }

package/dist/download-utils.js

@@ -6,6 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const pixiv_ts_1 = __importDefault(require("pixiv.ts"));
+const sharp_phash_1 = __importDefault(require("sharp-phash"));
+const distance_1 = __importDefault(require("sharp-phash/distance"));
+const sharp_1 = __importDefault(require("sharp"));
 const image_utils_1 = __importDefault(require("./image-utils"));
 class DownloadUtils {
     /**
@@ -143,5 +146,132 @@ class DownloadUtils {
             i++;
         }
     };
+    /**
+     * Filter out duplicates.
+     */
+    static filterDuplicates = async (folder) => {
+        const originalFolder = path_1.default.join(folder, "original");
+        const dupeFolder = path_1.default.join(folder, "duplicates");
+        if (!fs_1.default.existsSync(originalFolder))
+            fs_1.default.mkdirSync(originalFolder);
+        image_utils_1.default.moveImages(folder, originalFolder);
+        const binaryToHex = (bin) => {
+            return bin.match(/.{4}/g)?.reduce(function (acc, i) {
+                return acc + parseInt(i, 2).toString(16).toUpperCase();
+            }, "") || "";
+        };
+        const files = fs_1.default.readdirSync(originalFolder)
+            .filter((f) => f !== ".DS_Store")
+            .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
+        console.log("Generating hash map...");
+        let hashMap = {};
+        let i = 1;
+        for (const file of files) {
+            console.log(`${i}/${files.length}`);
+            let buffer = fs_1.default.readFileSync(path_1.default.join(originalFolder, file));
+            const binary = await (0, sharp_phash_1.default)(buffer);
+            hashMap[file] = binaryToHex(binary);
+            i++;
+        }
+        console.log("Checking for duplicates...");
+        const processed = new Set();
+        for (let i = 0; i < files.length; i++) {
+            const fileA = files[i];
+            if (processed.has(fileA))
+                continue;
+            let folderKey = path_1.default.basename(fileA, path_1.default.extname(fileA));
+            let shouldMove = false;
+            for (let j = i + 1; j < files.length; j++) {
+                const fileB = files[j];
+                if (processed.has(fileB))
+                    continue;
+                if ((0, distance_1.default)(hashMap[fileA], hashMap[fileB]) < 6) {
+                    let destFolder = path_1.default.join(dupeFolder, folderKey);
+                    if (!fs_1.default.existsSync(destFolder))
+                        fs_1.default.mkdirSync(destFolder, { recursive: true });
+                    let src = path_1.default.join(originalFolder, fileB);
+                    let dest = path_1.default.join(destFolder, fileB);
+                    fs_1.default.renameSync(src, dest);
+                    processed.add(fileB);
+                    shouldMove = true;
+                }
+            }
+            if (shouldMove) {
+                let destFolder = path_1.default.join(dupeFolder, folderKey);
+                if (!fs_1.default.existsSync(destFolder))
+                    fs_1.default.mkdirSync(destFolder, { recursive: true });
+                let src = path_1.default.join(originalFolder, fileA);
+                let dest = path_1.default.join(destFolder, fileA);
+                fs_1.default.renameSync(src, dest);
+            }
+            processed.add(fileA);
+        }
+    };
+    static normalizeFilename = (input) => {
+        let { name, ext } = path_1.default.parse(input);
+        name = name.replace(/\(.*?\)/g, "").trim();
+        name = name.replace(/^[\d_]+/, "").trim();
+        const isCamelCase = /^[a-z]+(?:[A-Z][a-z0-9]*)*$/.test(name);
+        if (isCamelCase)
+            return `${name}${ext}`;
+        const words = name
+            .replace(/[^a-zA-Z0-9]+/g, " ").trim()
+            .split(/\s+/).filter(Boolean);
+        if (words.length === 0)
+            return `${name}${ext}`;
+        const camel = words.map((w, i) => i === 0 ? w.toLowerCase()
+            : w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()).join("");
+        return `${camel || name}${ext}`;
+    };
+    /**
+     * Process emojis.
+     */
+    static emojiProcessing = async (folder) => {
+        const originalFolder = path_1.default.join(folder, "original");
+        const processedFolder = path_1.default.join(folder, "processed");
+        if (!fs_1.default.existsSync(originalFolder))
+            fs_1.default.mkdirSync(originalFolder);
+        if (!fs_1.default.existsSync(processedFolder))
+            fs_1.default.mkdirSync(processedFolder);
+        image_utils_1.default.moveImages(folder, originalFolder);
+        image_utils_1.default.copyImages(originalFolder, processedFolder);
+        console.log("Processing images...");
+        await image_utils_1.default.processImages(processedFolder, async (file) => image_utils_1.default.resizeImage(file, 200), async (file) => {
+            let newName = this.normalizeFilename(path_1.default.basename(file));
+            let newDest = path_1.default.join(path_1.default.dirname(file), newName);
+            fs_1.default.renameSync(file, newDest);
+            return newDest;
+        });
+    };
+    /**
+     * Move images that do not have a very square aspect ratio.
+     */
+    static moveNonSquare = async (folder, tolerance = 0.1) => {
+        const originalFolder = path_1.default.join(folder, "original");
+        const nonSquareFolder = path_1.default.join(folder, "nosquare");
+        if (!fs_1.default.existsSync(originalFolder))
+            fs_1.default.mkdirSync(originalFolder);
+        image_utils_1.default.moveImages(folder, originalFolder);
+        const files = fs_1.default.readdirSync(originalFolder)
+            .filter((f) => f !== ".DS_Store")
+            .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
+        let i = 1;
+        for (const file of files) {
+            console.log(`${i}/${files.length}`);
+            const filePath = path_1.default.join(originalFolder, file);
+            const meta = await (0, sharp_1.default)(filePath).metadata();
+            if (!meta.width || !meta.height)
+                continue;
+            const aspect = meta.width / meta.height;
+            const lower = 1 - tolerance;
+            const upper = 1 + tolerance;
+            if (aspect < lower || aspect > upper) {
+                if (!fs_1.default.existsSync(nonSquareFolder))
+                    fs_1.default.mkdirSync(nonSquareFolder);
+                fs_1.default.renameSync(filePath, path_1.default.join(nonSquareFolder, file));
+            }
+            i++;
+        }
+    };
 }
 exports.default = DownloadUtils;

package/download-utils.ts

@@ -1,6 +1,9 @@
 import fs from "fs"
 import path from "path"
 import Pixiv from "pixiv.ts"
+import phash from "sharp-phash"
+import dist from "sharp-phash/distance"
+import sharp from "sharp"
 import imageUtils from "./image-utils"
 
 export default class DownloadUtils {
@@ -132,4 +135,148 @@ export default class DownloadUtils {
             i++
         }
     }
+
+    /**
+     * Filter out duplicates.
+     */
+    public static filterDuplicates = async (folder: string) => {
+        const originalFolder = path.join(folder, "original")
+        const dupeFolder = path.join(folder, "duplicates")
+        if (!fs.existsSync(originalFolder)) fs.mkdirSync(originalFolder)
+
+        imageUtils.moveImages(folder, originalFolder)
+
+        const binaryToHex = (bin: string) => {
+            return bin.match(/.{4}/g)?.reduce(function(acc, i) {
+                return acc + parseInt(i, 2).toString(16).toUpperCase()
+            }, "") || ""
+        }
+
+        const files = fs.readdirSync(originalFolder)
+            .filter((f) => f !== ".DS_Store")
+            .sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
+        
+        console.log("Generating hash map...")
+
+        let hashMap = {} as {[key: string]: string}
+        let i = 1
+        for (const file of files) {
+            console.log(`${i}/${files.length}`)
+            let buffer = fs.readFileSync(path.join(originalFolder, file))
+            const binary = await phash(buffer)
+            hashMap[file] = binaryToHex(binary)
+            i++
+        }
+
+        console.log("Checking for duplicates...")
+
+        const processed = new Set<string>()
+
+        for (let i = 0; i < files.length; i++) {
+            const fileA = files[i]
+            if (processed.has(fileA)) continue
+
+            let folderKey = path.basename(fileA, path.extname(fileA))
+            let shouldMove = false
+            for (let j = i + 1; j < files.length; j++) {
+                const fileB = files[j]
+                if (processed.has(fileB)) continue
+
+                if (dist(hashMap[fileA], hashMap[fileB]) < 6) {
+                    let destFolder = path.join(dupeFolder, folderKey)
+                    if (!fs.existsSync(destFolder)) fs.mkdirSync(destFolder, {recursive: true})
+                    let src = path.join(originalFolder, fileB)
+                    let dest = path.join(destFolder, fileB)
+                    fs.renameSync(src, dest)
+                    processed.add(fileB)
+                    shouldMove = true
+                }
+            }
+            if (shouldMove) {
+                let destFolder = path.join(dupeFolder, folderKey)
+                if (!fs.existsSync(destFolder)) fs.mkdirSync(destFolder, {recursive: true})
+                let src = path.join(originalFolder, fileA)
+                let dest = path.join(destFolder, fileA)
+                fs.renameSync(src, dest)
+            }
+            processed.add(fileA)
+        }
+    }
+
+    private static normalizeFilename = (input: string) => {
+        let {name, ext} = path.parse(input)
+        name = name.replace(/\(.*?\)/g, "").trim()
+        name = name.replace(/^[\d_]+/, "").trim()
+
+        const isCamelCase = /^[a-z]+(?:[A-Z][a-z0-9]*)*$/.test(name)
+        if (isCamelCase) return `${name}${ext}`
+
+        const words = name
+            .replace(/[^a-zA-Z0-9]+/g, " ").trim()
+            .split(/\s+/).filter(Boolean)
+
+        if (words.length === 0) return `${name}${ext}`
+
+        const camel = words.map((w, i) => i === 0 ? w.toLowerCase() 
+            : w.charAt(0).toUpperCase() + w.slice(1).toLowerCase()).join("")
+
+        return `${camel || name}${ext}`
+    }
+
+    /**
+     * Process emojis.
+     */
+    public static emojiProcessing = async (folder: string) => {
+        const originalFolder = path.join(folder, "original")
+        const processedFolder = path.join(folder, "processed")
+        if (!fs.existsSync(originalFolder)) fs.mkdirSync(originalFolder)
+        if (!fs.existsSync(processedFolder)) fs.mkdirSync(processedFolder)
+
+        imageUtils.moveImages(folder, originalFolder)
+        imageUtils.copyImages(originalFolder, processedFolder)
+        
+        console.log("Processing images...")
+        await imageUtils.processImages(processedFolder, 
+            async (file) => imageUtils.resizeImage(file, 200),
+            async (file) => {
+                let newName = this.normalizeFilename(path.basename(file))
+                let newDest = path.join(path.dirname(file), newName)
+                fs.renameSync(file, newDest)
+                return newDest
+            }
+        )
+    }
+
+    /**
+     * Move images that do not have a very square aspect ratio.
+     */
+    public static moveNonSquare = async (folder: string, tolerance = 0.1) => {
+        const originalFolder = path.join(folder, "original")
+        const nonSquareFolder = path.join(folder, "nosquare")
+        if (!fs.existsSync(originalFolder)) fs.mkdirSync(originalFolder)
+
+        imageUtils.moveImages(folder, originalFolder)
+
+        const files = fs.readdirSync(originalFolder)
+            .filter((f) => f !== ".DS_Store")
+            .sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
+        
+        let i = 1
+        for (const file of files) {
+            console.log(`${i}/${files.length}`)
+            const filePath = path.join(originalFolder, file)
+            const meta = await sharp(filePath).metadata()
+            if (!meta.width || !meta.height) continue
+
+            const aspect = meta.width / meta.height
+            const lower = 1 - tolerance
+            const upper = 1 + tolerance
+
+            if (aspect < lower || aspect > upper) {
+                if (!fs.existsSync(nonSquareFolder)) fs.mkdirSync(nonSquareFolder)
+                fs.renameSync(filePath, path.join(nonSquareFolder, file))
+            }
+            i++
+        }
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "animepic-utils",
-  "version": "0.0.9",
+  "version": "0.1.0",
   "description": "Node.js utils for processing anime images",
   "main": "dist/utils.js",
   "types": "dist/utils.d.ts",
@@ -29,6 +29,7 @@
     "jszip": "^3.10.1",
     "pixiv.ts": "^0.8.7",
     "sharp": "^0.34.4",
+    "sharp-phash": "^2.2.0",
     "waifu2x": "^1.5.1"
   }
 }