animepic-utils 0.0.4 → 0.0.6

package/dist/image-utils.d.ts

@@ -74,10 +74,14 @@ export default class ImageUtils {
     /**
      * Reverse searches the image to find danbooru post.
      */
-    static reverseDanbooruSearch: (filepath: string, minSimilarity?: number) => Promise<any[]>;
+    static reverseImageSearch: (filepath: string, minSimilarity?: number) => Promise<any>;
     /**
-     * Attempts to recover arbitrarily named posts from danbooru, if they exist.
+     * Attempts to recover arbitrarily named posts from pixiv, or danbooru as fallback.
      */
-    static recoverFromDanbooru: (folder: string) => Promise<void>;
+    static recoverFromPixiv: (folder: string, pixivRefreshToken?: string) => Promise<void>;
+    /**
+     * Attempts to filter AI images on a folder containing images from pixiv.
+     */
+    static filterAIImages: (folder: string, pixivRefreshToken?: string) => Promise<void>;
 }
 export {};

package/dist/image-utils.js

@@ -40,6 +40,7 @@ const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const sharp_1 = __importDefault(require("sharp"));
 const waifu2x_1 = __importDefault(require("waifu2x"));
+const pixiv_ts_1 = __importDefault(require("pixiv.ts"));
 const cheerio = __importStar(require("cheerio"));
 class ImageUtils {
     /**
@@ -295,18 +296,17 @@ class ImageUtils {
     /**
      * Reverse searches the image to find danbooru post.
      */
-    static reverseDanbooruSearch = async (filepath, minSimilarity = 75) => {
+    static reverseImageSearch = async (filepath, minSimilarity = 75) => {
         const buffer = new Uint8Array(fs_1.default.readFileSync(filepath)).buffer;
         const form = new FormData();
         form.append("file", new Blob([buffer], { type: "image/png" }));
         const html = await fetch("https://iqdb.org/", { method: "POST", body: form }).then((r) => r.text());
         const $ = cheerio.load(html);
-        let result = [];
+        let result = {};
         let downloadLinks = [];
         let promises = [];
         const appendDanbooru = async (link) => {
-            const json = await fetch(`${link}.json`).then((r) => r.json());
-            result.push(json);
+            result = await fetch(`${link}.json`).then((r) => r.json());
         };
         const appendZerochanDownload = async (link) => {
             const json = await fetch(`${link}?json`).then((r) => r.json());
@@ -323,7 +323,6 @@ class ImageUtils {
             downloadLinks.push(result[0]?.file_url);
         };
         const appendYandereDownload = async (link) => {
-            console.log(link.match(/\d+/)?.[0]);
             const result = await fetch(`https://yande.re/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json());
             downloadLinks.push(result[0]?.file_url);
         };
@@ -353,82 +352,166 @@ class ImageUtils {
             }
         });
         await Promise.allSettled(promises);
-        if (result.length === 1) {
-            if (!result[0].file_url)
-                result[0].file_url = downloadLinks[0];
+        if (result.id) {
+            if (!result.file_url)
+                result.file_url = downloadLinks[0];
         }
         return result;
     };
     /**
-     * Attempts to recover arbitrarily named posts from danbooru, if they exist.
+     * Attempts to recover arbitrarily named posts from pixiv, or danbooru as fallback.
      */
-    static recoverFromDanbooru = async (folder) => {
-        const original = path_1.default.join(folder, "original");
-        const pixiv = path_1.default.join(folder, "pixiv");
-        const twitter = path_1.default.join(folder, "twitter");
-        const comic = path_1.default.join(folder, "comic");
-        const unrecoverable = path_1.default.join(folder, "unrecoverable");
-        if (!fs_1.default.existsSync(original))
-            fs_1.default.mkdirSync(original);
-        this.moveImages(folder, original);
-        const files = fs_1.default.readdirSync(original).filter((f) => f !== ".DS_Store")
+    static recoverFromPixiv = async (folder, pixivRefreshToken) => {
+        const pixiv = await pixiv_ts_1.default.refreshLogin(pixivRefreshToken);
+        const originalFolder = path_1.default.join(folder, "original");
+        const pixivFolder = path_1.default.join(folder, "pixiv");
+        const twitterFolder = path_1.default.join(folder, "twitter");
+        const otherFolder = path_1.default.join(folder, "other");
+        const comicFolder = path_1.default.join(folder, "comic");
+        const unrecoverableFolder = path_1.default.join(folder, "unrecoverable");
+        if (!fs_1.default.existsSync(originalFolder))
+            fs_1.default.mkdirSync(originalFolder);
+        this.moveImages(folder, originalFolder);
+        const files = fs_1.default.readdirSync(originalFolder).filter((f) => f !== ".DS_Store")
             .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
+        let i = 1;
         for (const file of files) {
-            const pixivID = file.match(/\d+/)?.[0];
-            let result = [];
-            if (pixivID)
-                result = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json());
-            if (!result.length)
-                result = await this.reverseDanbooruSearch(path_1.default.join(original, file));
-            if (result.length) {
-                let isComic = false;
-                for (const json of result) {
+            console.log(`${i}/${files.length}  -> ${file}`);
+            let pixivID = file.match(/^\d{5,}(?=$|_)/)?.[0];
+            let danbooruPosts = [];
+            let isComic = false;
+            if (pixivID) {
+                danbooruPosts = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json());
+            }
+            else {
+                const danbooruPost = await this.reverseImageSearch(path_1.default.join(originalFolder, file));
+                if (Object.keys(danbooruPost).length)
+                    danbooruPosts = [danbooruPost];
+                if (danbooruPosts[0]?.source.includes("pximg.net") || danbooruPosts[0]?.source.includes("pixiv.net")) {
+                    pixivID = path_1.default.basename(danbooruPosts[0].source).match(/\d+/)?.[0];
+                }
+            }
+            if (danbooruPosts.length) {
+                for (const json of danbooruPosts) {
                     if (json.tag_string.includes("comic"))
                         isComic = true;
                 }
-                for (const json of result) {
+            }
+            try {
+                if (pixivID) {
+                    let illust = await pixiv.illust.get(pixivID);
+                    if (illust.width === 100 && illust.height === 100 && path_1.default.basename(illust.image_urls.medium)
+                        .includes("limit_unknown"))
+                        throw new Error("bad illust");
+                    let multiFolder = isComic ? comicFolder : pixivFolder;
+                    await pixiv.util.downloadIllust(illust, pixivFolder, "original", multiFolder);
+                    i++;
+                    continue;
+                }
+            }
+            catch { }
+            if (danbooruPosts.length) {
+                for (const json of danbooruPosts) {
                     let filename = path_1.default.basename(json.source);
-                    if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
+                    if (!filename.includes("."))
+                        filename += ".png";
+                    if (json.source.includes("pximg.net") || json.source.includes("pixiv.net")) {
+                        filename = path_1.default.basename(json.source);
+                    }
+                    else if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
                         filename = `twitter_${filename}.${json.file_ext}`;
                     }
                     const downloadLink = json.file_url;
-                    if (!downloadLink) {
-                        if (!fs_1.default.existsSync(unrecoverable))
-                            fs_1.default.mkdirSync(unrecoverable);
-                        let src = path_1.default.join(original, file);
-                        let dest = path_1.default.join(unrecoverable, file);
-                        fs_1.default.copyFileSync(src, dest);
-                    }
-                    else {
+                    if (downloadLink) {
                         const buffer = await fetch(downloadLink).then((r) => r.arrayBuffer());
                         if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
-                            if (!fs_1.default.existsSync(twitter))
-                                fs_1.default.mkdirSync(twitter);
-                            let dest = path_1.default.join(twitter, filename);
+                            if (!fs_1.default.existsSync(twitterFolder))
+                                fs_1.default.mkdirSync(twitterFolder);
+                            let dest = path_1.default.join(twitterFolder, filename);
                             fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
                         }
                         else if (isComic) {
-                            if (!fs_1.default.existsSync(comic))
-                                fs_1.default.mkdirSync(comic);
-                            let dest = path_1.default.join(comic, filename);
+                            if (!fs_1.default.existsSync(comicFolder))
+                                fs_1.default.mkdirSync(comicFolder);
+                            let dest = path_1.default.join(comicFolder, filename);
+                            fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
+                        }
+                        else if (json.source.includes("pximg.net") || json.source.includes("pixiv.net")) {
+                            if (!fs_1.default.existsSync(pixivFolder))
+                                fs_1.default.mkdirSync(pixivFolder);
+                            let dest = path_1.default.join(pixivFolder, filename);
                             fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
                         }
                         else {
-                            if (!fs_1.default.existsSync(pixiv))
-                                fs_1.default.mkdirSync(pixiv);
-                            let dest = path_1.default.join(pixiv, filename);
+                            if (!fs_1.default.existsSync(otherFolder))
+                                fs_1.default.mkdirSync(otherFolder);
+                            let dest = path_1.default.join(otherFolder, filename);
                             fs_1.default.writeFileSync(dest, new Uint8Array(buffer));
                         }
                     }
+                    else {
+                        if (!fs_1.default.existsSync(unrecoverableFolder))
+                            fs_1.default.mkdirSync(unrecoverableFolder);
+                        let src = path_1.default.join(originalFolder, file);
+                        let dest = path_1.default.join(unrecoverableFolder, file);
+                        fs_1.default.copyFileSync(src, dest);
+                    }
                 }
             }
             else {
-                if (!fs_1.default.existsSync(unrecoverable))
-                    fs_1.default.mkdirSync(unrecoverable);
-                let src = path_1.default.join(original, file);
-                let dest = path_1.default.join(unrecoverable, file);
+                if (!fs_1.default.existsSync(unrecoverableFolder))
+                    fs_1.default.mkdirSync(unrecoverableFolder);
+                let src = path_1.default.join(originalFolder, file);
+                let dest = path_1.default.join(unrecoverableFolder, file);
                 fs_1.default.copyFileSync(src, dest);
             }
+            i++;
+        }
+    };
+    /**
+     * Attempts to filter AI images on a folder containing images from pixiv.
+     */
+    static filterAIImages = async (folder, pixivRefreshToken) => {
+        const pixiv = await pixiv_ts_1.default.refreshLogin(pixivRefreshToken);
+        const originalFolder = path_1.default.join(folder, "original");
+        const aiFolder = path_1.default.join(folder, "ai");
+        const errorFolder = path_1.default.join(folder, "error");
+        if (!fs_1.default.existsSync(originalFolder))
+            fs_1.default.mkdirSync(originalFolder);
+        this.moveImages(folder, originalFolder);
+        const files = fs_1.default.readdirSync(originalFolder).filter((f) => f !== ".DS_Store")
+            .sort(new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare);
+        let i = 1;
+        for (const file of files) {
+            console.log(`${i}/${files.length}  -> ${file}`);
+            let pixivID = file.match(/^\d{5,}(?=\.|_)/)?.[0];
+            if (pixivID) {
+                try {
+                    let illust = await pixiv.illust.get(pixivID);
+                    if (pixiv.util.isAI(illust)) {
+                        if (!fs_1.default.existsSync(aiFolder))
+                            fs_1.default.mkdirSync(aiFolder);
+                        let src = path_1.default.join(originalFolder, file);
+                        let dest = path_1.default.join(aiFolder, file);
+                        fs_1.default.renameSync(src, dest);
+                    }
+                }
+                catch {
+                    if (!fs_1.default.existsSync(errorFolder))
+                        fs_1.default.mkdirSync(errorFolder);
+                    let src = path_1.default.join(originalFolder, file);
+                    let dest = path_1.default.join(errorFolder, file);
+                    fs_1.default.renameSync(src, dest);
+                }
+            }
+            else {
+                if (!fs_1.default.existsSync(errorFolder))
+                    fs_1.default.mkdirSync(errorFolder);
+                let src = path_1.default.join(originalFolder, file);
+                let dest = path_1.default.join(errorFolder, file);
+                fs_1.default.renameSync(src, dest);
+            }
+            i++;
         }
     };
 }

package/dist/pics.js

@@ -6,8 +6,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 require("dotenv/config");
 const image_utils_1 = __importDefault(require("./image-utils"));
 const start = async () => {
-    // await imageUtils.recoverFromDanbooru(process.env.FOLDER!)
-    await image_utils_1.default.moepicsProcess(process.env.FOLDER);
+    let pixivKey = process.env.PIXIV_REFRESH_TOKEN;
+    let folder = process.env.FOLDER;
+    // await imageUtils.recoverFromPixiv(folder, pixivKey)
+    await image_utils_1.default.filterAIImages(folder, pixivKey);
+    // await imageUtils.moepicsProcess(process.env.FOLDER!)
     // imageUtils.changeQualifiers(process.env.FOLDER!, "c")
 };
 start();

package/image-utils.ts

@@ -2,6 +2,7 @@ import fs from "fs"
 import path from "path"
 import sharp from "sharp"
 import waifu2x, {Waifu2xOptions} from "waifu2x"
+import Pixiv, { PixivIllust } from "pixiv.ts"
 import * as cheerio from "cheerio"
 
 type Formats = "jpg" | "png" | "webp" | "avif" | "jxl"
@@ -300,7 +301,7 @@ export default class ImageUtils {
     /**
      * Reverse searches the image to find danbooru post.
      */
-    public static reverseDanbooruSearch = async (filepath: string, minSimilarity = 75) => {
+    public static reverseImageSearch = async (filepath: string, minSimilarity = 75) => {
         const buffer = new Uint8Array(fs.readFileSync(filepath)).buffer
 
         const form = new FormData()
@@ -309,15 +310,13 @@ export default class ImageUtils {
         const html = await fetch("https://iqdb.org/", {method: "POST", body: form}).then((r) => r.text())
         const $ = cheerio.load(html)
 
-        let result = [] as any[]
+        let result = {} as any
         let downloadLinks = [] as string[]
         let promises = [] as Promise<void>[]
 
         const appendDanbooru = async (link: string) => {
-            const json = await fetch(`${link}.json`).then((r) => r.json())
-            result.push(json)
+            result = await fetch(`${link}.json`).then((r) => r.json())
         }
-
         const appendZerochanDownload = async (link: string) => {
             const json = await fetch(`${link}?json`).then((r) => r.json())
             downloadLinks.push(json.full)
@@ -333,7 +332,6 @@ export default class ImageUtils {
             downloadLinks.push(result[0]?.file_url)
         }
         const appendYandereDownload = async (link: string) => {
-            console.log(link.match(/\d+/)?.[0])
             const result = await fetch(`https://yande.re/post.json?tags=id:${link.match(/\d+/)?.[0]}`).then((r) => r.json())
             downloadLinks.push(result[0]?.file_url)
         }
@@ -361,71 +359,151 @@ export default class ImageUtils {
         })
 
         await Promise.allSettled(promises)
-        if (result.length === 1) {
-            if (!result[0].file_url) result[0].file_url = downloadLinks[0]
+        if (result.id) {
+            if (!result.file_url) result.file_url = downloadLinks[0]
         }
         return result
     }
 
     /**
-     * Attempts to recover arbitrarily named posts from danbooru, if they exist.
+     * Attempts to recover arbitrarily named posts from pixiv, or danbooru as fallback.
      */
-    public static recoverFromDanbooru = async (folder: string) => {
-        const original = path.join(folder, "original")
-        const pixiv = path.join(folder, "pixiv")
-        const twitter = path.join(folder, "twitter")
-        const comic = path.join(folder, "comic")
-        const unrecoverable = path.join(folder, "unrecoverable")
-        if (!fs.existsSync(original)) fs.mkdirSync(original)
-
-        this.moveImages(folder, original)
-
-        const files = fs.readdirSync(original).filter((f) => f !== ".DS_Store")
+    public static recoverFromPixiv = async (folder: string, pixivRefreshToken?: string) => {
+        const pixiv = await Pixiv.refreshLogin(pixivRefreshToken!)
+        const originalFolder = path.join(folder, "original")
+        const pixivFolder = path.join(folder, "pixiv")
+        const twitterFolder = path.join(folder, "twitter")
+        const otherFolder = path.join(folder, "other")
+        const comicFolder = path.join(folder, "comic")
+        const unrecoverableFolder = path.join(folder, "unrecoverable")
+        if (!fs.existsSync(originalFolder)) fs.mkdirSync(originalFolder)
+
+        this.moveImages(folder, originalFolder)
+
+        const files = fs.readdirSync(originalFolder).filter((f) => f !== ".DS_Store")
         .sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
+
+        let i = 1
         for (const file of files) {
-            const pixivID = file.match(/\d+/)?.[0]
-            let result = [] as any[]
-            if (pixivID) result = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json())
-            if (!result.length) result = await this.reverseDanbooruSearch(path.join(original, file))
-            if (result.length) {
-                let isComic = false
-                for (const json of result) {
+            console.log(`${i}/${files.length}  -> ${file}`)
+            let pixivID = file.match(/^\d{5,}(?=$|_)/)?.[0]
+            let danbooruPosts: any[] = []
+            let isComic = false
+
+            if (pixivID) {
+                danbooruPosts = await fetch(`https://danbooru.donmai.us/posts.json?tags=pixiv_id%3A${pixivID}&limit=1000`).then((r) => r.json())
+            } else {
+                const danbooruPost = await this.reverseImageSearch(path.join(originalFolder, file))
+                if (Object.keys(danbooruPost).length) danbooruPosts = [danbooruPost]
+                if (danbooruPosts[0]?.source.includes("pximg.net") || danbooruPosts[0]?.source.includes("pixiv.net")) {
+                    pixivID = path.basename(danbooruPosts[0].source).match(/\d+/)?.[0]
+                }
+            }
+
+            if (danbooruPosts.length) {
+                for (const json of danbooruPosts) {
                     if (json.tag_string.includes("comic")) isComic = true
                 }
-                for (const json of result) {
+            }
+
+            try {
+                if (pixivID) {
+                    let illust = await pixiv.illust.get(pixivID)
+                    if (illust.width === 100 && illust.height === 100 && path.basename(illust.image_urls.medium)
+                        .includes("limit_unknown")) throw new Error("bad illust")
+                    let multiFolder = isComic ? comicFolder : pixivFolder
+                    await pixiv.util.downloadIllust(illust, pixivFolder, "original", multiFolder)
+                    i++
+                    continue
+                }
+            } catch {}
+
+            if (danbooruPosts.length) {
+                for (const json of danbooruPosts) {
                     let filename = path.basename(json.source)
-                    if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
+                    if (!filename.includes(".")) filename += ".png"
+                    if (json.source.includes("pximg.net") || json.source.includes("pixiv.net")) {
+                        filename = path.basename(json.source)
+                    } else if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
                         filename = `twitter_${filename}.${json.file_ext}`
                     }
                     const downloadLink = json.file_url
-                    if (!downloadLink) {
-                        if (!fs.existsSync(unrecoverable)) fs.mkdirSync(unrecoverable)
-                        let src = path.join(original, file)
-                        let dest = path.join(unrecoverable, file)
-                        fs.copyFileSync(src, dest)
-                    } else {
+                    if (downloadLink) {
                         const buffer = await fetch(downloadLink).then((r) => r.arrayBuffer())
                         if (json.source.includes("twitter.com") || json.source.includes("x.com")) {
-                            if (!fs.existsSync(twitter)) fs.mkdirSync(twitter)
-                            let dest = path.join(twitter, filename)
+                            if (!fs.existsSync(twitterFolder)) fs.mkdirSync(twitterFolder)
+                            let dest = path.join(twitterFolder, filename)
                             fs.writeFileSync(dest, new Uint8Array(buffer))
                         } else if (isComic) {
-                            if (!fs.existsSync(comic)) fs.mkdirSync(comic)
-                            let dest = path.join(comic, filename)
+                            if (!fs.existsSync(comicFolder)) fs.mkdirSync(comicFolder)
+                            let dest = path.join(comicFolder, filename)
+                            fs.writeFileSync(dest, new Uint8Array(buffer))
+                        } else if (json.source.includes("pximg.net") || json.source.includes("pixiv.net")) {  
+                            if (!fs.existsSync(pixivFolder)) fs.mkdirSync(pixivFolder)
+                            let dest = path.join(pixivFolder, filename)
                             fs.writeFileSync(dest, new Uint8Array(buffer))
-                        } else {  
-                            if (!fs.existsSync(pixiv)) fs.mkdirSync(pixiv)
-                            let dest = path.join(pixiv, filename)
+                        } else {
+                            if (!fs.existsSync(otherFolder)) fs.mkdirSync(otherFolder)
+                            let dest = path.join(otherFolder, filename)
                             fs.writeFileSync(dest, new Uint8Array(buffer))
                         }
+                    } else {
+                        if (!fs.existsSync(unrecoverableFolder)) fs.mkdirSync(unrecoverableFolder)
+                        let src = path.join(originalFolder, file)
+                        let dest = path.join(unrecoverableFolder, file)
+                        fs.copyFileSync(src, dest)
                     }
                 }
             } else {
-                if (!fs.existsSync(unrecoverable)) fs.mkdirSync(unrecoverable)
-                let src = path.join(original, file)
-                let dest = path.join(unrecoverable, file)
+                if (!fs.existsSync(unrecoverableFolder)) fs.mkdirSync(unrecoverableFolder)
+                let src = path.join(originalFolder, file)
+                let dest = path.join(unrecoverableFolder, file)
                 fs.copyFileSync(src, dest)
             }
+            i++
+        }
+    }
+
+    /**
+     * Attempts to filter AI images on a folder containing images from pixiv.
+     */
+    public static filterAIImages = async (folder: string, pixivRefreshToken?: string) => {
+        const pixiv = await Pixiv.refreshLogin(pixivRefreshToken!)
+        const originalFolder = path.join(folder, "original")
+        const aiFolder = path.join(folder, "ai")
+        const errorFolder = path.join(folder, "error")
+        if (!fs.existsSync(originalFolder)) fs.mkdirSync(originalFolder)
+
+        this.moveImages(folder, originalFolder)
+
+        const files = fs.readdirSync(originalFolder).filter((f) => f !== ".DS_Store")
+        .sort(new Intl.Collator(undefined, {numeric: true, sensitivity: "base"}).compare)
+        let i = 1
+        for (const file of files) {
+            console.log(`${i}/${files.length}  -> ${file}`)
+            let pixivID = file.match(/^\d{5,}(?=\.|_)/)?.[0]
+            if (pixivID) {
+                try {
+                    let illust = await pixiv.illust.get(pixivID)
+                    if (pixiv.util.isAI(illust)) {
+                        if (!fs.existsSync(aiFolder)) fs.mkdirSync(aiFolder)
+                        let src = path.join(originalFolder, file)
+                        let dest = path.join(aiFolder, file)
+                        fs.renameSync(src, dest)
+                    }
+                } catch {
+                    if (!fs.existsSync(errorFolder)) fs.mkdirSync(errorFolder)
+                    let src = path.join(originalFolder, file)
+                    let dest = path.join(errorFolder, file)
+                    fs.renameSync(src, dest)
+                }
+            } else {
+                if (!fs.existsSync(errorFolder)) fs.mkdirSync(errorFolder)
+                let src = path.join(originalFolder, file)
+                let dest = path.join(errorFolder, file)
+                fs.renameSync(src, dest)
+            }
+            i++
         }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "animepic-utils",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Node.js utils for processing anime images",
   "main": "dist/image-utils.js",
   "types": "dist/image-utils.d.ts",
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "cheerio": "^1.1.2",
+    "pixiv.ts": "^0.7.9",
     "sharp": "^0.34.4",
     "waifu2x": "^1.5.1"
   }

package/pics.ts

@@ -2,8 +2,12 @@ import "dotenv/config"
 import imageUtils from "./image-utils"
 
 const start = async () => {
-    // await imageUtils.recoverFromDanbooru(process.env.FOLDER!)
-    await imageUtils.moepicsProcess(process.env.FOLDER!)
+    let pixivKey = process.env.PIXIV_REFRESH_TOKEN!
+    let folder = process.env.FOLDER!
+
+    // await imageUtils.recoverFromPixiv(folder, pixivKey)
+    await imageUtils.filterAIImages(folder, pixivKey)
+    // await imageUtils.moepicsProcess(process.env.FOLDER!)
     // imageUtils.changeQualifiers(process.env.FOLDER!, "c")
 }
 

package/pixivauth.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+from argparse import ArgumentParser
+from base64 import urlsafe_b64encode
+from hashlib import sha256
+from pprint import pprint
+from secrets import token_urlsafe
+from sys import exit
+from urllib.parse import urlencode
+from webbrowser import open as open_url
+
+import requests
+
+# Latest app version can be found using GET /v1/application-info/android
+USER_AGENT = "PixivAndroidApp/5.0.234 (Android 11; Pixel 5)"
+REDIRECT_URI = "https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback"
+LOGIN_URL = "https://app-api.pixiv.net/web/v1/login"
+AUTH_TOKEN_URL = "https://oauth.secure.pixiv.net/auth/token"
+CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
+CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
+
+
+def s256(data):
+    """S256 transformation method."""
+
+    return urlsafe_b64encode(sha256(data).digest()).rstrip(b"=").decode("ascii")
+
+
+def oauth_pkce(transform):
+    """Proof Key for Code Exchange by OAuth Public Clients (RFC7636)."""
+
+    code_verifier = token_urlsafe(32)
+    code_challenge = transform(code_verifier.encode("ascii"))
+
+    return code_verifier, code_challenge
+
+
+def print_auth_token_response(response):
+    data = response.json()
+
+    try:
+        access_token = data["access_token"]
+        refresh_token = data["refresh_token"]
+    except KeyError:
+        print("error:")
+        pprint(data)
+        exit(1)
+
+    print("access_token:", access_token)
+    print("refresh_token:", refresh_token)
+    print("expires_in:", data.get("expires_in", 0))
+
+
+def login():
+    code_verifier, code_challenge = oauth_pkce(s256)
+    login_params = {
+        "code_challenge": code_challenge,
+        "code_challenge_method": "S256",
+        "client": "pixiv-android",
+    }
+
+    open_url(f"{LOGIN_URL}?{urlencode(login_params)}")
+
+    try:
+        code = input("code: ").strip()
+    except (EOFError, KeyboardInterrupt):
+        return
+
+    response = requests.post(
+        AUTH_TOKEN_URL,
+        data={
+            "client_id": CLIENT_ID,
+            "client_secret": CLIENT_SECRET,
+            "code": code,
+            "code_verifier": code_verifier,
+            "grant_type": "authorization_code",
+            "include_policy": "true",
+            "redirect_uri": REDIRECT_URI,
+        },
+        headers={"User-Agent": USER_AGENT},
+    )
+
+    print_auth_token_response(response)
+
+
+def refresh(refresh_token):
+    response = requests.post(
+        AUTH_TOKEN_URL,
+        data={
+            "client_id": CLIENT_ID,
+            "client_secret": CLIENT_SECRET,
+            "grant_type": "refresh_token",
+            "include_policy": "true",
+            "refresh_token": refresh_token,
+        },
+        headers={"User-Agent": USER_AGENT},
+    )
+    print_auth_token_response(response)
+
+
+def main():
+    parser = ArgumentParser()
+    subparsers = parser.add_subparsers()
+    parser.set_defaults(func=lambda _: parser.print_usage())
+    login_parser = subparsers.add_parser("login")
+    login_parser.set_defaults(func=lambda _: login())
+    refresh_parser = subparsers.add_parser("refresh")
+    refresh_parser.add_argument("refresh_token")
+    refresh_parser.set_defaults(func=lambda ns: refresh(ns.refresh_token))
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

package/readme.md

@@ -9,6 +9,11 @@
 
 Some utilities for processing anime images. (However I guess it'll work for any images).
 
+### Pixiv Refresh Token
+
+To obtain your pixiv refresh token for the methods that require it, you can use the pixivauth.py script 
+from PixivPy. I included it in the base of the github repository.
+
 The primary function is `processImages` that accepts a folder of images, and then a variable 
 amount of processing functions that will be applied to every image in the folder. The processing functions 
 should take the current file parameter and return the path to the output, this is then fed back as the 
@@ -37,14 +42,32 @@ await imageUtils.fixFileExtensions(folder)
 
 ### Anime specific
 
-The function `recoverFromDanbooru` takes a folder of arbitrarily named images and attempts to recover 
-the original pixiv/twitter files from the largest mirror site, danbooru. Files which are unrecoverable 
-will be put in an "unrecoverable" folder, usually because it wasn't found on danbooru. 
+The function `recoverFromPixiv` takes a folder of arbitrarily named images and attempts to recover 
+the original images from pixiv, or danbooru as a fallback. Files which are unrecoverable 
+will be put into an "unrecoverable" folder, usually because it was deleted and not archived anywhere.
+
+```ts
+import imageUtils from "animepic-utils"
+
+await imageUtils.recoverFromPixiv(folder, process.env.PIXIV_REFRESH_TOKEN)
+```
+
+The `reverseImageSearch` function reverse searches the image on iqdb and returns the found danbooru post, 
+since usually this is the site with the most info.
+
+```ts
+import imageUtils from "animepic-utils"
+
+await imageUtils.reverseImageSearch(imagePath)
+```
+
+The `filterAIImages` function attempts to filter out AI images from a folder containing pixiv images. If 
+you have images that aren't on pixiv, you will have to use your best judgement...
 
 ```ts
 import imageUtils from "animepic-utils"
 
-await imageUtils.recoverFromDanbooru(folder)
+await imageUtils.filterAIImages(folder, process.env.PIXIV_REFRESH_TOKEN)
 ```
 
 The function `moepicsProcess` takes a folder of anime images and will generate the compressed