mediac 1.5.1 → 1.6.2

package/README.md

@@ -2,7 +2,7 @@
 
 MediaCli is a multimedia file processing tool that utilizes ffmpeg and exiftool, among others, to compress/convert/rename/delete/organize media files, including images, videos, and audio.
 
-created at 2021.07
+created at 2021.07, updated at 2024.04.07
 
 ## Installation
 
@@ -41,15 +41,18 @@ Commands:
                                           lder                     [aliases: mu]
   media_cli.js prefix <input> [output]    Rename files by append dir name or str
                                           ing                  [aliases: pf, px]
-
-Positionals:
-  input  Input folder that contains files                               [string]
+  media_cli.js fixname <input> [output]   Fix filenames (fix messy, clean, conve
+                                          rt tc to sc)        [aliases: fn, fxn]
+  media_cli.js zipu <input> [output]      Smart unzip command (auto detect encod
+                                          ing)             [aliases: zipunicode]
+  media_cli.js decode <strings...>        Decode text with messy or invalid char
+                                          s                        [aliases: dc]
 
 Options:
       --version  Show version number                                   [boolean]
   -h, --help     Show help                                             [boolean]
 
-Media Cli: Image/Raw/Video filename processing utilities
+MediaCli is a multimedia file processing tool.
 Copyright 2021-2025 @ Zhang Xiaoke
 
 ```

package/cmd/cmd_compress.js

@@ -113,12 +113,12 @@ const handler = async function cmdCompress(argv) {
     };
     log.showGreen(logTag, `Walking files ...`);
     let files = await mf.walk(root, walkOpts);
-    if (!files || files.length == 0) {
+    if (!files || files.length === 0) {
         log.showYellow(logTag, "no files found, abort.");
         return;
     }
     log.show(logTag, `total ${files.length} files found (all)`);
-    if (files.length == 0) {
+    if (files.length === 0) {
         log.showYellow("Nothing to do, abort.");
         return;
     }
@@ -164,7 +164,7 @@ const handler = async function cmdCompress(argv) {
     if (skipped > 0) {
         log.showYellow(logTag, `${skipped} thumbs skipped`)
     }
-    if (tasks.length == 0) {
+    if (tasks.length === 0) {
         log.showYellow("Nothing to do, abort.");
         return;
     }

package/cmd/cmd_dcim.js

@@ -132,7 +132,7 @@ const handler = async function cmdRename(argv) {
             `Total ${skippedByDate.length} media files are skipped by date`
         );
     }
-    if (files.length == 0) {
+    if (files.length === 0) {
         log.showYellow(LOG_TAG, "Nothing to do, exit now.");
         return;
     }
@@ -160,7 +160,7 @@ const handler = async function cmdRename(argv) {
             log.showYellow(LOG_TAG, `All ${files.length} files, NO file renamed in TEST MODE.`);
         }
         else {
-            const results = await renameFiles(files);
+            const results = await renameFiles(files, false);
             log.showGreen(LOG_TAG, `All ${results.length} file were renamed.`,);
         }
     } else {

package/cmd/cmd_decode.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+/*
+ * File: cmd_decode.js
+ * Created: 2024-04-05 17:01:29
+ * Modified: 2024-04-08 22:18:50
+ * Author: mcxiaoke (github@mcxiaoke.com)
+ * License: Apache License 2.0
+ */
+
+
+import chardet from 'chardet';
+import * as log from '../lib/debug.js';
+import * as enc from '../lib/encoding.js';
+import * as unicode from '../lib/unicode.js';
+
+
+const ENC_LIST = [
+    'ISO-8859-1',
+    'UTF8',
+    'UTF-16',
+    'GBK',
+    'BIG5',
+    'SHIFT_JIS',
+    'EUC-JP',
+    'EUC-KR',
+]
+
+export { aliases, builder, command, describe, handler };
+const command = "decode <strings...>"
+const aliases = ["dc"]
+const describe = 'Decode text with messy or invalid chars'
+
+const builder = function addOptions(ya, helpOrVersionSet) {
+    return ya
+        .positional('strings', {
+            describe: 'string list to decode',
+            type: 'string',
+        })
+        // 修复文件名乱码
+        .option("from-enc", {
+            alias: "f",
+            type: "choices",
+            choices: ['utf8', 'gbk', 'shift_jis', 'big5', 'euc-kr'],
+            description: "from encoding name eg. utf8|gbk|shift_jis",
+        })
+        .option("to-enc", {
+            alias: "t",
+            type: "choices",
+            choices: ['utf8', 'gbk', 'shift_jis', 'big5', 'euc-kr'],
+            description: "to encoding name tg. utf8|gbk|shift_jis",
+        }).po
+}
+
+const handler = async function cmdDecode(argv) {
+    const logTag = "cmdDecode";
+    log.info(logTag, 'Args:', argv);
+    const strArgs = argv.strings;
+    if (strArgs?.length === 0) {
+        throw new Error(`text input required`);
+    }
+    const fromEnc = argv.fromEnc?.length > 0 ? [argv.fromEnc] : ENC_LIST;
+    const toEnc = argv.toEnc?.length > 0 ? [argv.toEnc] : ENC_LIST;
+    const threhold = log.isVerbose() ? 1 : 50;
+    log.show(logTag, `Input:`, strArgs)
+    log.show(logTag, `fromEnc:`, JSON.stringify(fromEnc))
+    log.show(logTag, `toEnc:`, JSON.stringify(toEnc))
+
+    for (const str of strArgs) {
+        log.show(logTag, 'TryDecoding:', [str])
+        const results = decodeText(str, fromEnc, toEnc, threhold)
+        results.forEach(showResults)
+        log.show('INPUT:', [str, str.length],)
+        log.show('OUPUT:', results.pop())
+        console.log()
+    }
+}
+
+function decodeText(str, fromEnc = ENC_LIST, toEnc = ENC_LIST, threhold = 50) {
+    let results = enc.tryDecodeText(str, fromEnc, toEnc, threhold)
+    return results.reverse()
+}
+
+function showResults(r) {
+    log.info(`-`)
+    const str = r[0]
+    const print = (a, b) => log.info(a.padEnd(16, ' '), b)
+    log.show('Result:', str.padEnd(16, ' '), r.slice(1))
+    let cr = chardet.analyse(Buffer.from(str))
+    cr = cr.filter(ct => ct.confidence >= 70)
+    cr?.length > 0 && print('Encoding', cr)
+    print('String', Array.from(str))
+    print('Unicode', Array.from(str).map(c => c.codePointAt(0).toString(16)))
+    const badUnicode = enc.checkBadUnicode(str)
+    badUnicode?.length > 0 && log.info(`badUnicode=true`)
+    log.info(`MESSY_UNICODE=${enc.REGEX_MESSY_UNICODE.test(str)}`,
+        `MESSY_CJK=${enc.REGEX_MESSY_CJK.test(str)}`,
+        `MESSY_CJK_EXT=${enc.REGEX_MESSY_CJK_EXT.test(str)}`)
+    log.info(`OnlyJapanese=${unicode.strOnlyJapanese(str)}`,
+        `OnlyJpHan=${unicode.strOnlyJapaneseHan(str)}`,
+        `HasHiraKana=${unicode.strHasHiraKana(str)}`
+    )
+    log.info(`HasHangul=${unicode.strHasHangul(str)}`,
+        `OnlyHangul=${unicode.strOnlyHangul(str)}`)
+    log.info(`HasChinese=${unicode.strHasChinese(str)}`,
+        `OnlyChinese=${unicode.strOnlyChinese(str)}`,
+        `OnlyChn3500=${enc.RE_CHARS_MOST_USED.test(str)}`)
+}

package/cmd/cmd_moveup.js

@@ -172,7 +172,7 @@ const handler = async function cmdMoveUp(argv) {
             }
         }
         log.showGreen(logTag, `${files.length} files in ${helper.pathShort(subDirPath)} are moved.`, testMode ? "[DRY RUN]" : "");
-    };
+    }
     log.showGreen(logTag, `Total ${movedCount}/${totalCount} files moved.`, testMode ? "[DRY RUN]" : "");
     log.showYellow(logTag, "There are some unused folders left after moving up operations.")
 
@@ -190,7 +190,8 @@ const handler = async function cmdMoveUp(argv) {
     }
 
     keepDirList = new Set([...keepDirList].map(x => path.resolve(x)));
-    let subDirList = await mf.walkDir(root);
+    let subDirEntries = await mf.walk(root, { withDirs: true, withFiles: false });
+    let subDirList = subDirEntries.map(x => x.path);
     subDirList = new Set([...subDirList].map(x => path.resolve(x)));
     const toRemoveDirList = setDifference(subDirList, keepDirList)
 
@@ -219,7 +220,7 @@ const handler = async function cmdMoveUp(argv) {
             ++delCount;
             log.fileLog(`SafeDel: <${td}>`, logTag);
         }
-        log.showGreen(logTag, "SafeDel", helper.pathShort(td), testMode ? "[DRY RUN]" : "");
+        log.show(logTag, "SafeDel", helper.pathShort(td), testMode ? "[DRY RUN]" : "");
     }
     log.showGreen(logTag, `${delCount} dirs were SAFE DELETED ${testMode ? "[DRY RUN]" : ""}`);
 }

package/cmd/cmd_prefix.js

@@ -18,7 +18,7 @@ import { asyncFilter } from '../lib/core.js';
 import * as log from '../lib/debug.js';
 import * as mf from '../lib/file.js';
 import * as helper from '../lib/helper.js';
-import { renameFiles } from "./cmd_shared.js";
+import { RE_MEDIA_DIR_NAME, RE_ONLY_ASCII, RE_ONLY_NUMBER, RE_UGLY_CHARS, RE_UGLY_CHARS_BORDER, cleanFileName, renameFiles } from "./cmd_shared.js";
 
 const MODE_AUTO = "auto";
 const MODE_DIR = "dirname";
@@ -111,95 +111,7 @@ const builder = function addOptions(ya, helpOrVersionSet) {
         })
 }
 
-// 正则：仅包含数字
-const reOnlyNum = /^\d+$/gi;
-// 视频文件名各种前后缀
-const reVideoName = helper.combineRegexG(
-    /HD1080P|2160p|1080p|720p|BDRip/,
-    /H264|H265|X265|HEVC|AVC|8BIT|10bit/,
-    /WEB-DL|SMURF|Web|AAC5\.1|Atmos/,
-    /H\.264|DD5\.1|DDP5\.1|AAC/,
-    /DJWEB|Play|VINEnc|DSNP|END/,
-    /高清|特效|字幕组|公众号|画质|电影|搬运/,
-    /\[.+?\]/,
-)
-// 图片文件名各种前后缀
-const reImageName = /更新|合集|画师|图片|视频|插画|视图|作品|订阅|限定|差分|拷贝|自购|付费|内容|R18|PSD|PIXIV|PIC|ZIP|RAR/gi
-// Unicode Symbols
-// https://en.wikipedia.org/wiki/Script_%28Unicode%29
-// https://www.regular-expressions.info/unicode.html
-// https://symbl.cc/cn/unicode/blocks/halfwidth-and-fullwidth-forms/
-// https://www.unicode.org/reports/tr18/
-// https://ayaka.shn.hk/hanregex/
-// 特例字符	中英	全半角	unicode范围	unicode码表名
-// 单双引号	中文	全/半	0x2018-0x201F	常用标点
-// 句号、顿号	中文	全/半	0x300x-0x303F	中日韩符号和标点
-// 空格	中/英	全角	0x3000	中日韩符号和标点
-// -	英	半角	0x0021~0x007E	半角符号
-// -	英	全角	0xFF01~0xFF5E	全角符号
-// -	中	全/半	0xFF01~0xFF5E	全角符号
-// 正则：匹配除 [中文日文标点符号] 之外的特殊字符
-// u flag is required
-// \p{sc=Han} CJK全部汉字 比 \u4E00-\u9FFF = \p{InCJK_Unified_Ideographs} 范围大
-// 匹配汉字还可以使用 \p{Unified_Ideograph}
-// \p{sc=Hira} 日文平假名
-// \p{P} 拼写符号
-// \p{ASCII} ASCII字符
-// \uFE10-\uFE1F 中文全角标点
-// \uFF01-\uFF11 中文全角标点
-const reNonChars = /[^\p{Unified_Ideograph}\p{sc=Hira}\p{sc=Kana}\w]/ugi;
-// 匹配空白字符和特殊字符
-// https://www.unicode.org/charts/PDF/U3000.pdf
-// https://www.asciitable.com/
-const reUglyChars = /[\s\x00-\x1F\x21-\x2F\x3A-\x40\x5B-\x60\x7b-\xFF]+/gi;
-// 匹配开头和结尾的空白和特殊字符
-const reStripUglyChars = /(^[\s\x21-\x2F\x3A-\x40\x5B-\x60\x7b-\xFF\p{P}]+)|([\s\x21-\x2F\x3A-\x40\x5B-\x60\x7b-\xFF\p{P}]+$)/gi;
-// 图片视频子文件夹名过滤
-// 如果有表示，test() 会随机饭后true or false，是一个bug
-// 使用 string.match 函数没有问题
-// 参考 https://stackoverflow.com/questions/47060553
-// The g modifier causes the regex object to maintain state. 
-// It tracks the index after the last match.
-const reMediaDirName = /^图片|视频|电影|电视剧|Image|Video|Thumbs$/gi;
-// 可以考虑将日文和韩文罗马化处理
-// https://github.com/lovell/hepburn
-// https://github.com/fujaru/aromanize-js
-// https://www.npmjs.com/package/aromanize
-// https://www.npmjs.com/package/@lazy-cjk/japanese
-function cleanFileName(nameString, sep, filename, keepNumber = false) {
-    let nameStr = nameString;
-    // 去掉方括号 [xxx] 的内容
-    // nameStr = nameStr.replaceAll(/\[.+?\]/gi, "");
-    // 去掉图片集说明文字
-    nameStr = nameStr.replaceAll(reImageName, sep);
-    // 去掉视频说明文字
-    nameStr = nameStr.replaceAll(reVideoName, "");
-    // 去掉日期字符串
-    if (!keepNumber) {
-        nameStr = nameStr.replaceAll(/\d+年\d+月/ugi, "");
-        nameStr = nameStr.replaceAll(/\d{4}-\d{2}-\d{2}/ugi, "");
-    }
-    // 去掉 [100P5V 2.25GB] No.46 这种图片集说明
-    nameStr = nameStr.replaceAll(/\[\d+P.*(\d+V)?.*?\]/ugi, "");
-    nameStr = nameStr.replaceAll(/No\.\d+|\d+\.?\d+GB?|\d+P|\d+V|NO\.(\d+)/ugi, "$1");
-    if (helper.isImageFile(filename)) {
-        // 去掉 2024.03.22 这种格式的日期
-        nameStr = nameStr.replaceAll(/\d{4}\.\d{2}\.\d{2}/ugi, "");
-    }
-    // 去掉中文标点特殊符号
-    nameStr = nameStr.replaceAll(/[\u3000-\u303F\uFE10-\uFE2F]/ugi, "");
-    // () [] {} <> . - 改为下划线
-    nameStr = nameStr.replaceAll(/[\(\)\[\]{}<>\.\-]/ugi, sep);
-    // 日文转罗马字母
-    // nameStr = hepburn.fromKana(nameStr);
-    // nameStr = wanakana.toRomaji(nameStr);
-    // 韩文转罗马字母
-    // nameStr = aromanize.hangulToLatin(nameStr, 'rr-translit');
-    // 繁体转换为简体中文
-    nameStr = sify(nameStr);
-    // 去掉所有特殊字符
-    return nameStr.replaceAll(reNonChars, sep);
-}
+
 
 function getAutoModePrefix(dir, sep) {
     // 从左到右的目录层次
@@ -227,7 +139,8 @@ function parseNameMode(argv) {
 }
 
 // 重复文件名Set，检测重复，防止覆盖
-const nameDuplicateSet = new Set();
+const nameDupSet = new Set();
+let nameDupIndex = 0;
 async function createNewNameByMode(f) {
     const argv = f.argv;
     const mode = parseNameMode(argv);
@@ -260,7 +173,7 @@ async function createNewNameByMode(f) {
             {
                 sep = ".";
                 prefix = dirName;
-                if (prefix.match(reMediaDirName)) {
+                if (prefix.match(RE_MEDIA_DIR_NAME)) {
                     prefix = dirParts[2];
                 }
                 if (prefix.length < 4 && /^[A-Za-z0-9]+$/.test(prefix)) {
@@ -278,7 +191,7 @@ async function createNewNameByMode(f) {
             {
                 sep = "_";
                 prefix = dirName;
-                if (prefix.match(reMediaDirName)) {
+                if (prefix.match(RE_MEDIA_DIR_NAME)) {
                     prefix = dirParts[2];
                 }
             }
@@ -287,9 +200,10 @@ async function createNewNameByMode(f) {
             {
                 sep = "_";
                 prefix = getAutoModePrefix(dir, sep);
-                const applyToAll = argv.all || false;
-                if (!reOnlyNum.test(base) && !applyToAll) {
-                    log.showYellow(logTag, `Ignore: ${ipx} ${helper.pathShort(f.path)}`);
+                const shouldCheck = RE_ONLY_NUMBER.test(base) && base.length < 10;
+                const forceAll = argv.all || false;
+                if (!shouldCheck && !forceAll) {
+                    log.info(logTag, `Ignore: ${ipx} ${helper.pathShort(f.path)} [Auto]`);
                     return;
                 }
             }
@@ -305,54 +219,68 @@ async function createNewNameByMode(f) {
             throw new Error(`No prefix supplied!`);
         }
     }
-    log.show(prefix)
-    let newPathFixed = null;
+
     // 是否净化文件名，去掉各种特殊字符
     if (argv.clean || mode === MODE_CLEAN) {
-        prefix = cleanFileName(prefix, sep, oldName, false);
-        oldBase = cleanFileName(oldBase, sep, oldName, true);
+        prefix = cleanFileName(prefix, { separator: sep, keepDateStr: false, tc2sc: true });
+        oldBase = cleanFileName(oldBase, { separator: sep, keepDateStr: true, tc2sc: true });
     }
     // 不添加重复前缀
     if (oldBase.includes(prefix)) {
         log.info(logTag, `IgnorePrefix: ${ipx} ${helper.pathShort(f.path)}`);
         prefix = "";
     }
+    // 确保文件名不含有文件系统不允许的非法字符
+    oldBase = helper.filenameSafe(oldBase);
     let fullBase = prefix.length > 0 ? (prefix + sep + oldBase) : oldBase;
     // 去除首位空白和特殊字符
-    fullBase = fullBase.replaceAll(reStripUglyChars, "");
+    fullBase = fullBase.replaceAll(RE_UGLY_CHARS_BORDER, "");
     // 多余空白和字符替换为一个字符 _或.
-    fullBase = fullBase.replaceAll(reUglyChars, sep);
+    fullBase = fullBase.replaceAll(RE_UGLY_CHARS, sep);
     // 去掉重复词组，如目录名和人名
     fullBase = Array.from(new Set(fullBase.split(sep))).join(sep)
-    fullBase = unicodeStrLength(fullBase) > nameLength ? fullBase.slice(nameSlice) : fullBase;
+    fullBase = helper.unicodeLength(fullBase) > nameLength ? fullBase.slice(nameSlice) : fullBase;
     // 再次去掉首位的特殊字符和空白字符
-    fullBase = fullBase.replaceAll(reStripUglyChars, "");
+    fullBase = fullBase.replaceAll(RE_UGLY_CHARS_BORDER, "");
 
-    const newName = `${fullBase}${ext}`;
-    const newPath = newPathFixed ?? path.join(dir, newName);
+    let newName = `${fullBase}${ext}`;
+    let newPath = path.resolve(path.join(dir, newName));
     if (newPath === f.path) {
         log.info(logTag, `Same: ${ipx} ${helper.pathShort(newPath)}`);
         f.skipped = true;
     }
     else if (await fs.pathExists(newPath)) {
-        log.info(logTag, `Exists: ${ipx} ${helper.pathShort(newPath)}`);
-        f.skipped = true;
+        // 目标文件已存在
+        const stn = await fs.stat(newPath);
+        if (f.stats.size === stn.size) {
+            // 如果大小相等，认为是同一个文件
+            log.info(logTag, `Exists: ${ipx} ${helper.pathShort(newPath)}`);
+            f.skipped = true;
+        } else {
+            // 大小不相等，文件名添加后缀
+            // 找到一个不重复的新文件名
+            do {
+                newName = `${fullBase}${sep}D${++nameDupIndex}${ext}`;
+                newPath = path.resolve(path.join(dir, newName));
+            } while (nameDupSet.has(newPath))
+            log.info(logTag, `NewName: ${ipx} ${helper.pathShort(newPath)}`);
+        }
     }
-    else if (nameDuplicateSet.has(newPath)) {
+    else if (nameDupSet.has(newPath)) {
         log.info(logTag, `Duplicate: ${ipx} ${helper.pathShort(newPath)}`);
         f.skipped = true;
     }
-    nameDuplicateSet.add(newPath);
+    nameDupSet.add(newPath);
     if (f.skipped) {
         // log.fileLog(`Skip: ${ipx} ${f.path}`, logTag);
         // log.showGray(logTag, `Skip: ${ipx} ${f.path}`);
     } else {
         f.outName = newName;
         f.outPath = newPath;
-        log.show(logTag, `${ipx} ${f.path}`);
-        log.showGreen(logTag, `${ipx} ${newPath}`);
-        log.fileLog(`Prepare: ${ipx} <${f.path}> [FROM]`, logTag);
-        log.fileLog(`Prepare: ${ipx} <${newPath}> [TOTO]`, logTag);
+        log.showGray(logTag, `SRC: ${ipx} ${helper.pathShort(f.path)}`);
+        log.show(logTag, `DST: ${ipx} ${helper.pathShort(newPath)}`);
+        log.fileLog(`Prepare: ${ipx} <${f.path}> [SRC]`, logTag);
+        log.fileLog(`Prepare: ${ipx} <${newPath}> [DST]`, logTag);
     }
     return f;
 }
@@ -388,7 +316,6 @@ const handler = async function cmdPrefix(argv) {
     if (argv.include?.length >= 3) {
         // 处理include规则
         const pattern = new RegExp(argv.include, "gi");
-        log.showRed(pattern)
 
         files = await asyncFilter(files, x => x.path.match(pattern));
         log.show(logTag, `Total ${files.length} files left after include rules`);
@@ -417,6 +344,15 @@ const handler = async function cmdPrefix(argv) {
     let tasks = await pMap(files, createNewNameByMode, { concurrency: cpus().length * 4 })
     tasks = tasks.filter(f => f?.outName)
 
+    tasks = tasks.map((f, i) => {
+        return {
+            ...f,
+            argv: argv,
+            index: i,
+            total: files.length,
+        }
+    })
+
     const tCount = tasks.length;
     log.showYellow(
         logTag, `Total ${fCount - tCount} files are skipped.`
@@ -446,55 +382,13 @@ const handler = async function cmdPrefix(argv) {
     ]);
     if (answer.yes) {
         if (testMode) {
-            log.showYellow(logTag, `All ${tasks.length} files, BUT NO file renamed in TEST MODE.`);
+            log.showYellow(logTag, `${tasks.length} files, NO file renamed in TEST MODE.`);
         }
         else {
-            const results = await renameFiles(tasks);
+            const results = await renameFiles(tasks, false);
             log.showGreen(logTag, `All ${results.length} file were renamed.`);
         }
     } else {
         log.showYellow(logTag, "Will do nothing, aborted by user.");
     }
-}
-
-
-
-// 计算字符串长度，中文算2，英文算1
-function unicodeStrLength(str) {
-    var len = 0;
-    for (var i = 0; i < str.length; i++) {
-        var c = str.charCodeAt(i);
-        //单字节加1 
-        if ((c >= 0x0001 && c <= 0x007e) || (0xff60 <= c && c <= 0xff9f)) {
-            len++;
-        }
-        else {
-            len += 2;
-        }
-    }
-    return len;
-}
-
-function unicodeStrSlice(str, len) {
-    var str_length = 0;
-    var str_len = 0;
-    str_cut = new String();
-    str_len = str.length;
-    for (var i = 0; i < str_len; i++) {
-        a = str.charAt(i);
-        str_length++;
-        if (encodeURI(a).length > 4) {
-            //中文字符的长度经编码之后大于4
-            str_length++;
-        }
-        str_cut = str_cut.concat(a);
-        if (str_length >= len) {
-            // str_cut = str_cut.concat("...");
-            return str_cut;
-        }
-    }
-    //如果给定字符串小于指定长度，则返回源字符串；
-    if (str_length < len) {
-        return str;
-    }
 }