node-automator 1.0.0

package/utils/file_tool.js

@@ -0,0 +1,529 @@
+const glob = require("glob");
+const path = require("path");
+const fs = require("fs");
+const display_tool = require("./display_tool");
+const {
+    info,
+    whisper,
+    warn,
+    setLastError
+} = require("../utils/log_tool");
+const tmps = {
+    arr: [],
+};
+const iconv = require('iconv-lite');
+// const chardet = require("jschardet");
+const chardet = require("chardet");
+const { get_full_path } = require("../commands/share_data");
+const { parse, stringify } = require("./parse_tool");
+/**
+ * 
+ * @param {Array | string} src 
+ */
+function get_file_list(src, noGlob = false, onlyFile = false, real = false) {
+    src = tmps.arr.concat(src);
+    let files = [];
+    src.forEach(v => {
+        v = get_full_path(v);
+        if (!noGlob) {
+            try {
+                !fs.lstatSync(v);
+                files.push(v);
+            } catch (error) {
+                Array.prototype.push.apply(files, glob.sync(v));
+            }
+        } else {
+            files.push(v);
+        }
+    });
+    if (onlyFile) {
+        files = files.filter(v => {
+            return !fs.lstatSync(v).isDirectory();
+        });
+    }
+    if (real) {
+        files = files.map(v => v && fs.realpathSync(v));
+    }
+    return files;
+}
+
+function get_fst_file(src, real, noGlob) {
+    let files = get_file_list(src, noGlob, undefined, real);
+    let file = files[0];
+    return file;
+}
+
+function get_file_size(src) {
+    let files = get_file_list(src);
+    return get_files_size(files);
+}
+
+function get_files_size(files) {
+    return files.reduce((prev, file) => {
+        let stat = fs.statSync(file);
+        return stat.size += prev;
+    }, 0);
+}
+
+function format_bytes(size) {
+    if (size < 2 ** 10) {
+        return `${size}B`;
+    }
+    if (size < 2 ** 20) {
+        return `${+(size / 2 ** 10).toFixed(2)}KB`;
+    }
+    if (size < 2 ** 30) {
+        return `${+(size / 2 ** 20).toFixed(2)}MB`;
+    }
+    if (size < 2 ** 40) {
+        return `${+(size / 2 ** 30).toFixed(2)}GB`;
+    }
+    if (size < 2 ** 50) {
+        return `${+(size / 2 ** 40).toFixed(2)}TB`;
+    }
+    if (size < 2 ** 60) {
+        return `${+(size / 2 ** 50).toFixed(2)}PB`;
+    }
+    if (size < 2 ** 70) {
+        return `${+(size / 2 ** 60).toFixed(2)}EB`;
+    }
+    if (size < 2 ** 80) {
+        return `${+(size / 2 ** 70).toFixed(2)}ZB`;
+    }
+    return `${+(size / 2 ** 80).toFixed(2)}YB`;
+}
+
+function read_plain(src, options) {
+    try {
+        let fst_src = get_fst_file(src);
+        if (!fst_src) {
+            setLastError(`${src} 未找到`);
+            return;
+        }
+        let value = fs.readFileSync(fst_src, {
+            flag: "r"
+        });
+        if (!options?.raw) {
+            let encoding = options && options.encoding;
+            let auto_encoding = options && options.auto_encoding;
+            if (auto_encoding) {
+                // encoding = chardet.detect(value).encoding;
+                encoding = chardet.detect(value);
+            } else {
+                encoding = encoding || "utf-8";
+            }
+            value = iconv.decode(value, encoding);
+        }
+        return value;
+    } catch (err) {
+        throw (`读取文本文件 ${err}`);
+    }
+}
+
+function write_plain(dst, data, options) {
+    try {
+        dst = get_full_path(dst);
+        options = Object.assign({}, {
+            encoding: "utf-8",
+            flag: "w"
+        }, options);
+        // 创建不存在的目录
+        let dstDir = path.dirname(dst);
+        if (!fs.existsSync(dstDir)) {
+            fs.mkdirSync(dstDir, {
+                recursive: true
+            });
+        }
+        let value;
+        if (options.raw) {
+            value = fs.writeFileSync(dst, data);
+        } else {
+            value = fs.writeFileSync(dst, data, options);
+        }
+        return value;
+    } catch (err) {
+        throw (`保存文本文件 ${err}`);
+    }
+}
+
+async function write_with_type(data, dst, type, options) {
+    if (Buffer.isBuffer(data) || typeof data  === "string") {
+        write_plain(dst, data);
+    } else {
+        if (!type) {
+            const extname = path.extname(dst);
+            switch (extname) {
+                case ".yaml":
+                case ".yml": {
+                    type = "yaml";
+                    break;
+                }
+                case ".json": {
+                    type = "json";
+                    break;
+                }
+            }
+        }
+        switch (options?.mode) {
+            case "merge": {
+                data = {...(await read_cfg(dst, type)), ...data};
+                break;
+            }
+            case "concat": {
+                data = [...(await read_cfg(dst, type) || []), ...data];
+                break;
+            }
+        }
+        let content = await stringify(data, type, options);
+        write_plain(dst, content);
+    }
+}
+
+async function read_cfg(src, type, options) {
+    src = get_fst_file(src);
+    if (!src) {
+        return;
+    }
+    if (!type) {
+        const extname = path.extname(src);
+        switch (extname) {
+            case ".yaml":
+            case ".yml": {
+                type = "yaml";
+                break;
+            }
+            case ".xml": {
+                type = "xml";
+                break;
+            }
+            case ".lua": {
+                type = "lua";
+                break;
+            }
+            default: {
+                type = "json";
+                break;
+            }
+        }
+    }
+    let content = await read_plain(src, options);
+    if (content == null) {
+        return;
+    }
+    return parse(content, type, options);
+}
+
+function processRename(fullpath, params) {
+    if (!params) return fullpath;
+    let dirname = path.dirname(fullpath);
+    switch (typeof params) {
+        case "object": {
+            let extname = path.extname(fullpath);
+            let basename = path.basename(fullpath, extname);
+            if (params.base_trim_left) {
+                if (basename.indexOf(params.base_trim_left) == 0) {
+                    basename = basename.substr(params.base_trim_left.length);
+                }
+            }
+            if (params.base_trim_right) {
+                if (basename.indexOf(params.base_trim_right) == basename.length - params.base_trim_right.length) {
+                    basename = basename.substr(0, basename.length - params.base_trim_right.length);
+                }
+            }
+            if (params.dir_trim_left) {
+                if (dirname.indexOf(params.dir_trim_left) == 0) {
+                    dirname = dirname.substr(params.dir_trim_left.length);
+                }
+            }
+            if (params.dir_trim_right) {
+                if (dirname.indexOf(params.dir_trim_right) == dirname.length - params.dir_trim_right.length) {
+                    dirname = dirname.substr(0, dirname.length - params.dir_trim_right.length);
+                }
+            }
+            if (params.dir_prefix) dirname = params.dir_prefix + dirname;
+            if (params.dir_suffix) dirname = dirname + params.dir_suffix;
+            if (params.base_prefix) basename = params.base_prefix + basename;
+            if (params.base_suffix) basename = basename + params.base_suffix;
+            return path.join(dirname, basename + extname);
+        }
+    }
+    return path.join(dirname, params);
+}
+
+async function copy(src, dst, base, options) {
+    srcs = get_file_list(src);
+    dsts = get_file_list(dst, true);
+    if (base) {
+        base = get_fst_file(base);
+        if (!base) {
+            return;
+        }
+    }
+    options = Object.assign({
+        overwrite: true
+    }, options);
+    let filesize = get_files_size(srcs);
+    info("");
+    whisper(`复制 ${srcs[0]} 等 ${srcs.length} 个文件(夹) ${format_bytes(filesize)} ${options.overwrite ? "[覆盖现有文件]" : "[不覆盖现有文件]"} 到 ${dsts} options: ${JSON.stringify(options)}`, undefined, true);
+    let dir_num = 0;
+    let actual_files = [];
+    for (let srcIndex = 0; srcIndex < srcs.length; srcIndex++) {
+        const src = srcs[srcIndex];
+        let show_src = !base ? src : path.relative(base, src);
+        if (fs.lstatSync(src).isDirectory()) {
+            ++dir_num;
+            continue;
+        }
+        display_tool.progress(srcIndex + 1, srcs.length, {
+            desc: `Copy ${show_src}`,
+            depth: 0,
+            color: "green",
+            align: true,
+        });
+        for (let dstIndex = 0; dstIndex < dsts.length; dstIndex++) {
+            const dst = dsts[dstIndex];
+            // 完整的目标文件路径
+            let dstFile;
+            if (!base) {
+                dstFile = path.join(dst, path.basename(src));
+            } else {
+                dstFile = path.join(dst, path.relative(base, src));
+            }
+            dstFile = processRename(dstFile, options.rename);
+            let dstDir = path.dirname(dstFile);
+            if (!fs.existsSync(dstDir) || !fs.lstatSync(dstDir).isDirectory()) {
+                try {
+                    fs.mkdirSync(dstDir, {
+                        recursive: true
+                    });
+                } catch (err) {
+                    throw (err);
+                }
+            }
+            if (fs.existsSync(dstFile) && !options.overwrite) {
+                continue;
+            } else {
+                if (!actual_files.includes(src)) {
+                    actual_files.push(src);
+                }
+                await new Promise(resolve => {
+                    // !options.overwrite ? fs.constants.COPYFILE_EXCL : 0
+                    fs.copyFile(src, dstFile, 0, (err) => {
+                        if (err) {
+                            throw (err);
+                        }
+                        resolve();
+                    });
+                });
+            }
+        }
+    }
+    
+    info("");
+    whisper(`复制完成 (文件: ${actual_files.length}, 目录: ${dir_num}, 总计: ${srcs.length})`, undefined, true);
+}
+
+async function move(src, dst, base, options) {
+    srcs = get_file_list(src);
+    dsts = get_file_list(dst, true);
+    if (base) {
+        base = get_fst_file(base);
+        if (!base) {
+            return;
+        }
+    }
+    options = Object.assign({
+        overwrite: true
+    }, options);
+    let filesize = get_files_size(srcs);
+    info("");
+    whisper(`移动 ${srcs[0]} 等 ${srcs.length} 个文件(夹) ${format_bytes(filesize)} ${options.overwrite ? "[覆盖现有文件]" : "[不覆盖现有文件]"} 到 ${dsts} options: ${JSON.stringify(options)}`, undefined, true);
+    let actual_files = [];
+    let dir_num = 0;
+    for (let srcIndex = 0; srcIndex < srcs.length; srcIndex++) {
+        const src = srcs[srcIndex];
+        let show_src = !base ? src : path.relative(base, src);
+        if (fs.lstatSync(src).isDirectory()) {
+            ++dir_num;
+            continue;
+        }
+        display_tool.progress(srcIndex + 1, srcs.length, {
+            desc: `Move ${show_src}`,
+            depth: 0,
+            color: "cyan",
+            align: true,
+        });
+        // 先复制
+        for (let dstIndex = 0; dstIndex < dsts.length; dstIndex++) {
+            const dst = dsts[dstIndex];
+            // 完整的目标文件路径
+            let dstFile;
+            if (!base) {
+                dstFile = path.join(dst, path.basename(src));
+            } else {
+                dstFile = path.join(dst, path.relative(base, src));
+            }
+            dstFile = processRename(dstFile, options.rename);
+            let dstDir = path.dirname(dstFile);
+            if (!fs.existsSync(dstDir) || !fs.lstatSync(dstDir).isDirectory()) {
+                fs.mkdirSync(dstDir, {
+                    recursive: true
+                });
+            }
+            if (fs.existsSync(dstFile) && !options.overwrite) {
+                continue;
+            } else {
+                if (!actual_files.includes(src)) {
+                    actual_files.push(src);
+                }
+                await new Promise(resolve => {
+                    if (!options.overwrite && fs.existsSync(dstFile)) {
+                        resolve();
+                    } else {
+                        fs.copyFile(src, dstFile, 0, (err) => {
+                            if (err) {
+                                throw (err);
+                            }
+                            resolve();
+                        });
+                    }
+                });
+            }
+        }
+        // 再删除
+        fs.unlinkSync(src);
+    }
+    
+    info("");
+    whisper(`移动完成 (文件: ${actual_files.length}, 目录: ${dir_num}, 总计: ${srcs.length})`, undefined, true);
+}
+
+async function move_local(src, dst, base, options) {
+    srcs = get_file_list(src);
+    dsts = get_file_list(dst, true);
+    if (base) {
+        base = get_fst_file(base);
+        if (!base) {
+            return;
+        }
+    }
+    options = Object.assign({
+        overwrite: true
+    }, options);
+    let filesize = get_files_size(srcs);
+    info("");
+    whisper(`移动 ${srcs[0]} 等 ${srcs.length} 个文件(夹) ${format_bytes(filesize)} ${options.overwrite ? "[覆盖现有文件]" : "[不覆盖现有文件]"} 到 ${dsts} options: ${JSON.stringify(options)}`, undefined, true);
+    let actual_files = [];
+    let dir_num = 0;
+    for (let srcIndex = 0; srcIndex < srcs.length; srcIndex++) {
+        const src = srcs[srcIndex];
+        let show_src = !base ? src : path.relative(base, src);
+        if (fs.lstatSync(src).isDirectory()) {
+            ++dir_num;
+        }
+        display_tool.progress(srcIndex + 1, srcs.length, {
+            desc: `Rename ${show_src}`,
+            depth: 0,
+            color: "cyan",
+            align: true,
+        });
+        for (let dstIndex = 0; dstIndex < dsts.length; dstIndex++) {
+            const dst = dsts[dstIndex];
+            // 完整的目标文件路径
+            let dstFile;
+            if (!base) {
+                dstFile = path.join(dst, path.basename(src));
+            } else {
+                dstFile = path.join(dst, path.relative(base, src));
+            }
+            dstFile = processRename(dstFile, options.rename);
+            let dstDir = path.dirname(dstFile);
+            if (!fs.existsSync(dstDir) || !fs.lstatSync(dstDir).isDirectory()) {
+                fs.mkdirSync(dstDir, {
+                    recursive: true
+                });
+            }
+            if (fs.existsSync(dstFile) && !options.overwrite) {
+                continue;
+            } else {
+                if (!actual_files.includes(src)) {
+                    actual_files.push(src);
+                }
+                skip = false;
+                await new Promise(resolve => {
+                    if (!options.overwrite && fs.existsSync(dstFile)) {
+                        resolve();
+                    } else {
+                        fs.rename(src, dstFile, (err) => {
+                            if (err) {
+                                throw (err);
+                            }
+                            resolve();
+                        });
+                    }
+                });
+            }
+        }
+    }
+    
+    info("");
+    whisper(`移动完成 (文件: ${actual_files.length}, 目录: ${dir_num}, 总计: ${srcs.length})`, undefined, true);
+}
+
+async function remove(src, options) {
+    srcs = get_file_list(src);
+    options = Object.assign({ recursive: true }, options);
+    let filesize = get_files_size(srcs);
+    info("");
+    whisper(`删除 ${srcs[0]} 等 ${srcs.length} 个文件(夹) ${format_bytes(filesize)} options: ${JSON.stringify(options)}`, undefined, true);
+    let actual_files = [];
+    let dir_num = 0;
+    for (let srcIndex = 0; srcIndex < srcs.length; srcIndex++) {
+        const src = srcs[srcIndex];
+        let show_src = src;
+        try {
+            display_tool.progress(srcIndex + 1, srcs.length, {
+                desc: `Remove ${show_src}`,
+                depth: 0,
+                color: "cyan",
+                align: true,
+            });
+            if (fs.lstatSync(src).isDirectory()) {
+                fs.rmdirSync(src, options);
+                ++dir_num;
+            } else {
+                if (fs.existsSync(src)) {
+                    if (!actual_files.includes(src)) {
+                        actual_files.push(src);
+                    }
+                    fs.unlinkSync(src);
+                }
+            }
+        } catch (err) {
+            warn(err);
+        }
+    }
+    
+    info("");
+    whisper(`删除完成 (文件: ${actual_files.length}, 目录: ${dir_num}, 总计: ${srcs.length})`, undefined, true);
+}
+
+module.exports = {
+    get_file_list,
+    get_fst_file,
+    get_full_path,
+    read_cfg,
+    read_plain,
+    write_plain,
+    write_with_type,
+    copy,
+    move,
+    remove,
+    parse,
+    stringify,
+    move_local,
+    get_file_size,
+    get_files_size,
+    format_bytes,
+    processRename,
+};

package/utils/hash_tool.js

@@ -0,0 +1,8 @@
+const crypto = require("crypto");
+function hash(target, algorithm = "md5", digestEncoding = "hex") {
+  return crypto.createHash(algorithm).update(target).digest(digestEncoding);
+}
+
+module.exports = {
+  hash,
+};