@zwa73/dev-utils 1.0.86 → 1.0.87

package/data/template/base/tsconfig.compile.json

@@ -1,5 +1,5 @@
 {
     "extends": "./tsconfig.json",
     "include": ["./src/**/*.ts", "./src/**/*.js"],
-    "exclude": ["./node_modules/**/*","./src/**/*.macro.ts"]
+    "exclude": ["./node_modules/**/*","./src/**/*.macro.ts","./src/**/*.schema.ts"]
 }

package/dist/cjs/Command/MapPath.js

@@ -56,7 +56,7 @@ overwrite 覆盖重名
         if (await utils_1.UtilFT.pathExists(newFilePath)) {
             if (DupMethodWithoutMove.includes(options.duplicateHandling)) {
                 const fixhd = duplicateHandling;
-                await (0, utils_1.matchProc)(fixhd, {
+                await (0, utils_1.match)(fixhd, {
                     'skip': () => utils_1.SLogger.info(`重名文件存在，跳过：${newFilePath}`),
                     'overwrite': () => mapPath(filePath, newFilePath),
                 });

package/dist/cjs/Command/ScanDups.js

@@ -8,6 +8,8 @@ const utils_1 = require("@zwa73/utils");
 const fs_1 = __importDefault(require("fs"));
 const crypto_1 = __importDefault(require("crypto"));
 const pathe_1 = __importDefault(require("pathe"));
+const cli_progress_1 = __importDefault(require("cli-progress"));
+const { tap } = utils_1.UtilFP;
 async function calculateHash(filePath) {
     return new Promise((resolve, reject) => {
         const hash = crypto_1.default.createHash('md5');
@@ -17,34 +19,83 @@ async function calculateHash(filePath) {
         stream.on('error', reject);
     });
 }
+async function calculateSampledHash(filePath, chunkSize = 1024, // 每块大小（例如 1KB）
+chunkCount = 10) {
+    const stats = await fs_1.default.promises.stat(filePath);
+    const totalSize = stats.size;
+    if (totalSize < chunkSize * chunkCount)
+        return calculateHash(filePath);
+    const positions = Array.from({ length: chunkCount }, (_, i) => Math.floor((totalSize / chunkCount) * i));
+    const hash = crypto_1.default.createHash("md5"); // 或其他哈希算法，如 CRC32/BLAKE3
+    for (const position of positions) {
+        const buffer = await new Promise((resolve, reject) => {
+            // 创建一个文件流，限制读取范围
+            const stream = fs_1.default.createReadStream(filePath, {
+                start: position,
+                end: Math.min(position + chunkSize - 1, totalSize - 1), // 确保不会超出文件末尾
+                highWaterMark: chunkSize, // 高效流式块
+            });
+            const chunks = [];
+            stream.on("data", (chunk) => chunks.push(chunk));
+            stream.on("end", () => resolve(Buffer.concat(chunks))); // 合并块数据
+            stream.on("error", reject);
+        });
+        hash.update(buffer); // 更新哈希计算
+    }
+    return hash.digest("hex"); // 返回最终哈希值
+}
 /**重命名文件或路径 scan_duplicates */
 const CmdScanDups = (program) => program
     .command("Scan-Dups")
     .alias("scandups")
     .description("扫描当前目录下hash重复的文件")
     .option("-re, --regex <regex>", "文件的正则表达式, 使用posix路径", ".*")
-    .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandupsOut.json, 为 \"console\" 时无文件输出", "scandupsOut")
+    .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
     .option("-r, --recursive", "是否处理子目录, 默认 true", true)
     .action(async (options) => {
     const regex = new RegExp(options.regex);
     const basePath = process.cwd();
-    const pList = (await utils_1.UtilFT.fileSearchRegex(basePath, regex.source, {
-        relative: options.recursive,
-    })).map(async (filePath) => ({ filePath, hash: await calculateHash(filePath) }));
-    const hashMap = (await Promise.all(pList)).reduce((obj, cur) => {
-        obj[cur.hash] = (obj[cur.hash] ?? []).concat(cur.filePath);
-        return obj;
+    // **添加一个多步进度条**
+    const progressBar = new cli_progress_1.default.MultiBar({
+        clearOnComplete: true, hideCursor: true,
+        format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
+    }, cli_progress_1.default.Presets.shades_classic);
+    // 采样哈希进度条
+    const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
+    // 完整哈希进度条
+    const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
+    const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
+        const files = acc[cur.hash] ?? [];
+        acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
+        return acc;
     }, {});
-    const out = {};
-    for (const hash in hashMap) {
-        const duplicateFiles = hashMap[hash];
-        if (duplicateFiles.length <= 1)
-            continue;
-        out[hash] = duplicateFiles;
-    }
-    if (options.out === "console")
-        utils_1.SLogger.info(out);
-    else
-        await utils_1.UtilFT.writeJSONFile(pathe_1.default.join(basePath, options.out), out);
+    await (0, utils_1.pipe)(
+    // 第一步：文件搜索，获取符合正则的文件路径
+    utils_1.UtilFT.fileSearchRegex(basePath, regex.source, { recursive: options.recursive }), tap(list => void sampledProgress.setTotal(list.length) ??
+        void sampledProgress.update(0, { status: `总计 ${list.length} 个文件` })), 
+    // 第二步：快速扫描，计算采样哈希
+    list => utils_1.Stream.from(list, 8)
+        .map(async (filePath) => ({
+        filePath,
+        hash: await calculateSampledHash(filePath).then(tap(() => sampledProgress.increment())),
+    }))
+        .toArray(), 
+    // 第三步：筛选重复的采样哈希（去掉唯一的采样哈希）
+    reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
+    []), tap(dupPaths => void fullHashProgress.setTotal(dupPaths.length) ??
+        void fullHashProgress.update(0, { status: `快速扫描检出 ${dupPaths.length} 个可能的相等项` })), 
+    // 第四步：对筛选出的重复文件路径并发计算完整哈希
+    dups => utils_1.Stream.from(dups, 8)
+        .map(async (filePath) => ({
+        filePath,
+        hash: await calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
+    }))
+        .toArray(), 
+    // 第五步：重新整理完整哈希结果，过滤唯一哈希
+    reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}), 
+    // 第六步：输出结果
+    tap(() => progressBar.stop()), out => (0, utils_1.match)(options.out, {
+        "console": () => utils_1.SLogger.info(out),
+    }, () => utils_1.UtilFT.writeJSONFile(pathe_1.default.join(basePath, options.out), out)));
 });
 exports.CmdScanDups = CmdScanDups;

package/dist/cjs/UtilDevTool.js

@@ -161,7 +161,8 @@ var UtilDT;
     UtilDT.batchNode = batchNode;
     //#region macro工具
     const parseMacroPaths = (opt) => {
-        const loc = utils_1.UtilFunc.getFuncLoc(3);
+        // JsFunc -> ComposeFunc -> xxxmacro -> parseMacroPaths -> sourceTS
+        const loc = utils_1.UtilFunc.getFuncLoc(4);
         if (!loc && !opt?.filePath)
             (0, utils_1.throwError)(`parseMacroPaths 未能找到函数位置`);
         const basePath = loc?.filePath;
@@ -232,7 +233,7 @@ var UtilDT;
                 else if (!opt?.glob)
                     utils_1.SLogger.error(`UtilDT.regionMacro 无法找到区域 ${regionId}`);
             };
-            plist.push(utils_1.UtilFunc.queueProc(pathe_1.default.normalize(filePath), queuefunc));
+            plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }
@@ -290,7 +291,7 @@ var UtilDT;
                 else if (!opt?.glob)
                     utils_1.SLogger.error(`UtilDT.commentMacro 无法找到注释 ${commentId}`);
             };
-            plist.push(utils_1.UtilFunc.queueProc(pathe_1.default.normalize(filePath), queuefunc));
+            plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }
@@ -317,7 +318,7 @@ var UtilDT;
                 });
                 await fs.promises.writeFile(filePath, parseCode, 'utf-8');
             };
-            plist.push(utils_1.UtilFunc.queueProc(pathe_1.default.normalize(filePath), queuefunc));
+            plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }

package/dist/mjs/Command/MapPath.js

@@ -1,4 +1,4 @@
-import { SLogger, UtilFT, matchProc, throwError } from "@zwa73/utils";
+import { SLogger, UtilFT, match, throwError } from "@zwa73/utils";
 import fs from "fs";
 import path from "pathe";
 const DupMethodList = ["skip", "overwrite", "move"];
@@ -50,7 +50,7 @@ overwrite 覆盖重名
         if (await UtilFT.pathExists(newFilePath)) {
             if (DupMethodWithoutMove.includes(options.duplicateHandling)) {
                 const fixhd = duplicateHandling;
-                await matchProc(fixhd, {
+                await match(fixhd, {
                     'skip': () => SLogger.info(`重名文件存在，跳过：${newFilePath}`),
                     'overwrite': () => mapPath(filePath, newFilePath),
                 });

package/dist/mjs/Command/ScanDups.js

@@ -1,7 +1,9 @@
-import { SLogger, UtilFT } from "@zwa73/utils";
+import { match, pipe, SLogger, Stream, UtilFP, UtilFT } from "@zwa73/utils";
 import fs from "fs";
 import crypto from 'crypto';
 import path from "pathe";
+import cliProgress from "cli-progress";
+const { tap } = UtilFP;
 async function calculateHash(filePath) {
     return new Promise((resolve, reject) => {
         const hash = crypto.createHash('md5');
@@ -11,33 +13,82 @@ async function calculateHash(filePath) {
         stream.on('error', reject);
     });
 }
+async function calculateSampledHash(filePath, chunkSize = 1024, // 每块大小（例如 1KB）
+chunkCount = 10) {
+    const stats = await fs.promises.stat(filePath);
+    const totalSize = stats.size;
+    if (totalSize < chunkSize * chunkCount)
+        return calculateHash(filePath);
+    const positions = Array.from({ length: chunkCount }, (_, i) => Math.floor((totalSize / chunkCount) * i));
+    const hash = crypto.createHash("md5"); // 或其他哈希算法，如 CRC32/BLAKE3
+    for (const position of positions) {
+        const buffer = await new Promise((resolve, reject) => {
+            // 创建一个文件流，限制读取范围
+            const stream = fs.createReadStream(filePath, {
+                start: position,
+                end: Math.min(position + chunkSize - 1, totalSize - 1), // 确保不会超出文件末尾
+                highWaterMark: chunkSize, // 高效流式块
+            });
+            const chunks = [];
+            stream.on("data", (chunk) => chunks.push(chunk));
+            stream.on("end", () => resolve(Buffer.concat(chunks))); // 合并块数据
+            stream.on("error", reject);
+        });
+        hash.update(buffer); // 更新哈希计算
+    }
+    return hash.digest("hex"); // 返回最终哈希值
+}
 /**重命名文件或路径 scan_duplicates */
 export const CmdScanDups = (program) => program
     .command("Scan-Dups")
     .alias("scandups")
     .description("扫描当前目录下hash重复的文件")
     .option("-re, --regex <regex>", "文件的正则表达式, 使用posix路径", ".*")
-    .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandupsOut.json, 为 \"console\" 时无文件输出", "scandupsOut")
+    .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
     .option("-r, --recursive", "是否处理子目录, 默认 true", true)
     .action(async (options) => {
     const regex = new RegExp(options.regex);
     const basePath = process.cwd();
-    const pList = (await UtilFT.fileSearchRegex(basePath, regex.source, {
-        relative: options.recursive,
-    })).map(async (filePath) => ({ filePath, hash: await calculateHash(filePath) }));
-    const hashMap = (await Promise.all(pList)).reduce((obj, cur) => {
-        obj[cur.hash] = (obj[cur.hash] ?? []).concat(cur.filePath);
-        return obj;
+    // **添加一个多步进度条**
+    const progressBar = new cliProgress.MultiBar({
+        clearOnComplete: true, hideCursor: true,
+        format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
+    }, cliProgress.Presets.shades_classic);
+    // 采样哈希进度条
+    const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
+    // 完整哈希进度条
+    const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
+    const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
+        const files = acc[cur.hash] ?? [];
+        acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
+        return acc;
     }, {});
-    const out = {};
-    for (const hash in hashMap) {
-        const duplicateFiles = hashMap[hash];
-        if (duplicateFiles.length <= 1)
-            continue;
-        out[hash] = duplicateFiles;
-    }
-    if (options.out === "console")
-        SLogger.info(out);
-    else
-        await UtilFT.writeJSONFile(path.join(basePath, options.out), out);
+    await pipe(
+    // 第一步：文件搜索，获取符合正则的文件路径
+    UtilFT.fileSearchRegex(basePath, regex.source, { recursive: options.recursive }), tap(list => void sampledProgress.setTotal(list.length) ??
+        void sampledProgress.update(0, { status: `总计 ${list.length} 个文件` })), 
+    // 第二步：快速扫描，计算采样哈希
+    list => Stream.from(list, 8)
+        .map(async (filePath) => ({
+        filePath,
+        hash: await calculateSampledHash(filePath).then(tap(() => sampledProgress.increment())),
+    }))
+        .toArray(), 
+    // 第三步：筛选重复的采样哈希（去掉唯一的采样哈希）
+    reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
+    []), tap(dupPaths => void fullHashProgress.setTotal(dupPaths.length) ??
+        void fullHashProgress.update(0, { status: `快速扫描检出 ${dupPaths.length} 个可能的相等项` })), 
+    // 第四步：对筛选出的重复文件路径并发计算完整哈希
+    dups => Stream.from(dups, 8)
+        .map(async (filePath) => ({
+        filePath,
+        hash: await calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
+    }))
+        .toArray(), 
+    // 第五步：重新整理完整哈希结果，过滤唯一哈希
+    reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}), 
+    // 第六步：输出结果
+    tap(() => progressBar.stop()), out => match(options.out, {
+        "console": () => SLogger.info(out),
+    }, () => UtilFT.writeJSONFile(path.join(basePath, options.out), out)));
 });

package/dist/mjs/UtilDevTool.js

@@ -1,7 +1,7 @@
 import path from 'pathe';
 import * as TJS from 'typescript-json-schema';
 import * as fs from 'fs';
-import { SLogger, UtilFT, UtilFunc, dedent, throwError } from '@zwa73/utils';
+import { PromiseQueue, SLogger, UtilFT, UtilFunc, dedent, throwError } from '@zwa73/utils';
 import { Project, SyntaxKind } from 'ts-morph';
 export var UtilDT;
 (function (UtilDT) {
@@ -132,7 +132,8 @@ export var UtilDT;
     UtilDT.batchNode = batchNode;
     //#region macro工具
     const parseMacroPaths = (opt) => {
-        const loc = UtilFunc.getFuncLoc(3);
+        // JsFunc -> ComposeFunc -> xxxmacro -> parseMacroPaths -> sourceTS
+        const loc = UtilFunc.getFuncLoc(4);
         if (!loc && !opt?.filePath)
             throwError(`parseMacroPaths 未能找到函数位置`);
         const basePath = loc?.filePath;
@@ -203,7 +204,7 @@ export var UtilDT;
                 else if (!opt?.glob)
                     SLogger.error(`UtilDT.regionMacro 无法找到区域 ${regionId}`);
             };
-            plist.push(UtilFunc.queueProc(path.normalize(filePath), queuefunc));
+            plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }
@@ -261,7 +262,7 @@ export var UtilDT;
                 else if (!opt?.glob)
                     SLogger.error(`UtilDT.commentMacro 无法找到注释 ${commentId}`);
             };
-            plist.push(UtilFunc.queueProc(path.normalize(filePath), queuefunc));
+            plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }
@@ -288,7 +289,7 @@ export var UtilDT;
                 });
                 await fs.promises.writeFile(filePath, parseCode, 'utf-8');
             };
-            plist.push(UtilFunc.queueProc(path.normalize(filePath), queuefunc));
+            plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
         }
         await Promise.all(plist);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@zwa73/dev-utils",
-	"version": "1.0.86",
+	"version": "1.0.87",
 	"description": "编译与调试工具",
 	"exports": {
 		".": {
@@ -24,6 +24,7 @@
 	"dependencies": {
 		"@deepkit/type-compiler": "^1.0.1-alpha.150",
 		"@zwa73/utils": "*",
+		"cli-progress": "^3.12.0",
 		"commander": "^11.1.0",
 		"pathe": "^1.1.2",
 		"ts-morph": "^23.0.0",
@@ -32,6 +33,7 @@
 		"typescript-json-schema": "^0.64.0"
 	},
 	"devDependencies": {
+		"@types/cli-progress": "^3.11.6",
 		"@types/jest": "^29.5.12",
 		"@types/node": "^20.14.11",
 		"jest": "^29.7.0",