@zwa73/dev-utils 1.0.86 → 1.0.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/template/base/tsconfig.compile.json +1 -1
- package/dist/cjs/Command/MapPath.js +34 -25
- package/dist/cjs/Command/Release.js +3 -1
- package/dist/cjs/Command/ScanDups.js +102 -31
- package/dist/cjs/UtilDevTool.js +5 -4
- package/dist/mjs/Command/MapPath.js +35 -26
- package/dist/mjs/Command/Release.js +3 -1
- package/dist/mjs/Command/ScanDups.js +103 -32
- package/dist/mjs/UtilDevTool.js +6 -5
- package/package.json +3 -1
@@ -7,8 +7,7 @@ exports.CmdMapPath = void 0;
|
|
7
7
|
const utils_1 = require("@zwa73/utils");
|
8
8
|
const fs_1 = __importDefault(require("fs"));
|
9
9
|
const pathe_1 = __importDefault(require("pathe"));
|
10
|
-
const DupMethodList = ["skip", "overwrite"
|
11
|
-
const DupMethodWithoutMove = DupMethodList.filter(t => t != 'move');
|
10
|
+
const DupMethodList = ["skip", "overwrite"];
|
12
11
|
/**重命名文件或路径 */
|
13
12
|
const CmdMapPath = (program) => program
|
14
13
|
.command("Map-Path")
|
@@ -17,12 +16,14 @@ const CmdMapPath = (program) => program
|
|
17
16
|
.argument("<regex>", "要匹配的正则表达式, posix风格路径")
|
18
17
|
.argument("<replacement>", "替换字符串")
|
19
18
|
.option("-e, --exclude <regex>", "排除文件的正则表达式")
|
20
|
-
.option(`-d, --duplicate-handling <${
|
19
|
+
.option(`-d, --duplicate-handling <${DupMethodList.join('|')}|[path:string]>`, `处理重名文件的方式:
|
21
20
|
skip 不进行处理
|
22
21
|
overwrite 覆盖重名
|
23
22
|
其他字符串 将重名部分映射到指定目录下的对应位置, 再次重复将会覆盖`, "skip")
|
24
23
|
.option("-r, --recursive", "是否处理子目录", false)
|
25
24
|
.option("-m, --move", "重命名而不是复制文件", false)
|
25
|
+
.option("-o, --output <path>", "输出到某个绝对路径而非当前目录", '')
|
26
|
+
.option("-i, --input <path>", "扫描某个绝对路径而非当前目录", '')
|
26
27
|
.option("-t, --test", "不对文件进行实际操作, 在控制台输出映射结果", false)
|
27
28
|
.action(async (regexStr, replacement, options) => {
|
28
29
|
const regex = new RegExp(regexStr);
|
@@ -30,42 +31,50 @@ overwrite 覆盖重名
|
|
30
31
|
if (!DupMethodList.includes(options.duplicateHandling))
|
31
32
|
(0, utils_1.throwError)(`${options.duplicateHandling} 不是有效的 duplicate-handling`);
|
32
33
|
const duplicateHandling = options.duplicateHandling;
|
33
|
-
const
|
34
|
+
const absout = options.output.length > 0;
|
35
|
+
const absin = options.input.length > 0;
|
36
|
+
const searchPath = absin ? options.input : process.cwd();
|
37
|
+
const outPath = absout ? options.output : process.cwd();
|
34
38
|
// 遍历当前目录下的所有文件
|
35
|
-
const filePaths = (await utils_1.UtilFT.fileSearchRegex(
|
36
|
-
.map(
|
37
|
-
.filter(
|
39
|
+
const filePaths = (await utils_1.UtilFT.fileSearchRegex(searchPath, regex, { relative: options.recursive }))
|
40
|
+
.map(fp => pathe_1.default.relative(searchPath, fp))
|
41
|
+
.filter(fp => excludeRegex ? (!excludeRegex.test(fp)) : true);
|
38
42
|
//对单个路径映射
|
39
43
|
const mapPath = async (source, target) => {
|
40
|
-
const dir = pathe_1.default.parse(target).dir;
|
41
|
-
await utils_1.UtilFT.ensurePathExists(dir, { dir: true });
|
42
44
|
if (options.test)
|
43
45
|
return utils_1.SLogger.info(`${source} -> ${target}`);
|
46
|
+
const dir = pathe_1.default.parse(target).dir;
|
47
|
+
await utils_1.UtilFT.ensurePathExists(dir, { dir: true });
|
44
48
|
if (options.move)
|
45
|
-
|
46
|
-
|
47
|
-
await fs_1.default.promises.copyFile(source, target);
|
49
|
+
return fs_1.default.promises.rename(source, target);
|
50
|
+
return fs_1.default.promises.copyFile(source, target);
|
48
51
|
};
|
49
|
-
|
52
|
+
await Promise.all(filePaths.map(async (rawfilePath) => {
|
53
|
+
const filePath = pathe_1.default.normalize(rawfilePath);
|
54
|
+
const replacedFilePath = filePath.replace(regex, replacement);
|
50
55
|
// 重命名文件
|
51
|
-
const
|
56
|
+
const oldFilePath = absin
|
57
|
+
? pathe_1.default.join(searchPath, filePath)
|
58
|
+
: filePath;
|
59
|
+
const newFilePath = absout
|
60
|
+
? pathe_1.default.join(outPath, replacedFilePath) //如果是绝对路径输出则拼接绝对路径
|
61
|
+
: replacedFilePath;
|
52
62
|
// 如果文件名发生了变化
|
53
|
-
if (newFilePath ===
|
54
|
-
|
63
|
+
if (newFilePath === oldFilePath)
|
64
|
+
return;
|
55
65
|
//如果文件已存在
|
56
66
|
if (await utils_1.UtilFT.pathExists(newFilePath)) {
|
57
|
-
|
58
|
-
|
59
|
-
|
67
|
+
//如果是跳过或覆盖
|
68
|
+
if (DupMethodList.includes(options.duplicateHandling)) {
|
69
|
+
return (0, utils_1.match)(duplicateHandling, {
|
60
70
|
'skip': () => utils_1.SLogger.info(`重名文件存在,跳过:${newFilePath}`),
|
61
|
-
'overwrite': () => mapPath(
|
71
|
+
'overwrite': () => mapPath(oldFilePath, newFilePath),
|
62
72
|
});
|
63
73
|
}
|
64
|
-
|
65
|
-
|
74
|
+
//如果是转移位置
|
75
|
+
return mapPath(oldFilePath, pathe_1.default.join(outPath, duplicateHandling, replacedFilePath));
|
66
76
|
}
|
67
|
-
|
68
|
-
|
69
|
-
}
|
77
|
+
return mapPath(oldFilePath, newFilePath);
|
78
|
+
}));
|
70
79
|
});
|
71
80
|
exports.CmdMapPath = CmdMapPath;
|
@@ -34,6 +34,8 @@ function checkVersion(oldVersion, newVersion) {
|
|
34
34
|
async function updateVersion(newVersion) {
|
35
35
|
const packagePath = pathe_1.default.join(RouteInterface_1.PROCESS_PATH, "package.json");
|
36
36
|
const packageData = await (0, utils_1.memoize)(utils_1.UtilFT.loadJSONFile)(packagePath);
|
37
|
+
if (newVersion == 'current')
|
38
|
+
return packageData.version;
|
37
39
|
if (newVersion) {
|
38
40
|
checkVersion(packageData.version, newVersion);
|
39
41
|
packageData.version = newVersion;
|
@@ -59,7 +61,7 @@ const CmdRelease = (program) => program
|
|
59
61
|
.command("Release")
|
60
62
|
.alias("release")
|
61
63
|
.description("更新版本号并发布包")
|
62
|
-
.option("-v, --version <version>", "
|
64
|
+
.option("-v, --version <version>", "指定发布的版本号, 为 `current` 时不更新版本号, 格式应为 `${number}.${number}.${number}`")
|
63
65
|
.option("-a, --access <access>", "npm publish 的 access 参数 默认 public", "public")
|
64
66
|
.option("-l, --local <path>", "仅打包到本地对印目录下 如./build/", undefined)
|
65
67
|
.action(async (opt) => {
|
@@ -5,46 +5,117 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
6
|
exports.CmdScanDups = void 0;
|
7
7
|
const utils_1 = require("@zwa73/utils");
|
8
|
-
const fs_1 = __importDefault(require("fs"));
|
9
|
-
const crypto_1 = __importDefault(require("crypto"));
|
10
8
|
const pathe_1 = __importDefault(require("pathe"));
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
9
|
+
const cli_progress_1 = __importDefault(require("cli-progress"));
|
10
|
+
const fs_1 = __importDefault(require("fs"));
|
11
|
+
const { tap } = utils_1.UtilFP;
|
12
|
+
/**hashlist转为dupmap */
|
13
|
+
const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
|
14
|
+
const files = acc[cur.hash] ?? [];
|
15
|
+
acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
|
16
|
+
return acc;
|
17
|
+
}, {});
|
18
|
+
/**从dupmap提取重复的filepath */
|
19
|
+
const reduce2DupFpList = (map) => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
|
20
|
+
[]);
|
21
|
+
/**扫描文件结构 */
|
22
|
+
const scanDirStruct = (0, utils_1.memoize)(async (root) => {
|
23
|
+
const list = await fs_1.default.promises.readdir(root, { withFileTypes: true });
|
24
|
+
const stack = await Promise.all(list.map(async (fp) => fp.isDirectory()
|
25
|
+
? [fp.name, await scanDirStruct(pathe_1.default.join(root, fp.name))]
|
26
|
+
: [fp.name, pathe_1.default.join(root, fp.name)]));
|
27
|
+
return stack.reduce((acc, cur) => ({
|
28
|
+
...acc, [cur[0]]: cur[1],
|
29
|
+
}), {});
|
30
|
+
});
|
31
|
+
const hashQueue = new utils_1.PromiseQueue({ concurrency: 8 });
|
32
|
+
/**计算文件结构的hash */
|
33
|
+
async function struct2hash(struct, fn) {
|
34
|
+
const recursion = (struct) => {
|
35
|
+
return Object.entries(struct).map(async ([k, v]) => {
|
36
|
+
if (typeof v === "string")
|
37
|
+
return [await hashQueue.enqueue(async () => fn(v))];
|
38
|
+
return (await Promise.all(recursion(v))).flat();
|
39
|
+
});
|
40
|
+
};
|
41
|
+
return utils_1.UtilFunc.calcHash((await Promise.all(recursion(struct)))
|
42
|
+
.flat().join('|'), { algorithm: "blake2b512" });
|
19
43
|
}
|
20
44
|
/**重命名文件或路径 scan_duplicates */
|
21
45
|
const CmdScanDups = (program) => program
|
22
46
|
.command("Scan-Dups")
|
23
47
|
.alias("scandups")
|
24
48
|
.description("扫描当前目录下hash重复的文件")
|
25
|
-
.option("-
|
26
|
-
.option("-o, --out <dir|console>", "输出的json文件路径, 默认
|
49
|
+
.option("-i, --include <regex>", "文件的正则表达式, 使用posix路径", ".*")
|
50
|
+
.option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
|
27
51
|
.option("-r, --recursive", "是否处理子目录, 默认 true", true)
|
52
|
+
.option("-s, --struct", `结构模式
|
53
|
+
只扫描 -i 参数目录下的单层目录, 并计算目录是否出现相同的层次结构
|
54
|
+
默认 false`, true)
|
55
|
+
.option("-d, --dir <dirs...>", `扫描的根目录, 以空格分隔 -d "a/b c" "d e/f", 默认为命令行当前目录`, [])
|
28
56
|
.action(async (options) => {
|
29
|
-
const
|
57
|
+
const { out: outtype, recursive, include, struct, dir } = options;
|
58
|
+
const regex = new RegExp(include);
|
30
59
|
const basePath = process.cwd();
|
31
|
-
const
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
}
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
60
|
+
const pathList = dir.length <= 0
|
61
|
+
? [process.cwd()]
|
62
|
+
: dir.map(d => pathe_1.default.isAbsolute(d) ? d : pathe_1.default.join(process.cwd(), d));
|
63
|
+
// 添加一个多步进度条
|
64
|
+
const progressBar = new cli_progress_1.default.MultiBar({
|
65
|
+
clearOnComplete: true, hideCursor: true,
|
66
|
+
format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
|
67
|
+
}, cli_progress_1.default.Presets.shades_classic);
|
68
|
+
// 文件结构进度条
|
69
|
+
const structProgress = struct ? progressBar.create(1, 0, { task: "结构扫描", status: "准备中..." }) : undefined;
|
70
|
+
// 采样哈希进度条
|
71
|
+
const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
|
72
|
+
// 完整哈希进度条
|
73
|
+
const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
|
74
|
+
await (0, utils_1.pipe)(
|
75
|
+
// 第一步:文件搜索,获取符合正则的文件路径
|
76
|
+
Promise.all(pathList.map(async (p) => {
|
77
|
+
if (struct)
|
78
|
+
return (await fs_1.default.promises.readdir(p, { withFileTypes: true }))
|
79
|
+
.filter(fp => fp.isDirectory())
|
80
|
+
.map(fp => pathe_1.default.join(p, fp.name));
|
81
|
+
return await utils_1.UtilFT.fileSearchRegex(p, regex.source, { recursive });
|
82
|
+
})), stacklist => stacklist.flat(), // 扁平化文件路径列表
|
83
|
+
// 如果是结构模式则先筛选相同结构
|
84
|
+
// 扁平化文件路径列表
|
85
|
+
list => struct ? (0, utils_1.pipe)(list, tap(dupFpList => (structProgress.setTotal(dupFpList.length),
|
86
|
+
structProgress.update(0, { status: `结构扫描检出 ${dupFpList.length} 个可能的相等项` }))), list => utils_1.Stream.from(list, 8)
|
87
|
+
.map(async (filePath) => ({
|
88
|
+
filePath,
|
89
|
+
hash: await struct2hash(await scanDirStruct(filePath), cpath => pathe_1.default.relative(filePath, cpath)).then(tap(() => structProgress.increment()))
|
90
|
+
}))
|
91
|
+
.toArray(), reduce2Dupmap, reduce2DupFpList) : list,
|
92
|
+
// 第二步:快速扫描,计算采样哈希
|
93
|
+
tap(list => (sampledProgress.setTotal(list.length),
|
94
|
+
sampledProgress.update(0, { status: `总计 ${list.length} 个文件` }))), list => utils_1.Stream.from(list, 8)
|
95
|
+
.map(async (filePath) => ({
|
96
|
+
filePath,
|
97
|
+
hash: struct
|
98
|
+
? await struct2hash(await scanDirStruct(filePath), (str) => utils_1.UtilFT.calculateHash(str, { sampled: true })).then(tap(() => sampledProgress.increment()))
|
99
|
+
: await utils_1.UtilFT.calculateHash(filePath, { sampled: true }).then(tap(() => sampledProgress.increment())),
|
100
|
+
}))
|
101
|
+
.toArray(),
|
102
|
+
// 第三步:筛选重复的采样哈希 (去掉唯一的采样哈希)
|
103
|
+
reduce2Dupmap, reduce2DupFpList,
|
104
|
+
// 第四步:对筛选出的重复文件路径并发计算完整哈希
|
105
|
+
tap(dupFpList => (fullHashProgress.setTotal(dupFpList.length),
|
106
|
+
fullHashProgress.update(0, { status: `快速扫描检出 ${dupFpList.length} 个可能的相等项` }))), dupFpList => utils_1.Stream.from(dupFpList, 8)
|
107
|
+
.map(async (filePath) => ({
|
108
|
+
filePath,
|
109
|
+
hash: struct
|
110
|
+
? await struct2hash(await scanDirStruct(filePath), (str) => utils_1.UtilFT.calculateHash(str)).then(tap(() => sampledProgress.increment()))
|
111
|
+
: await utils_1.UtilFT.calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
|
112
|
+
}))
|
113
|
+
.toArray(),
|
114
|
+
// 第五步:重新整理完整哈希结果, 过滤唯一哈希
|
115
|
+
reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}),
|
116
|
+
// 第六步:输出结果
|
117
|
+
tap(() => progressBar.stop()), out => (0, utils_1.match)(outtype, {
|
118
|
+
"console": () => utils_1.SLogger.info(out),
|
119
|
+
}, () => utils_1.UtilFT.writeJSONFile(pathe_1.default.join(basePath, outtype), out)));
|
49
120
|
});
|
50
121
|
exports.CmdScanDups = CmdScanDups;
|
package/dist/cjs/UtilDevTool.js
CHANGED
@@ -161,7 +161,8 @@ var UtilDT;
|
|
161
161
|
UtilDT.batchNode = batchNode;
|
162
162
|
//#region macro工具
|
163
163
|
const parseMacroPaths = (opt) => {
|
164
|
-
|
164
|
+
// JsFunc -> ComposeFunc -> xxxmacro -> parseMacroPaths -> sourceTS
|
165
|
+
const loc = utils_1.UtilFunc.getFuncLoc(4);
|
165
166
|
if (!loc && !opt?.filePath)
|
166
167
|
(0, utils_1.throwError)(`parseMacroPaths 未能找到函数位置`);
|
167
168
|
const basePath = loc?.filePath;
|
@@ -232,7 +233,7 @@ var UtilDT;
|
|
232
233
|
else if (!opt?.glob)
|
233
234
|
utils_1.SLogger.error(`UtilDT.regionMacro 无法找到区域 ${regionId}`);
|
234
235
|
};
|
235
|
-
plist.push(utils_1.
|
236
|
+
plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
|
236
237
|
}
|
237
238
|
await Promise.all(plist);
|
238
239
|
}
|
@@ -290,7 +291,7 @@ var UtilDT;
|
|
290
291
|
else if (!opt?.glob)
|
291
292
|
utils_1.SLogger.error(`UtilDT.commentMacro 无法找到注释 ${commentId}`);
|
292
293
|
};
|
293
|
-
plist.push(utils_1.
|
294
|
+
plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
|
294
295
|
}
|
295
296
|
await Promise.all(plist);
|
296
297
|
}
|
@@ -317,7 +318,7 @@ var UtilDT;
|
|
317
318
|
});
|
318
319
|
await fs.promises.writeFile(filePath, parseCode, 'utf-8');
|
319
320
|
};
|
320
|
-
plist.push(utils_1.
|
321
|
+
plist.push(utils_1.PromiseQueue.enqueue(pathe_1.default.normalize(filePath), queuefunc));
|
321
322
|
}
|
322
323
|
await Promise.all(plist);
|
323
324
|
}
|
@@ -1,8 +1,7 @@
|
|
1
|
-
import { SLogger, UtilFT,
|
1
|
+
import { SLogger, UtilFT, match, throwError } from "@zwa73/utils";
|
2
2
|
import fs from "fs";
|
3
3
|
import path from "pathe";
|
4
|
-
const DupMethodList = ["skip", "overwrite"
|
5
|
-
const DupMethodWithoutMove = DupMethodList.filter(t => t != 'move');
|
4
|
+
const DupMethodList = ["skip", "overwrite"];
|
6
5
|
/**重命名文件或路径 */
|
7
6
|
export const CmdMapPath = (program) => program
|
8
7
|
.command("Map-Path")
|
@@ -11,12 +10,14 @@ export const CmdMapPath = (program) => program
|
|
11
10
|
.argument("<regex>", "要匹配的正则表达式, posix风格路径")
|
12
11
|
.argument("<replacement>", "替换字符串")
|
13
12
|
.option("-e, --exclude <regex>", "排除文件的正则表达式")
|
14
|
-
.option(`-d, --duplicate-handling <${
|
13
|
+
.option(`-d, --duplicate-handling <${DupMethodList.join('|')}|[path:string]>`, `处理重名文件的方式:
|
15
14
|
skip 不进行处理
|
16
15
|
overwrite 覆盖重名
|
17
16
|
其他字符串 将重名部分映射到指定目录下的对应位置, 再次重复将会覆盖`, "skip")
|
18
17
|
.option("-r, --recursive", "是否处理子目录", false)
|
19
18
|
.option("-m, --move", "重命名而不是复制文件", false)
|
19
|
+
.option("-o, --output <path>", "输出到某个绝对路径而非当前目录", '')
|
20
|
+
.option("-i, --input <path>", "扫描某个绝对路径而非当前目录", '')
|
20
21
|
.option("-t, --test", "不对文件进行实际操作, 在控制台输出映射结果", false)
|
21
22
|
.action(async (regexStr, replacement, options) => {
|
22
23
|
const regex = new RegExp(regexStr);
|
@@ -24,41 +25,49 @@ overwrite 覆盖重名
|
|
24
25
|
if (!DupMethodList.includes(options.duplicateHandling))
|
25
26
|
throwError(`${options.duplicateHandling} 不是有效的 duplicate-handling`);
|
26
27
|
const duplicateHandling = options.duplicateHandling;
|
27
|
-
const
|
28
|
+
const absout = options.output.length > 0;
|
29
|
+
const absin = options.input.length > 0;
|
30
|
+
const searchPath = absin ? options.input : process.cwd();
|
31
|
+
const outPath = absout ? options.output : process.cwd();
|
28
32
|
// 遍历当前目录下的所有文件
|
29
|
-
const filePaths = (await UtilFT.fileSearchRegex(
|
30
|
-
.map(
|
31
|
-
.filter(
|
33
|
+
const filePaths = (await UtilFT.fileSearchRegex(searchPath, regex, { relative: options.recursive }))
|
34
|
+
.map(fp => path.relative(searchPath, fp))
|
35
|
+
.filter(fp => excludeRegex ? (!excludeRegex.test(fp)) : true);
|
32
36
|
//对单个路径映射
|
33
37
|
const mapPath = async (source, target) => {
|
34
|
-
const dir = path.parse(target).dir;
|
35
|
-
await UtilFT.ensurePathExists(dir, { dir: true });
|
36
38
|
if (options.test)
|
37
39
|
return SLogger.info(`${source} -> ${target}`);
|
40
|
+
const dir = path.parse(target).dir;
|
41
|
+
await UtilFT.ensurePathExists(dir, { dir: true });
|
38
42
|
if (options.move)
|
39
|
-
|
40
|
-
|
41
|
-
await fs.promises.copyFile(source, target);
|
43
|
+
return fs.promises.rename(source, target);
|
44
|
+
return fs.promises.copyFile(source, target);
|
42
45
|
};
|
43
|
-
|
46
|
+
await Promise.all(filePaths.map(async (rawfilePath) => {
|
47
|
+
const filePath = path.normalize(rawfilePath);
|
48
|
+
const replacedFilePath = filePath.replace(regex, replacement);
|
44
49
|
// 重命名文件
|
45
|
-
const
|
50
|
+
const oldFilePath = absin
|
51
|
+
? path.join(searchPath, filePath)
|
52
|
+
: filePath;
|
53
|
+
const newFilePath = absout
|
54
|
+
? path.join(outPath, replacedFilePath) //如果是绝对路径输出则拼接绝对路径
|
55
|
+
: replacedFilePath;
|
46
56
|
// 如果文件名发生了变化
|
47
|
-
if (newFilePath ===
|
48
|
-
|
57
|
+
if (newFilePath === oldFilePath)
|
58
|
+
return;
|
49
59
|
//如果文件已存在
|
50
60
|
if (await UtilFT.pathExists(newFilePath)) {
|
51
|
-
|
52
|
-
|
53
|
-
|
61
|
+
//如果是跳过或覆盖
|
62
|
+
if (DupMethodList.includes(options.duplicateHandling)) {
|
63
|
+
return match(duplicateHandling, {
|
54
64
|
'skip': () => SLogger.info(`重名文件存在,跳过:${newFilePath}`),
|
55
|
-
'overwrite': () => mapPath(
|
65
|
+
'overwrite': () => mapPath(oldFilePath, newFilePath),
|
56
66
|
});
|
57
67
|
}
|
58
|
-
|
59
|
-
|
68
|
+
//如果是转移位置
|
69
|
+
return mapPath(oldFilePath, path.join(outPath, duplicateHandling, replacedFilePath));
|
60
70
|
}
|
61
|
-
|
62
|
-
|
63
|
-
}
|
71
|
+
return mapPath(oldFilePath, newFilePath);
|
72
|
+
}));
|
64
73
|
});
|
@@ -28,6 +28,8 @@ function checkVersion(oldVersion, newVersion) {
|
|
28
28
|
async function updateVersion(newVersion) {
|
29
29
|
const packagePath = path.join(PROCESS_PATH, "package.json");
|
30
30
|
const packageData = await memoize(UtilFT.loadJSONFile)(packagePath);
|
31
|
+
if (newVersion == 'current')
|
32
|
+
return packageData.version;
|
31
33
|
if (newVersion) {
|
32
34
|
checkVersion(packageData.version, newVersion);
|
33
35
|
packageData.version = newVersion;
|
@@ -53,7 +55,7 @@ export const CmdRelease = (program) => program
|
|
53
55
|
.command("Release")
|
54
56
|
.alias("release")
|
55
57
|
.description("更新版本号并发布包")
|
56
|
-
.option("-v, --version <version>", "
|
58
|
+
.option("-v, --version <version>", "指定发布的版本号, 为 `current` 时不更新版本号, 格式应为 `${number}.${number}.${number}`")
|
57
59
|
.option("-a, --access <access>", "npm publish 的 access 参数 默认 public", "public")
|
58
60
|
.option("-l, --local <path>", "仅打包到本地对印目录下 如./build/", undefined)
|
59
61
|
.action(async (opt) => {
|
@@ -1,43 +1,114 @@
|
|
1
|
-
import { SLogger, UtilFT } from "@zwa73/utils";
|
2
|
-
import fs from "fs";
|
3
|
-
import crypto from 'crypto';
|
1
|
+
import { match, memoize, pipe, PromiseQueue, SLogger, Stream, UtilFP, UtilFT, UtilFunc } from "@zwa73/utils";
|
4
2
|
import path from "pathe";
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
import cliProgress from "cli-progress";
|
4
|
+
import fs from 'fs';
|
5
|
+
const { tap } = UtilFP;
|
6
|
+
/**hashlist转为dupmap */
|
7
|
+
const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
|
8
|
+
const files = acc[cur.hash] ?? [];
|
9
|
+
acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
|
10
|
+
return acc;
|
11
|
+
}, {});
|
12
|
+
/**从dupmap提取重复的filepath */
|
13
|
+
const reduce2DupFpList = (map) => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
|
14
|
+
[]);
|
15
|
+
/**扫描文件结构 */
|
16
|
+
const scanDirStruct = memoize(async (root) => {
|
17
|
+
const list = await fs.promises.readdir(root, { withFileTypes: true });
|
18
|
+
const stack = await Promise.all(list.map(async (fp) => fp.isDirectory()
|
19
|
+
? [fp.name, await scanDirStruct(path.join(root, fp.name))]
|
20
|
+
: [fp.name, path.join(root, fp.name)]));
|
21
|
+
return stack.reduce((acc, cur) => ({
|
22
|
+
...acc, [cur[0]]: cur[1],
|
23
|
+
}), {});
|
24
|
+
});
|
25
|
+
const hashQueue = new PromiseQueue({ concurrency: 8 });
|
26
|
+
/**计算文件结构的hash */
|
27
|
+
async function struct2hash(struct, fn) {
|
28
|
+
const recursion = (struct) => {
|
29
|
+
return Object.entries(struct).map(async ([k, v]) => {
|
30
|
+
if (typeof v === "string")
|
31
|
+
return [await hashQueue.enqueue(async () => fn(v))];
|
32
|
+
return (await Promise.all(recursion(v))).flat();
|
33
|
+
});
|
34
|
+
};
|
35
|
+
return UtilFunc.calcHash((await Promise.all(recursion(struct)))
|
36
|
+
.flat().join('|'), { algorithm: "blake2b512" });
|
13
37
|
}
|
14
38
|
/**重命名文件或路径 scan_duplicates */
|
15
39
|
export const CmdScanDups = (program) => program
|
16
40
|
.command("Scan-Dups")
|
17
41
|
.alias("scandups")
|
18
42
|
.description("扫描当前目录下hash重复的文件")
|
19
|
-
.option("-
|
20
|
-
.option("-o, --out <dir|console>", "输出的json文件路径, 默认
|
43
|
+
.option("-i, --include <regex>", "文件的正则表达式, 使用posix路径", ".*")
|
44
|
+
.option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
|
21
45
|
.option("-r, --recursive", "是否处理子目录, 默认 true", true)
|
46
|
+
.option("-s, --struct", `结构模式
|
47
|
+
只扫描 -i 参数目录下的单层目录, 并计算目录是否出现相同的层次结构
|
48
|
+
默认 false`, true)
|
49
|
+
.option("-d, --dir <dirs...>", `扫描的根目录, 以空格分隔 -d "a/b c" "d e/f", 默认为命令行当前目录`, [])
|
22
50
|
.action(async (options) => {
|
23
|
-
const
|
51
|
+
const { out: outtype, recursive, include, struct, dir } = options;
|
52
|
+
const regex = new RegExp(include);
|
24
53
|
const basePath = process.cwd();
|
25
|
-
const
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
}
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
54
|
+
const pathList = dir.length <= 0
|
55
|
+
? [process.cwd()]
|
56
|
+
: dir.map(d => path.isAbsolute(d) ? d : path.join(process.cwd(), d));
|
57
|
+
// 添加一个多步进度条
|
58
|
+
const progressBar = new cliProgress.MultiBar({
|
59
|
+
clearOnComplete: true, hideCursor: true,
|
60
|
+
format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
|
61
|
+
}, cliProgress.Presets.shades_classic);
|
62
|
+
// 文件结构进度条
|
63
|
+
const structProgress = struct ? progressBar.create(1, 0, { task: "结构扫描", status: "准备中..." }) : undefined;
|
64
|
+
// 采样哈希进度条
|
65
|
+
const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
|
66
|
+
// 完整哈希进度条
|
67
|
+
const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
|
68
|
+
await pipe(
|
69
|
+
// 第一步:文件搜索,获取符合正则的文件路径
|
70
|
+
Promise.all(pathList.map(async (p) => {
|
71
|
+
if (struct)
|
72
|
+
return (await fs.promises.readdir(p, { withFileTypes: true }))
|
73
|
+
.filter(fp => fp.isDirectory())
|
74
|
+
.map(fp => path.join(p, fp.name));
|
75
|
+
return await UtilFT.fileSearchRegex(p, regex.source, { recursive });
|
76
|
+
})), stacklist => stacklist.flat(), // 扁平化文件路径列表
|
77
|
+
// 如果是结构模式则先筛选相同结构
|
78
|
+
// 扁平化文件路径列表
|
79
|
+
list => struct ? pipe(list, tap(dupFpList => (structProgress.setTotal(dupFpList.length),
|
80
|
+
structProgress.update(0, { status: `结构扫描检出 ${dupFpList.length} 个可能的相等项` }))), list => Stream.from(list, 8)
|
81
|
+
.map(async (filePath) => ({
|
82
|
+
filePath,
|
83
|
+
hash: await struct2hash(await scanDirStruct(filePath), cpath => path.relative(filePath, cpath)).then(tap(() => structProgress.increment()))
|
84
|
+
}))
|
85
|
+
.toArray(), reduce2Dupmap, reduce2DupFpList) : list,
|
86
|
+
// 第二步:快速扫描,计算采样哈希
|
87
|
+
tap(list => (sampledProgress.setTotal(list.length),
|
88
|
+
sampledProgress.update(0, { status: `总计 ${list.length} 个文件` }))), list => Stream.from(list, 8)
|
89
|
+
.map(async (filePath) => ({
|
90
|
+
filePath,
|
91
|
+
hash: struct
|
92
|
+
? await struct2hash(await scanDirStruct(filePath), (str) => UtilFT.calculateHash(str, { sampled: true })).then(tap(() => sampledProgress.increment()))
|
93
|
+
: await UtilFT.calculateHash(filePath, { sampled: true }).then(tap(() => sampledProgress.increment())),
|
94
|
+
}))
|
95
|
+
.toArray(),
|
96
|
+
// 第三步:筛选重复的采样哈希 (去掉唯一的采样哈希)
|
97
|
+
reduce2Dupmap, reduce2DupFpList,
|
98
|
+
// 第四步:对筛选出的重复文件路径并发计算完整哈希
|
99
|
+
tap(dupFpList => (fullHashProgress.setTotal(dupFpList.length),
|
100
|
+
fullHashProgress.update(0, { status: `快速扫描检出 ${dupFpList.length} 个可能的相等项` }))), dupFpList => Stream.from(dupFpList, 8)
|
101
|
+
.map(async (filePath) => ({
|
102
|
+
filePath,
|
103
|
+
hash: struct
|
104
|
+
? await struct2hash(await scanDirStruct(filePath), (str) => UtilFT.calculateHash(str)).then(tap(() => sampledProgress.increment()))
|
105
|
+
: await UtilFT.calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
|
106
|
+
}))
|
107
|
+
.toArray(),
|
108
|
+
// 第五步:重新整理完整哈希结果, 过滤唯一哈希
|
109
|
+
reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}),
|
110
|
+
// 第六步:输出结果
|
111
|
+
tap(() => progressBar.stop()), out => match(outtype, {
|
112
|
+
"console": () => SLogger.info(out),
|
113
|
+
}, () => UtilFT.writeJSONFile(path.join(basePath, outtype), out)));
|
43
114
|
});
|
package/dist/mjs/UtilDevTool.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import path from 'pathe';
|
2
2
|
import * as TJS from 'typescript-json-schema';
|
3
3
|
import * as fs from 'fs';
|
4
|
-
import { SLogger, UtilFT, UtilFunc, dedent, throwError } from '@zwa73/utils';
|
4
|
+
import { PromiseQueue, SLogger, UtilFT, UtilFunc, dedent, throwError } from '@zwa73/utils';
|
5
5
|
import { Project, SyntaxKind } from 'ts-morph';
|
6
6
|
export var UtilDT;
|
7
7
|
(function (UtilDT) {
|
@@ -132,7 +132,8 @@ export var UtilDT;
|
|
132
132
|
UtilDT.batchNode = batchNode;
|
133
133
|
//#region macro工具
|
134
134
|
const parseMacroPaths = (opt) => {
|
135
|
-
|
135
|
+
// JsFunc -> ComposeFunc -> xxxmacro -> parseMacroPaths -> sourceTS
|
136
|
+
const loc = UtilFunc.getFuncLoc(4);
|
136
137
|
if (!loc && !opt?.filePath)
|
137
138
|
throwError(`parseMacroPaths 未能找到函数位置`);
|
138
139
|
const basePath = loc?.filePath;
|
@@ -203,7 +204,7 @@ export var UtilDT;
|
|
203
204
|
else if (!opt?.glob)
|
204
205
|
SLogger.error(`UtilDT.regionMacro 无法找到区域 ${regionId}`);
|
205
206
|
};
|
206
|
-
plist.push(
|
207
|
+
plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
|
207
208
|
}
|
208
209
|
await Promise.all(plist);
|
209
210
|
}
|
@@ -261,7 +262,7 @@ export var UtilDT;
|
|
261
262
|
else if (!opt?.glob)
|
262
263
|
SLogger.error(`UtilDT.commentMacro 无法找到注释 ${commentId}`);
|
263
264
|
};
|
264
|
-
plist.push(
|
265
|
+
plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
|
265
266
|
}
|
266
267
|
await Promise.all(plist);
|
267
268
|
}
|
@@ -288,7 +289,7 @@ export var UtilDT;
|
|
288
289
|
});
|
289
290
|
await fs.promises.writeFile(filePath, parseCode, 'utf-8');
|
290
291
|
};
|
291
|
-
plist.push(
|
292
|
+
plist.push(PromiseQueue.enqueue(path.normalize(filePath), queuefunc));
|
292
293
|
}
|
293
294
|
await Promise.all(plist);
|
294
295
|
}
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@zwa73/dev-utils",
|
3
|
-
"version": "1.0.
|
3
|
+
"version": "1.0.88",
|
4
4
|
"description": "编译与调试工具",
|
5
5
|
"exports": {
|
6
6
|
".": {
|
@@ -24,6 +24,7 @@
|
|
24
24
|
"dependencies": {
|
25
25
|
"@deepkit/type-compiler": "^1.0.1-alpha.150",
|
26
26
|
"@zwa73/utils": "*",
|
27
|
+
"cli-progress": "^3.12.0",
|
27
28
|
"commander": "^11.1.0",
|
28
29
|
"pathe": "^1.1.2",
|
29
30
|
"ts-morph": "^23.0.0",
|
@@ -32,6 +33,7 @@
|
|
32
33
|
"typescript-json-schema": "^0.64.0"
|
33
34
|
},
|
34
35
|
"devDependencies": {
|
36
|
+
"@types/cli-progress": "^3.11.6",
|
35
37
|
"@types/jest": "^29.5.12",
|
36
38
|
"@types/node": "^20.14.11",
|
37
39
|
"jest": "^29.7.0",
|