@zwa73/dev-utils 1.0.87 → 1.0.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,8 +7,7 @@ exports.CmdMapPath = void 0;
7
7
  const utils_1 = require("@zwa73/utils");
8
8
  const fs_1 = __importDefault(require("fs"));
9
9
  const pathe_1 = __importDefault(require("pathe"));
10
- const DupMethodList = ["skip", "overwrite", "move"];
11
- const DupMethodWithoutMove = DupMethodList.filter(t => t != 'move');
10
+ const DupMethodList = ["skip", "overwrite"];
12
11
  /**重命名文件或路径 */
13
12
  const CmdMapPath = (program) => program
14
13
  .command("Map-Path")
@@ -17,12 +16,14 @@ const CmdMapPath = (program) => program
17
16
  .argument("<regex>", "要匹配的正则表达式, posix风格路径")
18
17
  .argument("<replacement>", "替换字符串")
19
18
  .option("-e, --exclude <regex>", "排除文件的正则表达式")
20
- .option(`-d, --duplicate-handling <${DupMethodWithoutMove.join('|')}|[path:string]>`, `处理重名文件的方式:
19
+ .option(`-d, --duplicate-handling <${DupMethodList.join('|')}|[path:string]>`, `处理重名文件的方式:
21
20
  skip 不进行处理
22
21
  overwrite 覆盖重名
23
22
  其他字符串 将重名部分映射到指定目录下的对应位置, 再次重复将会覆盖`, "skip")
24
23
  .option("-r, --recursive", "是否处理子目录", false)
25
24
  .option("-m, --move", "重命名而不是复制文件", false)
25
+ .option("-o, --output <path>", "输出到某个绝对路径而非当前目录", '')
26
+ .option("-i, --input <path>", "扫描某个绝对路径而非当前目录", '')
26
27
  .option("-t, --test", "不对文件进行实际操作, 在控制台输出映射结果", false)
27
28
  .action(async (regexStr, replacement, options) => {
28
29
  const regex = new RegExp(regexStr);
@@ -30,42 +31,50 @@ overwrite 覆盖重名
30
31
  if (!DupMethodList.includes(options.duplicateHandling))
31
32
  (0, utils_1.throwError)(`${options.duplicateHandling} 不是有效的 duplicate-handling`);
32
33
  const duplicateHandling = options.duplicateHandling;
33
- const basePath = process.cwd();
34
+ const absout = options.output.length > 0;
35
+ const absin = options.input.length > 0;
36
+ const searchPath = absin ? options.input : process.cwd();
37
+ const outPath = absout ? options.output : process.cwd();
34
38
  // 遍历当前目录下的所有文件
35
- const filePaths = (await utils_1.UtilFT.fileSearchRegex(basePath, regex.source, { relative: options.recursive }))
36
- .map((filePath) => pathe_1.default.relative(basePath, filePath))
37
- .filter((filePath) => excludeRegex ? (!excludeRegex.test(filePath)) : true);
39
+ const filePaths = (await utils_1.UtilFT.fileSearchRegex(searchPath, regex, { relative: options.recursive }))
40
+ .map(fp => pathe_1.default.relative(searchPath, fp))
41
+ .filter(fp => excludeRegex ? (!excludeRegex.test(fp)) : true);
38
42
  //对单个路径映射
39
43
  const mapPath = async (source, target) => {
40
- const dir = pathe_1.default.parse(target).dir;
41
- await utils_1.UtilFT.ensurePathExists(dir, { dir: true });
42
44
  if (options.test)
43
45
  return utils_1.SLogger.info(`${source} -> ${target}`);
46
+ const dir = pathe_1.default.parse(target).dir;
47
+ await utils_1.UtilFT.ensurePathExists(dir, { dir: true });
44
48
  if (options.move)
45
- await fs_1.default.promises.rename(source, target);
46
- else
47
- await fs_1.default.promises.copyFile(source, target);
49
+ return fs_1.default.promises.rename(source, target);
50
+ return fs_1.default.promises.copyFile(source, target);
48
51
  };
49
- for (const filePath of filePaths) {
52
+ await Promise.all(filePaths.map(async (rawfilePath) => {
53
+ const filePath = pathe_1.default.normalize(rawfilePath);
54
+ const replacedFilePath = filePath.replace(regex, replacement);
50
55
  // 重命名文件
51
- const newFilePath = filePath.replace(regex, replacement);
56
+ const oldFilePath = absin
57
+ ? pathe_1.default.join(searchPath, filePath)
58
+ : filePath;
59
+ const newFilePath = absout
60
+ ? pathe_1.default.join(outPath, replacedFilePath) //如果是绝对路径输出则拼接绝对路径
61
+ : replacedFilePath;
52
62
  // 如果文件名发生了变化
53
- if (newFilePath === filePath)
54
- continue;
63
+ if (newFilePath === oldFilePath)
64
+ return;
55
65
  //如果文件已存在
56
66
  if (await utils_1.UtilFT.pathExists(newFilePath)) {
57
- if (DupMethodWithoutMove.includes(options.duplicateHandling)) {
58
- const fixhd = duplicateHandling;
59
- await (0, utils_1.match)(fixhd, {
67
+ //如果是跳过或覆盖
68
+ if (DupMethodList.includes(options.duplicateHandling)) {
69
+ return (0, utils_1.match)(duplicateHandling, {
60
70
  'skip': () => utils_1.SLogger.info(`重名文件存在,跳过:${newFilePath}`),
61
- 'overwrite': () => mapPath(filePath, newFilePath),
71
+ 'overwrite': () => mapPath(oldFilePath, newFilePath),
62
72
  });
63
73
  }
64
- else
65
- await mapPath(filePath, pathe_1.default.join(duplicateHandling, newFilePath));
74
+ //如果是转移位置
75
+ return mapPath(oldFilePath, pathe_1.default.join(outPath, duplicateHandling, replacedFilePath));
66
76
  }
67
- else
68
- await mapPath(filePath, newFilePath);
69
- }
77
+ return mapPath(oldFilePath, newFilePath);
78
+ }));
70
79
  });
71
80
  exports.CmdMapPath = CmdMapPath;
@@ -34,6 +34,8 @@ function checkVersion(oldVersion, newVersion) {
34
34
  async function updateVersion(newVersion) {
35
35
  const packagePath = pathe_1.default.join(RouteInterface_1.PROCESS_PATH, "package.json");
36
36
  const packageData = await (0, utils_1.memoize)(utils_1.UtilFT.loadJSONFile)(packagePath);
37
+ if (newVersion == 'current')
38
+ return packageData.version;
37
39
  if (newVersion) {
38
40
  checkVersion(packageData.version, newVersion);
39
41
  packageData.version = newVersion;
@@ -59,7 +61,7 @@ const CmdRelease = (program) => program
59
61
  .command("Release")
60
62
  .alias("release")
61
63
  .description("更新版本号并发布包")
62
- .option("-v, --version <version>", "指定发布的版本号 格式应为 `${number}.${number}.${number}`")
64
+ .option("-v, --version <version>", "指定发布的版本号, 为 `current` 时不更新版本号, 格式应为 `${number}.${number}.${number}`")
63
65
  .option("-a, --access <access>", "npm publish 的 access 参数 默认 public", "public")
64
66
  .option("-l, --local <path>", "仅打包到本地对印目录下 如./build/", undefined)
65
67
  .action(async (opt) => {
@@ -5,97 +5,117 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
5
5
  Object.defineProperty(exports, "__esModule", { value: true });
6
6
  exports.CmdScanDups = void 0;
7
7
  const utils_1 = require("@zwa73/utils");
8
- const fs_1 = __importDefault(require("fs"));
9
- const crypto_1 = __importDefault(require("crypto"));
10
8
  const pathe_1 = __importDefault(require("pathe"));
11
9
  const cli_progress_1 = __importDefault(require("cli-progress"));
10
+ const fs_1 = __importDefault(require("fs"));
12
11
  const { tap } = utils_1.UtilFP;
13
- async function calculateHash(filePath) {
14
- return new Promise((resolve, reject) => {
15
- const hash = crypto_1.default.createHash('md5');
16
- const stream = fs_1.default.createReadStream(filePath);
17
- stream.on('data', (data) => hash.update(data));
18
- stream.on('end', () => resolve(hash.digest('hex')));
19
- stream.on('error', reject);
20
- });
21
- }
22
- async function calculateSampledHash(filePath, chunkSize = 1024, // 每块大小(例如 1KB)
23
- chunkCount = 10) {
24
- const stats = await fs_1.default.promises.stat(filePath);
25
- const totalSize = stats.size;
26
- if (totalSize < chunkSize * chunkCount)
27
- return calculateHash(filePath);
28
- const positions = Array.from({ length: chunkCount }, (_, i) => Math.floor((totalSize / chunkCount) * i));
29
- const hash = crypto_1.default.createHash("md5"); // 或其他哈希算法,如 CRC32/BLAKE3
30
- for (const position of positions) {
31
- const buffer = await new Promise((resolve, reject) => {
32
- // 创建一个文件流,限制读取范围
33
- const stream = fs_1.default.createReadStream(filePath, {
34
- start: position,
35
- end: Math.min(position + chunkSize - 1, totalSize - 1), // 确保不会超出文件末尾
36
- highWaterMark: chunkSize, // 高效流式块
37
- });
38
- const chunks = [];
39
- stream.on("data", (chunk) => chunks.push(chunk));
40
- stream.on("end", () => resolve(Buffer.concat(chunks))); // 合并块数据
41
- stream.on("error", reject);
12
+ /**hashlist转为dupmap */
13
+ const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
14
+ const files = acc[cur.hash] ?? [];
15
+ acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
16
+ return acc;
17
+ }, {});
18
+ /**从dupmap提取重复的filepath */
19
+ const reduce2DupFpList = (map) => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
20
+ []);
21
+ /**扫描文件结构 */
22
+ const scanDirStruct = (0, utils_1.memoize)(async (root) => {
23
+ const list = await fs_1.default.promises.readdir(root, { withFileTypes: true });
24
+ const stack = await Promise.all(list.map(async (fp) => fp.isDirectory()
25
+ ? [fp.name, await scanDirStruct(pathe_1.default.join(root, fp.name))]
26
+ : [fp.name, pathe_1.default.join(root, fp.name)]));
27
+ return stack.reduce((acc, cur) => ({
28
+ ...acc, [cur[0]]: cur[1],
29
+ }), {});
30
+ });
31
+ const hashQueue = new utils_1.PromiseQueue({ concurrency: 8 });
32
+ /**计算文件结构的hash */
33
+ async function struct2hash(struct, fn) {
34
+ const recursion = (struct) => {
35
+ return Object.entries(struct).map(async ([k, v]) => {
36
+ if (typeof v === "string")
37
+ return [await hashQueue.enqueue(async () => fn(v))];
38
+ return (await Promise.all(recursion(v))).flat();
42
39
  });
43
- hash.update(buffer); // 更新哈希计算
44
- }
45
- return hash.digest("hex"); // 返回最终哈希值
40
+ };
41
+ return utils_1.UtilFunc.calcHash((await Promise.all(recursion(struct)))
42
+ .flat().join('|'), { algorithm: "blake2b512" });
46
43
  }
47
44
  /**重命名文件或路径 scan_duplicates */
48
45
  const CmdScanDups = (program) => program
49
46
  .command("Scan-Dups")
50
47
  .alias("scandups")
51
48
  .description("扫描当前目录下hash重复的文件")
52
- .option("-re, --regex <regex>", "文件的正则表达式, 使用posix路径", ".*")
49
+ .option("-i, --include <regex>", "文件的正则表达式, 使用posix路径", ".*")
53
50
  .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
54
51
  .option("-r, --recursive", "是否处理子目录, 默认 true", true)
52
+ .option("-s, --struct", `结构模式
53
+ 只扫描 -i 参数目录下的单层目录, 并计算目录是否出现相同的层次结构
54
+ 默认 false`, true)
55
+ .option("-d, --dir <dirs...>", `扫描的根目录, 以空格分隔 -d "a/b c" "d e/f", 默认为命令行当前目录`, [])
55
56
  .action(async (options) => {
56
- const regex = new RegExp(options.regex);
57
+ const { out: outtype, recursive, include, struct, dir } = options;
58
+ const regex = new RegExp(include);
57
59
  const basePath = process.cwd();
58
- // **添加一个多步进度条**
60
+ const pathList = dir.length <= 0
61
+ ? [process.cwd()]
62
+ : dir.map(d => pathe_1.default.isAbsolute(d) ? d : pathe_1.default.join(process.cwd(), d));
63
+ // 添加一个多步进度条
59
64
  const progressBar = new cli_progress_1.default.MultiBar({
60
65
  clearOnComplete: true, hideCursor: true,
61
66
  format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
62
67
  }, cli_progress_1.default.Presets.shades_classic);
68
+ // 文件结构进度条
69
+ const structProgress = struct ? progressBar.create(1, 0, { task: "结构扫描", status: "准备中..." }) : undefined;
63
70
  // 采样哈希进度条
64
71
  const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
65
72
  // 完整哈希进度条
66
73
  const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
67
- const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
68
- const files = acc[cur.hash] ?? [];
69
- acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
70
- return acc;
71
- }, {});
72
74
  await (0, utils_1.pipe)(
73
75
  // 第一步:文件搜索,获取符合正则的文件路径
74
- utils_1.UtilFT.fileSearchRegex(basePath, regex.source, { recursive: options.recursive }), tap(list => void sampledProgress.setTotal(list.length) ??
75
- void sampledProgress.update(0, { status: `总计 ${list.length} 个文件` })),
76
+ Promise.all(pathList.map(async (p) => {
77
+ if (struct)
78
+ return (await fs_1.default.promises.readdir(p, { withFileTypes: true }))
79
+ .filter(fp => fp.isDirectory())
80
+ .map(fp => pathe_1.default.join(p, fp.name));
81
+ return await utils_1.UtilFT.fileSearchRegex(p, regex.source, { recursive });
82
+ })), stacklist => stacklist.flat(), // 扁平化文件路径列表
83
+ // 如果是结构模式则先筛选相同结构
84
+ // 扁平化文件路径列表
85
+ list => struct ? (0, utils_1.pipe)(list, tap(dupFpList => (structProgress.setTotal(dupFpList.length),
86
+ structProgress.update(0, { status: `结构扫描检出 ${dupFpList.length} 个可能的相等项` }))), list => utils_1.Stream.from(list, 8)
87
+ .map(async (filePath) => ({
88
+ filePath,
89
+ hash: await struct2hash(await scanDirStruct(filePath), cpath => pathe_1.default.relative(filePath, cpath)).then(tap(() => structProgress.increment()))
90
+ }))
91
+ .toArray(), reduce2Dupmap, reduce2DupFpList) : list,
76
92
  // 第二步:快速扫描,计算采样哈希
77
- list => utils_1.Stream.from(list, 8)
93
+ tap(list => (sampledProgress.setTotal(list.length),
94
+ sampledProgress.update(0, { status: `总计 ${list.length} 个文件` }))), list => utils_1.Stream.from(list, 8)
78
95
  .map(async (filePath) => ({
79
96
  filePath,
80
- hash: await calculateSampledHash(filePath).then(tap(() => sampledProgress.increment())),
97
+ hash: struct
98
+ ? await struct2hash(await scanDirStruct(filePath), (str) => utils_1.UtilFT.calculateHash(str, { sampled: true })).then(tap(() => sampledProgress.increment()))
99
+ : await utils_1.UtilFT.calculateHash(filePath, { sampled: true }).then(tap(() => sampledProgress.increment())),
81
100
  }))
82
101
  .toArray(),
83
- // 第三步:筛选重复的采样哈希(去掉唯一的采样哈希)
84
- reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
85
- []), tap(dupPaths => void fullHashProgress.setTotal(dupPaths.length) ??
86
- void fullHashProgress.update(0, { status: `快速扫描检出 ${dupPaths.length} 个可能的相等项` })),
102
+ // 第三步:筛选重复的采样哈希 (去掉唯一的采样哈希)
103
+ reduce2Dupmap, reduce2DupFpList,
87
104
  // 第四步:对筛选出的重复文件路径并发计算完整哈希
88
- dups => utils_1.Stream.from(dups, 8)
105
+ tap(dupFpList => (fullHashProgress.setTotal(dupFpList.length),
106
+ fullHashProgress.update(0, { status: `快速扫描检出 ${dupFpList.length} 个可能的相等项` }))), dupFpList => utils_1.Stream.from(dupFpList, 8)
89
107
  .map(async (filePath) => ({
90
108
  filePath,
91
- hash: await calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
109
+ hash: struct
110
+ ? await struct2hash(await scanDirStruct(filePath), (str) => utils_1.UtilFT.calculateHash(str)).then(tap(() => sampledProgress.increment()))
111
+ : await utils_1.UtilFT.calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
92
112
  }))
93
113
  .toArray(),
94
- // 第五步:重新整理完整哈希结果,过滤唯一哈希
114
+ // 第五步:重新整理完整哈希结果, 过滤唯一哈希
95
115
  reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}),
96
116
  // 第六步:输出结果
97
- tap(() => progressBar.stop()), out => (0, utils_1.match)(options.out, {
117
+ tap(() => progressBar.stop()), out => (0, utils_1.match)(outtype, {
98
118
  "console": () => utils_1.SLogger.info(out),
99
- }, () => utils_1.UtilFT.writeJSONFile(pathe_1.default.join(basePath, options.out), out)));
119
+ }, () => utils_1.UtilFT.writeJSONFile(pathe_1.default.join(basePath, outtype), out)));
100
120
  });
101
121
  exports.CmdScanDups = CmdScanDups;
@@ -1,8 +1,7 @@
1
1
  import { SLogger, UtilFT, match, throwError } from "@zwa73/utils";
2
2
  import fs from "fs";
3
3
  import path from "pathe";
4
- const DupMethodList = ["skip", "overwrite", "move"];
5
- const DupMethodWithoutMove = DupMethodList.filter(t => t != 'move');
4
+ const DupMethodList = ["skip", "overwrite"];
6
5
  /**重命名文件或路径 */
7
6
  export const CmdMapPath = (program) => program
8
7
  .command("Map-Path")
@@ -11,12 +10,14 @@ export const CmdMapPath = (program) => program
11
10
  .argument("<regex>", "要匹配的正则表达式, posix风格路径")
12
11
  .argument("<replacement>", "替换字符串")
13
12
  .option("-e, --exclude <regex>", "排除文件的正则表达式")
14
- .option(`-d, --duplicate-handling <${DupMethodWithoutMove.join('|')}|[path:string]>`, `处理重名文件的方式:
13
+ .option(`-d, --duplicate-handling <${DupMethodList.join('|')}|[path:string]>`, `处理重名文件的方式:
15
14
  skip 不进行处理
16
15
  overwrite 覆盖重名
17
16
  其他字符串 将重名部分映射到指定目录下的对应位置, 再次重复将会覆盖`, "skip")
18
17
  .option("-r, --recursive", "是否处理子目录", false)
19
18
  .option("-m, --move", "重命名而不是复制文件", false)
19
+ .option("-o, --output <path>", "输出到某个绝对路径而非当前目录", '')
20
+ .option("-i, --input <path>", "扫描某个绝对路径而非当前目录", '')
20
21
  .option("-t, --test", "不对文件进行实际操作, 在控制台输出映射结果", false)
21
22
  .action(async (regexStr, replacement, options) => {
22
23
  const regex = new RegExp(regexStr);
@@ -24,41 +25,49 @@ overwrite 覆盖重名
24
25
  if (!DupMethodList.includes(options.duplicateHandling))
25
26
  throwError(`${options.duplicateHandling} 不是有效的 duplicate-handling`);
26
27
  const duplicateHandling = options.duplicateHandling;
27
- const basePath = process.cwd();
28
+ const absout = options.output.length > 0;
29
+ const absin = options.input.length > 0;
30
+ const searchPath = absin ? options.input : process.cwd();
31
+ const outPath = absout ? options.output : process.cwd();
28
32
  // 遍历当前目录下的所有文件
29
- const filePaths = (await UtilFT.fileSearchRegex(basePath, regex.source, { relative: options.recursive }))
30
- .map((filePath) => path.relative(basePath, filePath))
31
- .filter((filePath) => excludeRegex ? (!excludeRegex.test(filePath)) : true);
33
+ const filePaths = (await UtilFT.fileSearchRegex(searchPath, regex, { relative: options.recursive }))
34
+ .map(fp => path.relative(searchPath, fp))
35
+ .filter(fp => excludeRegex ? (!excludeRegex.test(fp)) : true);
32
36
  //对单个路径映射
33
37
  const mapPath = async (source, target) => {
34
- const dir = path.parse(target).dir;
35
- await UtilFT.ensurePathExists(dir, { dir: true });
36
38
  if (options.test)
37
39
  return SLogger.info(`${source} -> ${target}`);
40
+ const dir = path.parse(target).dir;
41
+ await UtilFT.ensurePathExists(dir, { dir: true });
38
42
  if (options.move)
39
- await fs.promises.rename(source, target);
40
- else
41
- await fs.promises.copyFile(source, target);
43
+ return fs.promises.rename(source, target);
44
+ return fs.promises.copyFile(source, target);
42
45
  };
43
- for (const filePath of filePaths) {
46
+ await Promise.all(filePaths.map(async (rawfilePath) => {
47
+ const filePath = path.normalize(rawfilePath);
48
+ const replacedFilePath = filePath.replace(regex, replacement);
44
49
  // 重命名文件
45
- const newFilePath = filePath.replace(regex, replacement);
50
+ const oldFilePath = absin
51
+ ? path.join(searchPath, filePath)
52
+ : filePath;
53
+ const newFilePath = absout
54
+ ? path.join(outPath, replacedFilePath) //如果是绝对路径输出则拼接绝对路径
55
+ : replacedFilePath;
46
56
  // 如果文件名发生了变化
47
- if (newFilePath === filePath)
48
- continue;
57
+ if (newFilePath === oldFilePath)
58
+ return;
49
59
  //如果文件已存在
50
60
  if (await UtilFT.pathExists(newFilePath)) {
51
- if (DupMethodWithoutMove.includes(options.duplicateHandling)) {
52
- const fixhd = duplicateHandling;
53
- await match(fixhd, {
61
+ //如果是跳过或覆盖
62
+ if (DupMethodList.includes(options.duplicateHandling)) {
63
+ return match(duplicateHandling, {
54
64
  'skip': () => SLogger.info(`重名文件存在,跳过:${newFilePath}`),
55
- 'overwrite': () => mapPath(filePath, newFilePath),
65
+ 'overwrite': () => mapPath(oldFilePath, newFilePath),
56
66
  });
57
67
  }
58
- else
59
- await mapPath(filePath, path.join(duplicateHandling, newFilePath));
68
+ //如果是转移位置
69
+ return mapPath(oldFilePath, path.join(outPath, duplicateHandling, replacedFilePath));
60
70
  }
61
- else
62
- await mapPath(filePath, newFilePath);
63
- }
71
+ return mapPath(oldFilePath, newFilePath);
72
+ }));
64
73
  });
@@ -28,6 +28,8 @@ function checkVersion(oldVersion, newVersion) {
28
28
  async function updateVersion(newVersion) {
29
29
  const packagePath = path.join(PROCESS_PATH, "package.json");
30
30
  const packageData = await memoize(UtilFT.loadJSONFile)(packagePath);
31
+ if (newVersion == 'current')
32
+ return packageData.version;
31
33
  if (newVersion) {
32
34
  checkVersion(packageData.version, newVersion);
33
35
  packageData.version = newVersion;
@@ -53,7 +55,7 @@ export const CmdRelease = (program) => program
53
55
  .command("Release")
54
56
  .alias("release")
55
57
  .description("更新版本号并发布包")
56
- .option("-v, --version <version>", "指定发布的版本号 格式应为 `${number}.${number}.${number}`")
58
+ .option("-v, --version <version>", "指定发布的版本号, 为 `current` 时不更新版本号, 格式应为 `${number}.${number}.${number}`")
57
59
  .option("-a, --access <access>", "npm publish 的 access 参数 默认 public", "public")
58
60
  .option("-l, --local <path>", "仅打包到本地对印目录下 如./build/", undefined)
59
61
  .action(async (opt) => {
@@ -1,94 +1,114 @@
1
- import { match, pipe, SLogger, Stream, UtilFP, UtilFT } from "@zwa73/utils";
2
- import fs from "fs";
3
- import crypto from 'crypto';
1
+ import { match, memoize, pipe, PromiseQueue, SLogger, Stream, UtilFP, UtilFT, UtilFunc } from "@zwa73/utils";
4
2
  import path from "pathe";
5
3
  import cliProgress from "cli-progress";
4
+ import fs from 'fs';
6
5
  const { tap } = UtilFP;
7
- async function calculateHash(filePath) {
8
- return new Promise((resolve, reject) => {
9
- const hash = crypto.createHash('md5');
10
- const stream = fs.createReadStream(filePath);
11
- stream.on('data', (data) => hash.update(data));
12
- stream.on('end', () => resolve(hash.digest('hex')));
13
- stream.on('error', reject);
14
- });
15
- }
16
- async function calculateSampledHash(filePath, chunkSize = 1024, // 每块大小(例如 1KB)
17
- chunkCount = 10) {
18
- const stats = await fs.promises.stat(filePath);
19
- const totalSize = stats.size;
20
- if (totalSize < chunkSize * chunkCount)
21
- return calculateHash(filePath);
22
- const positions = Array.from({ length: chunkCount }, (_, i) => Math.floor((totalSize / chunkCount) * i));
23
- const hash = crypto.createHash("md5"); // 或其他哈希算法,如 CRC32/BLAKE3
24
- for (const position of positions) {
25
- const buffer = await new Promise((resolve, reject) => {
26
- // 创建一个文件流,限制读取范围
27
- const stream = fs.createReadStream(filePath, {
28
- start: position,
29
- end: Math.min(position + chunkSize - 1, totalSize - 1), // 确保不会超出文件末尾
30
- highWaterMark: chunkSize, // 高效流式块
31
- });
32
- const chunks = [];
33
- stream.on("data", (chunk) => chunks.push(chunk));
34
- stream.on("end", () => resolve(Buffer.concat(chunks))); // 合并块数据
35
- stream.on("error", reject);
6
+ /**hashlist转为dupmap */
7
+ const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
8
+ const files = acc[cur.hash] ?? [];
9
+ acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
10
+ return acc;
11
+ }, {});
12
+ /**从dupmap提取重复的filepath */
13
+ const reduce2DupFpList = (map) => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
14
+ []);
15
+ /**扫描文件结构 */
16
+ const scanDirStruct = memoize(async (root) => {
17
+ const list = await fs.promises.readdir(root, { withFileTypes: true });
18
+ const stack = await Promise.all(list.map(async (fp) => fp.isDirectory()
19
+ ? [fp.name, await scanDirStruct(path.join(root, fp.name))]
20
+ : [fp.name, path.join(root, fp.name)]));
21
+ return stack.reduce((acc, cur) => ({
22
+ ...acc, [cur[0]]: cur[1],
23
+ }), {});
24
+ });
25
+ const hashQueue = new PromiseQueue({ concurrency: 8 });
26
+ /**计算文件结构的hash */
27
+ async function struct2hash(struct, fn) {
28
+ const recursion = (struct) => {
29
+ return Object.entries(struct).map(async ([k, v]) => {
30
+ if (typeof v === "string")
31
+ return [await hashQueue.enqueue(async () => fn(v))];
32
+ return (await Promise.all(recursion(v))).flat();
36
33
  });
37
- hash.update(buffer); // 更新哈希计算
38
- }
39
- return hash.digest("hex"); // 返回最终哈希值
34
+ };
35
+ return UtilFunc.calcHash((await Promise.all(recursion(struct)))
36
+ .flat().join('|'), { algorithm: "blake2b512" });
40
37
  }
41
38
  /**重命名文件或路径 scan_duplicates */
42
39
  export const CmdScanDups = (program) => program
43
40
  .command("Scan-Dups")
44
41
  .alias("scandups")
45
42
  .description("扫描当前目录下hash重复的文件")
46
- .option("-re, --regex <regex>", "文件的正则表达式, 使用posix路径", ".*")
43
+ .option("-i, --include <regex>", "文件的正则表达式, 使用posix路径", ".*")
47
44
  .option("-o, --out <dir|console>", "输出的json文件路径, 默认 scandups.json, 为 \"console\" 时无文件输出", "scandups")
48
45
  .option("-r, --recursive", "是否处理子目录, 默认 true", true)
46
+ .option("-s, --struct", `结构模式
47
+ 只扫描 -i 参数目录下的单层目录, 并计算目录是否出现相同的层次结构
48
+ 默认 false`, true)
49
+ .option("-d, --dir <dirs...>", `扫描的根目录, 以空格分隔 -d "a/b c" "d e/f", 默认为命令行当前目录`, [])
49
50
  .action(async (options) => {
50
- const regex = new RegExp(options.regex);
51
+ const { out: outtype, recursive, include, struct, dir } = options;
52
+ const regex = new RegExp(include);
51
53
  const basePath = process.cwd();
52
- // **添加一个多步进度条**
54
+ const pathList = dir.length <= 0
55
+ ? [process.cwd()]
56
+ : dir.map(d => path.isAbsolute(d) ? d : path.join(process.cwd(), d));
57
+ // 添加一个多步进度条
53
58
  const progressBar = new cliProgress.MultiBar({
54
59
  clearOnComplete: true, hideCursor: true,
55
60
  format: " {task} [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {status}",
56
61
  }, cliProgress.Presets.shades_classic);
62
+ // 文件结构进度条
63
+ const structProgress = struct ? progressBar.create(1, 0, { task: "结构扫描", status: "准备中..." }) : undefined;
57
64
  // 采样哈希进度条
58
65
  const sampledProgress = progressBar.create(1, 0, { task: "快速扫描", status: "准备中..." });
59
66
  // 完整哈希进度条
60
67
  const fullHashProgress = progressBar.create(1, 0, { task: "完整扫描", status: "准备中..." });
61
- const reduce2Dupmap = (list) => list.reduce((acc, cur) => {
62
- const files = acc[cur.hash] ?? [];
63
- acc[cur.hash] = [...files, cur.filePath]; // 分类采样哈希到对应文件路径
64
- return acc;
65
- }, {});
66
68
  await pipe(
67
69
  // 第一步:文件搜索,获取符合正则的文件路径
68
- UtilFT.fileSearchRegex(basePath, regex.source, { recursive: options.recursive }), tap(list => void sampledProgress.setTotal(list.length) ??
69
- void sampledProgress.update(0, { status: `总计 ${list.length} 个文件` })),
70
+ Promise.all(pathList.map(async (p) => {
71
+ if (struct)
72
+ return (await fs.promises.readdir(p, { withFileTypes: true }))
73
+ .filter(fp => fp.isDirectory())
74
+ .map(fp => path.join(p, fp.name));
75
+ return await UtilFT.fileSearchRegex(p, regex.source, { recursive });
76
+ })), stacklist => stacklist.flat(), // 扁平化文件路径列表
77
+ // 如果是结构模式则先筛选相同结构
78
+ // 扁平化文件路径列表
79
+ list => struct ? pipe(list, tap(dupFpList => (structProgress.setTotal(dupFpList.length),
80
+ structProgress.update(0, { status: `结构扫描检出 ${dupFpList.length} 个可能的相等项` }))), list => Stream.from(list, 8)
81
+ .map(async (filePath) => ({
82
+ filePath,
83
+ hash: await struct2hash(await scanDirStruct(filePath), cpath => path.relative(filePath, cpath)).then(tap(() => structProgress.increment()))
84
+ }))
85
+ .toArray(), reduce2Dupmap, reduce2DupFpList) : list,
70
86
  // 第二步:快速扫描,计算采样哈希
71
- list => Stream.from(list, 8)
87
+ tap(list => (sampledProgress.setTotal(list.length),
88
+ sampledProgress.update(0, { status: `总计 ${list.length} 个文件` }))), list => Stream.from(list, 8)
72
89
  .map(async (filePath) => ({
73
90
  filePath,
74
- hash: await calculateSampledHash(filePath).then(tap(() => sampledProgress.increment())),
91
+ hash: struct
92
+ ? await struct2hash(await scanDirStruct(filePath), (str) => UtilFT.calculateHash(str, { sampled: true })).then(tap(() => sampledProgress.increment()))
93
+ : await UtilFT.calculateHash(filePath, { sampled: true }).then(tap(() => sampledProgress.increment())),
75
94
  }))
76
95
  .toArray(),
77
- // 第三步:筛选重复的采样哈希(去掉唯一的采样哈希)
78
- reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length > 1 ? [...acc, ...files] : acc, // 筛选出重复采样哈希的文件路径
79
- []), tap(dupPaths => void fullHashProgress.setTotal(dupPaths.length) ??
80
- void fullHashProgress.update(0, { status: `快速扫描检出 ${dupPaths.length} 个可能的相等项` })),
96
+ // 第三步:筛选重复的采样哈希 (去掉唯一的采样哈希)
97
+ reduce2Dupmap, reduce2DupFpList,
81
98
  // 第四步:对筛选出的重复文件路径并发计算完整哈希
82
- dups => Stream.from(dups, 8)
99
+ tap(dupFpList => (fullHashProgress.setTotal(dupFpList.length),
100
+ fullHashProgress.update(0, { status: `快速扫描检出 ${dupFpList.length} 个可能的相等项` }))), dupFpList => Stream.from(dupFpList, 8)
83
101
  .map(async (filePath) => ({
84
102
  filePath,
85
- hash: await calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
103
+ hash: struct
104
+ ? await struct2hash(await scanDirStruct(filePath), (str) => UtilFT.calculateHash(str)).then(tap(() => sampledProgress.increment()))
105
+ : await UtilFT.calculateHash(filePath).then(tap(() => fullHashProgress.increment())), // 计算完整哈希
86
106
  }))
87
107
  .toArray(),
88
- // 第五步:重新整理完整哈希结果,过滤唯一哈希
108
+ // 第五步:重新整理完整哈希结果, 过滤唯一哈希
89
109
  reduce2Dupmap, map => Object.entries(map).reduce((acc, [hash, files]) => files.length <= 1 ? acc : { ...acc, [hash]: files }, {}),
90
110
  // 第六步:输出结果
91
- tap(() => progressBar.stop()), out => match(options.out, {
111
+ tap(() => progressBar.stop()), out => match(outtype, {
92
112
  "console": () => SLogger.info(out),
93
- }, () => UtilFT.writeJSONFile(path.join(basePath, options.out), out)));
113
+ }, () => UtilFT.writeJSONFile(path.join(basePath, outtype), out)));
94
114
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zwa73/dev-utils",
3
- "version": "1.0.87",
3
+ "version": "1.0.88",
4
4
  "description": "编译与调试工具",
5
5
  "exports": {
6
6
  ".": {