@datatruck/cli 0.34.0 → 0.34.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +337 -72
- package/lib/actions/BackupAction.d.ts +4 -4
- package/lib/actions/BackupAction.js +2 -2
- package/lib/actions/CopyAction.d.ts +3 -3
- package/lib/actions/CopyAction.js +2 -2
- package/lib/actions/RestoreAction.d.ts +4 -4
- package/lib/actions/RestoreAction.js +2 -2
- package/lib/commands/CleanCacheCommand.js +2 -2
- package/lib/commands/CommandAbstract.d.ts +4 -4
- package/lib/commands/CommandAbstract.js +1 -1
- package/lib/commands/ConfigCommand.js +2 -2
- package/lib/commands/InitCommand.js +2 -2
- package/lib/commands/PruneCommand.js +2 -2
- package/lib/commands/SnapshotsCommand.js +2 -2
- package/lib/repositories/GitRepository.js +6 -6
- package/lib/repositories/ResticRepository.d.ts +1 -1
- package/lib/repositories/ResticRepository.js +10 -10
- package/lib/tasks/GitTask.js +33 -50
- package/lib/tasks/MariadbTask.js +43 -56
- package/lib/tasks/MssqlTask.js +5 -11
- package/lib/tasks/MysqlDumpTask.js +4 -4
- package/lib/tasks/PostgresqlDumpTask.d.ts +1 -1
- package/lib/tasks/PostgresqlDumpTask.js +9 -32
- package/lib/tasks/SqlDumpTaskAbstract.d.ts +1 -2
- package/lib/tasks/SqlDumpTaskAbstract.js +1 -1
- package/lib/utils/async-process.d.ts +66 -0
- package/lib/utils/async-process.js +242 -0
- package/lib/utils/async.d.ts +3 -5
- package/lib/utils/async.js +2 -2
- package/lib/utils/{DataFormat.d.ts → data-format.d.ts} +4 -4
- package/lib/utils/{DataFormat.js → data-format.js} +1 -1
- package/lib/utils/datatruck/command.d.ts +2 -2
- package/lib/utils/datatruck/config-type.d.ts +1 -1
- package/lib/utils/datatruck/cron-server.js +2 -2
- package/lib/utils/fs.d.ts +1 -2
- package/lib/utils/fs.js +3 -10
- package/lib/utils/{Git.d.ts → git.d.ts} +9 -7
- package/lib/utils/{Git.js → git.js} +30 -29
- package/lib/utils/list.d.ts +4 -4
- package/lib/utils/list.js +1 -1
- package/lib/utils/mysql.d.ts +8 -10
- package/lib/utils/mysql.js +60 -79
- package/lib/utils/process.d.ts +3 -92
- package/lib/utils/process.js +7 -311
- package/lib/utils/{Restic.d.ts → restic.d.ts} +10 -9
- package/lib/utils/{Restic.js → restic.js} +72 -82
- package/lib/utils/spawnSteps.js +9 -10
- package/lib/utils/stream.d.ts +8 -2
- package/lib/utils/stream.js +10 -3
- package/lib/utils/tar.js +29 -49
- package/package.json +2 -2
package/lib/tasks/MariadbTask.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MariadbTask = exports.mariadbTaskName = void 0;
|
|
4
|
+
const async_process_1 = require("../utils/async-process");
|
|
4
5
|
const cli_1 = require("../utils/cli");
|
|
5
6
|
const fs_1 = require("../utils/fs");
|
|
6
7
|
const math_1 = require("../utils/math");
|
|
7
|
-
const process_1 = require("../utils/process");
|
|
8
8
|
const tar_1 = require("../utils/tar");
|
|
9
9
|
const temp_1 = require("../utils/temp");
|
|
10
10
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
@@ -92,15 +92,15 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
92
92
|
await (0, fs_1.forEachFile)(sourcePath, () => {
|
|
93
93
|
total++;
|
|
94
94
|
});
|
|
95
|
-
let p1;
|
|
96
95
|
let lastLineText;
|
|
96
|
+
const controller = new AbortController();
|
|
97
97
|
const pathRegex = /((Copying|Streaming) .+) to/;
|
|
98
98
|
const onData = async (line) => {
|
|
99
99
|
const { text } = parseLine(line);
|
|
100
100
|
lastLineText = text;
|
|
101
101
|
if (line.includes("[ERROR] InnoDB: Unsupported redo log format.") ||
|
|
102
102
|
line.includes("Error: cannot read redo log header")) {
|
|
103
|
-
|
|
103
|
+
controller.abort();
|
|
104
104
|
}
|
|
105
105
|
else {
|
|
106
106
|
const matches = pathRegex.exec(text);
|
|
@@ -121,41 +121,34 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
121
121
|
const stats = { xbFiles: total };
|
|
122
122
|
await (0, promises_1.writeFile)((0, path_1.join)(snapshotPath, "stats.dtt.json"), JSON.stringify(stats));
|
|
123
123
|
if (compress) {
|
|
124
|
-
const
|
|
125
|
-
|
|
124
|
+
const fileStream = (0, fs_2.createWriteStream)((0, path_1.join)(snapshotPath, "db.xb.gz"));
|
|
125
|
+
const dumpProcess = new async_process_1.AsyncProcess(command, args, {
|
|
126
126
|
$log: {
|
|
127
127
|
exec: this.verbose,
|
|
128
128
|
stderr: this.verbose,
|
|
129
129
|
},
|
|
130
|
-
$
|
|
131
|
-
|
|
132
|
-
onData,
|
|
133
|
-
},
|
|
134
|
-
$onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
130
|
+
$controller: controller,
|
|
131
|
+
$exitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
135
132
|
});
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
133
|
+
const compressProcess = new async_process_1.AsyncProcess(compress.command, compress.args);
|
|
134
|
+
await Promise.all([
|
|
135
|
+
dumpProcess.stdout.pipe(compressProcess.stdin),
|
|
136
|
+
dumpProcess.stderr.parseLines(onData),
|
|
137
|
+
compressProcess.stdout.pipe(fileStream),
|
|
138
|
+
]);
|
|
140
139
|
}
|
|
141
140
|
else {
|
|
142
|
-
|
|
141
|
+
const dumpProcess = new async_process_1.AsyncProcess(command, args, {
|
|
143
142
|
$log: this.verbose,
|
|
144
|
-
$
|
|
145
|
-
|
|
146
|
-
onData,
|
|
147
|
-
},
|
|
148
|
-
$stderr: {
|
|
149
|
-
parseLines: true,
|
|
150
|
-
onData,
|
|
151
|
-
},
|
|
152
|
-
$onExitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
153
|
-
});
|
|
154
|
-
await p1;
|
|
155
|
-
await (0, process_1.exec)(command, [`--prepare`, `--target-dir=${snapshotPath}`], undefined, {
|
|
156
|
-
log: this.verbose,
|
|
157
|
-
stderr: { onData: () => { } },
|
|
143
|
+
$controller: controller,
|
|
144
|
+
$exitCode: (code) => `Exit code: ${code} - ${lastLineText}`,
|
|
158
145
|
});
|
|
146
|
+
await Promise.all([
|
|
147
|
+
dumpProcess.stdout.parseLines(onData),
|
|
148
|
+
dumpProcess.stderr.parseLines(onData),
|
|
149
|
+
]);
|
|
150
|
+
const prepareProcess = new async_process_1.AsyncProcess(command, [`--prepare`, `--target-dir=${snapshotPath}`], { $log: this.verbose });
|
|
151
|
+
await prepareProcess.waitForClose();
|
|
159
152
|
}
|
|
160
153
|
return { snapshotPath };
|
|
161
154
|
}
|
|
@@ -230,29 +223,25 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
230
223
|
});
|
|
231
224
|
let currentXbFiles = 0;
|
|
232
225
|
const { parallel } = normalizeConfig({ parallel: this.config.parallel });
|
|
233
|
-
const p1 =
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
},
|
|
253
|
-
});
|
|
254
|
-
xbStream.pipe(p1.stdin, { end: true });
|
|
255
|
-
await Promise.all([(0, fs_1.waitForClose)(xbStream), p1]);
|
|
226
|
+
const p1 = new async_process_1.AsyncProcess("mbstream", ["-x", "-C", snapshotPath, "-v", "-p", parallel], { $log: this.verbose });
|
|
227
|
+
await Promise.all([
|
|
228
|
+
p1.stdin.pipe(xbStream),
|
|
229
|
+
p1.stderr.parseLines((line) => {
|
|
230
|
+
const { text: path } = parseLine(line);
|
|
231
|
+
data.onProgress({
|
|
232
|
+
absolute,
|
|
233
|
+
relative: {
|
|
234
|
+
payload: path,
|
|
235
|
+
format: "amount",
|
|
236
|
+
current: ++currentXbFiles,
|
|
237
|
+
total: stats?.xbFiles,
|
|
238
|
+
percent: stats?.xbFiles
|
|
239
|
+
? (0, math_1.progressPercent)(stats.xbFiles, currentXbFiles)
|
|
240
|
+
: undefined,
|
|
241
|
+
},
|
|
242
|
+
});
|
|
243
|
+
}),
|
|
244
|
+
]);
|
|
256
245
|
}
|
|
257
246
|
// Prepare
|
|
258
247
|
absolute.current++;
|
|
@@ -262,10 +251,8 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
262
251
|
data.onProgress({
|
|
263
252
|
absolute,
|
|
264
253
|
});
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
stderr: { onData: () => { } },
|
|
268
|
-
});
|
|
254
|
+
const p = new async_process_1.AsyncProcess(this.command, [`--prepare`, `--target-dir=${snapshotPath}`], { $log: this.verbose });
|
|
255
|
+
await p.waitForClose();
|
|
269
256
|
}
|
|
270
257
|
await reloadFiles();
|
|
271
258
|
removeFiles.push(...files.filter((file) => file.startsWith("ib_logfile")));
|
package/lib/tasks/MssqlTask.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MssqlTask = exports.mssqlTaskName = void 0;
|
|
4
|
+
const async_process_1 = require("../utils/async-process");
|
|
4
5
|
const config_1 = require("../utils/datatruck/config");
|
|
5
6
|
const error_1 = require("../utils/datatruck/error");
|
|
6
7
|
const fs_1 = require("../utils/fs");
|
|
7
|
-
const process_1 = require("../utils/process");
|
|
8
8
|
const temp_1 = require("../utils/temp");
|
|
9
9
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
10
10
|
const promises_1 = require("fs/promises");
|
|
@@ -18,7 +18,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
18
18
|
return this.config.command ?? "sqlcmd";
|
|
19
19
|
}
|
|
20
20
|
async exec(query) {
|
|
21
|
-
const
|
|
21
|
+
const stdout = await async_process_1.AsyncProcess.stdout(this.command, [
|
|
22
22
|
...(this.config.hostname ? ["-S", this.config.hostname] : []),
|
|
23
23
|
...(this.config.username ? ["-U", this.config.username] : []),
|
|
24
24
|
...(this.config.passwordFile
|
|
@@ -32,16 +32,10 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
32
32
|
"999",
|
|
33
33
|
"-Q",
|
|
34
34
|
`SET nocount ON; ${query.replace(/[\n\t]/g, " ")}`,
|
|
35
|
-
],
|
|
36
|
-
log: this.verbose,
|
|
37
|
-
stderr: {
|
|
38
|
-
toExitCode: true,
|
|
39
|
-
},
|
|
40
|
-
stdout: {
|
|
41
|
-
save: true,
|
|
42
|
-
},
|
|
35
|
+
], {
|
|
36
|
+
$log: this.verbose,
|
|
43
37
|
});
|
|
44
|
-
return
|
|
38
|
+
return stdout
|
|
45
39
|
.split(/\n/g)
|
|
46
40
|
.map((row) => row.split(","))
|
|
47
41
|
.filter((row) => row.length);
|
|
@@ -80,7 +80,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
80
80
|
sharedPath: tableSharedPath,
|
|
81
81
|
items: [tableName],
|
|
82
82
|
database: this.config.database,
|
|
83
|
-
|
|
83
|
+
controller,
|
|
84
84
|
});
|
|
85
85
|
const files = await (0, promises_1.readdir)(tableSharedPath);
|
|
86
86
|
const schemaFile = `${tableName}.sql`;
|
|
@@ -106,7 +106,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
106
106
|
output: outPath,
|
|
107
107
|
items: [tableName],
|
|
108
108
|
database: this.config.database,
|
|
109
|
-
|
|
109
|
+
controller,
|
|
110
110
|
...(concurrency === 1 && {
|
|
111
111
|
onProgress(progress) {
|
|
112
112
|
data.onProgress({
|
|
@@ -256,7 +256,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
256
256
|
await sql.importFile({
|
|
257
257
|
path,
|
|
258
258
|
database: database.name,
|
|
259
|
-
|
|
259
|
+
controller,
|
|
260
260
|
});
|
|
261
261
|
}
|
|
262
262
|
finally {
|
|
@@ -306,7 +306,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
306
306
|
path: csvFile,
|
|
307
307
|
database: database.name,
|
|
308
308
|
table: tableName,
|
|
309
|
-
|
|
309
|
+
controller,
|
|
310
310
|
});
|
|
311
311
|
}
|
|
312
312
|
finally {
|
|
@@ -5,7 +5,7 @@ export declare class PostgresqlDumpTask extends SqlDumpTaskAbstract<PostgresqlDu
|
|
|
5
5
|
buildConnectionArgs(database?: string): Promise<string[]>;
|
|
6
6
|
onDatabaseIsEmpty(name: string): Promise<boolean>;
|
|
7
7
|
onCreateDatabase(database: TargetDatabase): Promise<void>;
|
|
8
|
-
onExecQuery(query: string): Promise<
|
|
8
|
+
onExecQuery(query: string): Promise<string>;
|
|
9
9
|
onFetchTableNames(database: string): Promise<string[]>;
|
|
10
10
|
onExportTables(tableNames: string[], output: string, onProgress: (progress: {
|
|
11
11
|
totalBytes: number;
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.PostgresqlDumpTask = exports.postgresqlDumpTaskName = void 0;
|
|
4
|
-
const
|
|
4
|
+
const async_process_1 = require("../utils/async-process");
|
|
5
5
|
const SqlDumpTaskAbstract_1 = require("./SqlDumpTaskAbstract");
|
|
6
|
-
const fs_1 = require("fs");
|
|
7
6
|
const path_1 = require("path");
|
|
8
7
|
exports.postgresqlDumpTaskName = "postgresql-dump";
|
|
9
8
|
class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
@@ -41,20 +40,12 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
41
40
|
await this.onExecQuery(query);
|
|
42
41
|
}
|
|
43
42
|
async onExecQuery(query) {
|
|
44
|
-
return await
|
|
43
|
+
return await async_process_1.AsyncProcess.stdout("psql", [
|
|
45
44
|
...(await this.buildConnectionArgs()),
|
|
46
45
|
"-t",
|
|
47
46
|
"-c",
|
|
48
47
|
query.replace(/\s{1,}/g, " "),
|
|
49
|
-
],
|
|
50
|
-
log: this.verbose,
|
|
51
|
-
stderr: {
|
|
52
|
-
toExitCode: true,
|
|
53
|
-
},
|
|
54
|
-
stdout: {
|
|
55
|
-
save: true,
|
|
56
|
-
},
|
|
57
|
-
});
|
|
48
|
+
], { $log: this.verbose });
|
|
58
49
|
}
|
|
59
50
|
async onFetchTableNames(database) {
|
|
60
51
|
return await this.fetchValues(`
|
|
@@ -70,31 +61,17 @@ class PostgresqlDumpTask extends SqlDumpTaskAbstract_1.SqlDumpTaskAbstract {
|
|
|
70
61
|
`);
|
|
71
62
|
}
|
|
72
63
|
async onExportTables(tableNames, output, onProgress) {
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
}),
|
|
79
|
-
(0, process_1.exec)("pg_dump", [
|
|
80
|
-
...(await this.buildConnectionArgs(this.config.database)),
|
|
81
|
-
...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
|
|
82
|
-
], null, {
|
|
83
|
-
pipe: { stream, onWriteProgress: onProgress },
|
|
84
|
-
stderr: {
|
|
85
|
-
toExitCode: true,
|
|
86
|
-
},
|
|
87
|
-
log: this.verbose,
|
|
88
|
-
}),
|
|
89
|
-
]);
|
|
64
|
+
const dumpProcess = new async_process_1.AsyncProcess("pg_dump", [
|
|
65
|
+
...(await this.buildConnectionArgs(this.config.database)),
|
|
66
|
+
...(tableNames?.flatMap((v) => ["-t", v]) ?? []),
|
|
67
|
+
], { $log: this.verbose });
|
|
68
|
+
await dumpProcess.stdout.pipe(output, onProgress);
|
|
90
69
|
}
|
|
91
70
|
async onExportStoredPrograms() {
|
|
92
71
|
throw new Error(`Method not implemented: onExportStoredPrograms`);
|
|
93
72
|
}
|
|
94
73
|
async onImport(path, database) {
|
|
95
|
-
await
|
|
96
|
-
log: this.verbose,
|
|
97
|
-
});
|
|
74
|
+
await async_process_1.AsyncProcess.exec("psql", [...(await this.buildConnectionArgs(database)), "-f", (0, path_1.normalize)(path)], { $log: this.verbose });
|
|
98
75
|
}
|
|
99
76
|
}
|
|
100
77
|
exports.PostgresqlDumpTask = PostgresqlDumpTask;
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { exec } from "../utils/process";
|
|
2
1
|
import { TaskBackupData, TaskPrepareRestoreData, TaskRestoreData, TaskAbstract } from "./TaskAbstract";
|
|
3
2
|
export type TargetDatabase = {
|
|
4
3
|
name: string;
|
|
@@ -29,7 +28,7 @@ export declare abstract class SqlDumpTaskAbstract<TConfig extends SqlDumpTaskCon
|
|
|
29
28
|
abstract onCreateDatabase(database: TargetDatabase): Promise<void>;
|
|
30
29
|
abstract onDatabaseIsEmpty(databaseName: string): Promise<boolean>;
|
|
31
30
|
abstract onFetchTableNames(database: string): Promise<string[]>;
|
|
32
|
-
abstract onExecQuery(query: string):
|
|
31
|
+
abstract onExecQuery(query: string): Promise<string>;
|
|
33
32
|
abstract onExportTables(tableNames: string[], output: string, onProgress: (data: {
|
|
34
33
|
totalBytes: number;
|
|
35
34
|
}) => void): Promise<void>;
|
|
@@ -56,7 +56,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
|
|
|
56
56
|
}
|
|
57
57
|
async fetchValues(query) {
|
|
58
58
|
const result = await this.onExecQuery(query);
|
|
59
|
-
return result.
|
|
59
|
+
return result.split(/\r?\n/).reduce((result, value) => {
|
|
60
60
|
value = value.trim();
|
|
61
61
|
if (value.length)
|
|
62
62
|
result.push(value);
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
/// <reference types="node" />
|
|
3
|
+
/// <reference types="node" />
|
|
4
|
+
/// <reference types="node" />
|
|
5
|
+
/// <reference types="node" />
|
|
6
|
+
import { ChildProcess, SpawnOptions } from "child_process";
|
|
7
|
+
import { ReadStream } from "fs";
|
|
8
|
+
import { Readable, Writable } from "stream";
|
|
9
|
+
export type AsyncProcessOptions = SpawnOptions & {
|
|
10
|
+
$log?: AsyncProcessLog | boolean;
|
|
11
|
+
$controller?: AbortController;
|
|
12
|
+
$exitCode?: ExitCode;
|
|
13
|
+
};
|
|
14
|
+
type ExitCode = ExitCodeValue | ((code: number) => ExitCodeValue | void | undefined);
|
|
15
|
+
type ExitCodeValue = Error | string | number | boolean;
|
|
16
|
+
type AsyncProcessArgv = (string | number)[] | undefined;
|
|
17
|
+
declare class StdIn {
|
|
18
|
+
readonly process: AsyncProcess;
|
|
19
|
+
readonly writable: Writable;
|
|
20
|
+
constructor(process: AsyncProcess, writable: Writable);
|
|
21
|
+
pipe(source: string | ReadStream, onProgress?: (data: {
|
|
22
|
+
totalBytes: number;
|
|
23
|
+
currentBytes: number;
|
|
24
|
+
progress: number;
|
|
25
|
+
}) => void): Promise<void>;
|
|
26
|
+
}
|
|
27
|
+
declare class Std {
|
|
28
|
+
protected type: "stdout" | "stderr";
|
|
29
|
+
readonly process: AsyncProcess;
|
|
30
|
+
readonly readable: Readable;
|
|
31
|
+
constructor(type: "stdout" | "stderr", process: AsyncProcess, readable: Readable);
|
|
32
|
+
onData(cb: (chunk: Buffer) => void): void;
|
|
33
|
+
fetch(): Promise<string>;
|
|
34
|
+
parseLines(cb: (line: string, total: number) => void): Promise<number>;
|
|
35
|
+
pipe(out: string | NodeJS.WritableStream | StdIn, onProgress?: (data: {
|
|
36
|
+
totalBytes: number;
|
|
37
|
+
}) => void): Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
export type AsyncProcessLog = {
|
|
40
|
+
colorize?: boolean;
|
|
41
|
+
exec?: boolean;
|
|
42
|
+
stdout?: boolean;
|
|
43
|
+
stderr?: boolean;
|
|
44
|
+
allToStderr?: boolean;
|
|
45
|
+
envNames?: string[];
|
|
46
|
+
};
|
|
47
|
+
export declare class AsyncProcess {
|
|
48
|
+
protected command: string;
|
|
49
|
+
protected argv: AsyncProcessArgv;
|
|
50
|
+
protected options: AsyncProcessOptions;
|
|
51
|
+
readonly child: ChildProcess;
|
|
52
|
+
readonly stdout: Std;
|
|
53
|
+
readonly stderr: Std;
|
|
54
|
+
readonly stdin: StdIn;
|
|
55
|
+
protected log: AsyncProcessLog | undefined;
|
|
56
|
+
protected controller: AbortController;
|
|
57
|
+
private lastStdError;
|
|
58
|
+
constructor(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions);
|
|
59
|
+
static exec(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions): Promise<number>;
|
|
60
|
+
static stdout(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions): Promise<string>;
|
|
61
|
+
private installLog;
|
|
62
|
+
private installAbortController;
|
|
63
|
+
private resolveExitCode;
|
|
64
|
+
waitForClose(): Promise<number>;
|
|
65
|
+
}
|
|
66
|
+
export {};
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AsyncProcess = void 0;
|
|
4
|
+
const cli_1 = require("./cli");
|
|
5
|
+
const math_1 = require("./math");
|
|
6
|
+
const process_1 = require("./process");
|
|
7
|
+
const stream_1 = require("./stream");
|
|
8
|
+
const child_process_1 = require("child_process");
|
|
9
|
+
const fs_1 = require("fs");
|
|
10
|
+
const promises_1 = require("fs/promises");
|
|
11
|
+
const readline_1 = require("readline");
|
|
12
|
+
function resolveLogOptions(log) {
|
|
13
|
+
return log === true
|
|
14
|
+
? { exec: {}, stdout: {}, stderr: {} }
|
|
15
|
+
: log || undefined;
|
|
16
|
+
}
|
|
17
|
+
function ensureDir(cwd) {
|
|
18
|
+
if (typeof cwd === "string") {
|
|
19
|
+
let isDir = false;
|
|
20
|
+
try {
|
|
21
|
+
isDir = (0, fs_1.statSync)(cwd).isDirectory();
|
|
22
|
+
}
|
|
23
|
+
catch (error) { }
|
|
24
|
+
if (!isDir)
|
|
25
|
+
throw new Error(`Current working directory does not exist: ${cwd}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
class StdIn {
|
|
29
|
+
process;
|
|
30
|
+
writable;
|
|
31
|
+
constructor(process, writable) {
|
|
32
|
+
this.process = process;
|
|
33
|
+
this.writable = writable;
|
|
34
|
+
}
|
|
35
|
+
async pipe(source, onProgress) {
|
|
36
|
+
if (this.process["log"]?.exec) {
|
|
37
|
+
const path = source instanceof fs_1.ReadStream
|
|
38
|
+
? "path" in source
|
|
39
|
+
? source.path
|
|
40
|
+
: "&readableStream"
|
|
41
|
+
: source;
|
|
42
|
+
(0, cli_1.logExec)(`[${this.process.child.pid || 0}] < ${path}`);
|
|
43
|
+
}
|
|
44
|
+
const stream = typeof source === "string" ? (0, fs_1.createReadStream)(source) : source;
|
|
45
|
+
const streamPath = stream.path.toString();
|
|
46
|
+
if (onProgress) {
|
|
47
|
+
const fileInfo = await (0, promises_1.stat)(streamPath);
|
|
48
|
+
const totalBytes = fileInfo.size;
|
|
49
|
+
let currentBytes = 0;
|
|
50
|
+
stream.on("data", (data) => {
|
|
51
|
+
currentBytes += data.length;
|
|
52
|
+
onProgress?.({
|
|
53
|
+
totalBytes: totalBytes,
|
|
54
|
+
currentBytes: currentBytes,
|
|
55
|
+
progress: (0, math_1.progressPercent)(totalBytes, currentBytes),
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
stream.pipe(this.writable);
|
|
60
|
+
await Promise.all([
|
|
61
|
+
this.process.waitForClose(),
|
|
62
|
+
(0, stream_1.waitForClose)(stream),
|
|
63
|
+
(0, stream_1.waitForClose)(this.writable),
|
|
64
|
+
]);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
class Std {
|
|
68
|
+
type;
|
|
69
|
+
process;
|
|
70
|
+
readable;
|
|
71
|
+
constructor(type, process, readable) {
|
|
72
|
+
this.type = type;
|
|
73
|
+
this.process = process;
|
|
74
|
+
this.readable = readable;
|
|
75
|
+
}
|
|
76
|
+
onData(cb) {
|
|
77
|
+
this.readable.on("data", cb);
|
|
78
|
+
}
|
|
79
|
+
async fetch() {
|
|
80
|
+
let data = "";
|
|
81
|
+
this.onData((chunk) => (data += chunk));
|
|
82
|
+
await this.process.waitForClose();
|
|
83
|
+
return data.trim();
|
|
84
|
+
}
|
|
85
|
+
async parseLines(cb) {
|
|
86
|
+
let total = 0;
|
|
87
|
+
const parser = (0, readline_1.createInterface)({ input: this.readable });
|
|
88
|
+
parser.on("line", (inLine) => {
|
|
89
|
+
const line = inLine.toString().trim();
|
|
90
|
+
if (line.length)
|
|
91
|
+
cb(line, ++total);
|
|
92
|
+
});
|
|
93
|
+
await Promise.all([this.process.waitForClose(), (0, stream_1.waitForClose)(parser)]);
|
|
94
|
+
return total;
|
|
95
|
+
}
|
|
96
|
+
async pipe(out, onProgress) {
|
|
97
|
+
if (this.process["log"]?.exec) {
|
|
98
|
+
const path = out instanceof StdIn
|
|
99
|
+
? `${[out.process.child.pid || 0]}`
|
|
100
|
+
: typeof out === "string"
|
|
101
|
+
? out
|
|
102
|
+
: "path" in out
|
|
103
|
+
? out.path
|
|
104
|
+
: "&writableStream";
|
|
105
|
+
(0, cli_1.logExec)(`[${this.process.child.pid || 0}] ${this.type === "stderr" ? 2 : ""}> ${path}`);
|
|
106
|
+
}
|
|
107
|
+
if (onProgress) {
|
|
108
|
+
let totalBytes = 0;
|
|
109
|
+
this.readable.on("data", (chunk) => {
|
|
110
|
+
totalBytes += chunk.length;
|
|
111
|
+
onProgress({ totalBytes });
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
const stream = out instanceof StdIn
|
|
115
|
+
? out.writable
|
|
116
|
+
: typeof out === "string"
|
|
117
|
+
? (0, fs_1.createWriteStream)(out)
|
|
118
|
+
: out;
|
|
119
|
+
this.readable.pipe(stream);
|
|
120
|
+
await Promise.all([
|
|
121
|
+
this.process.waitForClose().catch((error) => {
|
|
122
|
+
if ("destroy" in stream)
|
|
123
|
+
stream.destroy();
|
|
124
|
+
return Promise.reject(error);
|
|
125
|
+
}),
|
|
126
|
+
(0, stream_1.waitForClose)(stream),
|
|
127
|
+
(0, stream_1.waitForClose)(this.readable),
|
|
128
|
+
]);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
class AsyncProcess {
|
|
132
|
+
command;
|
|
133
|
+
argv;
|
|
134
|
+
options;
|
|
135
|
+
child;
|
|
136
|
+
stdout;
|
|
137
|
+
stderr;
|
|
138
|
+
stdin;
|
|
139
|
+
log;
|
|
140
|
+
controller;
|
|
141
|
+
lastStdError;
|
|
142
|
+
constructor(command, argv, options = {}) {
|
|
143
|
+
this.command = command;
|
|
144
|
+
this.argv = argv;
|
|
145
|
+
this.options = options;
|
|
146
|
+
const { $log, $controller, ...otherOptions } = options;
|
|
147
|
+
this.log = resolveLogOptions($log);
|
|
148
|
+
this.controller = $controller || new AbortController();
|
|
149
|
+
if (this.log?.exec)
|
|
150
|
+
(0, process_1.logProcess)(command, argv || [], this.log.exec === true ? {} : this.log.exec);
|
|
151
|
+
if (typeof options.cwd === "string")
|
|
152
|
+
ensureDir(options.cwd);
|
|
153
|
+
this.child = (0, child_process_1.spawn)(command, argv?.map(String) || [], otherOptions ?? {});
|
|
154
|
+
if (this.log)
|
|
155
|
+
this.installLog(this.log);
|
|
156
|
+
this.installAbortController(this.controller);
|
|
157
|
+
this.stdout = new Std("stdout", this, this.child.stdout);
|
|
158
|
+
this.stderr = new Std("stderr", this, this.child.stderr);
|
|
159
|
+
this.stdin = new StdIn(this, this.child.stdin);
|
|
160
|
+
}
|
|
161
|
+
static async exec(command, argv, options = {}) {
|
|
162
|
+
const p = new AsyncProcess(command, argv, options);
|
|
163
|
+
return await p.waitForClose();
|
|
164
|
+
}
|
|
165
|
+
static async stdout(command, argv, options = {}) {
|
|
166
|
+
const p = new AsyncProcess(command, argv, options);
|
|
167
|
+
return await p.stdout.fetch();
|
|
168
|
+
}
|
|
169
|
+
installLog(log) {
|
|
170
|
+
if (log.stdout)
|
|
171
|
+
this.child.stdout?.on("data", (chunk) => (0, process_1.logStdout)({
|
|
172
|
+
data: chunk.toString(),
|
|
173
|
+
colorize: log.colorize,
|
|
174
|
+
stderr: log.allToStderr,
|
|
175
|
+
}));
|
|
176
|
+
if (log.stderr)
|
|
177
|
+
this.child.stderr?.on("data", (chunk) => (0, process_1.logStdout)({
|
|
178
|
+
data: chunk.toString(),
|
|
179
|
+
colorize: log.colorize,
|
|
180
|
+
stderr: log.allToStderr,
|
|
181
|
+
}));
|
|
182
|
+
}
|
|
183
|
+
installAbortController(controller) {
|
|
184
|
+
if (controller.signal.aborted) {
|
|
185
|
+
this.child.kill();
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
controller.signal.addEventListener("abort", () => {
|
|
189
|
+
this.child.kill();
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
resolveExitCode(inExitCode) {
|
|
194
|
+
let exitCode = inExitCode ?? 32;
|
|
195
|
+
if (!exitCode)
|
|
196
|
+
return exitCode;
|
|
197
|
+
const lastStdError = this.lastStdError?.toString().trim().slice(0, 255);
|
|
198
|
+
let result = this.options.$exitCode ?? true;
|
|
199
|
+
let message = (inExitCode === null
|
|
200
|
+
? [`Process killed: ${this.command}`, lastStdError]
|
|
201
|
+
: [`Process exit code: ${exitCode} (${this.command})`, lastStdError])
|
|
202
|
+
.filter((v) => typeof v === "string" && v.length)
|
|
203
|
+
.join(" | ");
|
|
204
|
+
if (typeof result === "function")
|
|
205
|
+
result = result(exitCode);
|
|
206
|
+
if (typeof result === "string") {
|
|
207
|
+
message = result;
|
|
208
|
+
}
|
|
209
|
+
else if (typeof result === "number") {
|
|
210
|
+
exitCode = result;
|
|
211
|
+
}
|
|
212
|
+
else if (result instanceof Error) {
|
|
213
|
+
return result;
|
|
214
|
+
}
|
|
215
|
+
else if (result === false) {
|
|
216
|
+
return exitCode;
|
|
217
|
+
}
|
|
218
|
+
return new Error(message, {
|
|
219
|
+
cause: {
|
|
220
|
+
command: this.command,
|
|
221
|
+
argv: this.argv,
|
|
222
|
+
exitCode,
|
|
223
|
+
lastStdError,
|
|
224
|
+
},
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
async waitForClose() {
|
|
228
|
+
if (!this.lastStdError) {
|
|
229
|
+
this.lastStdError = Buffer.from([]);
|
|
230
|
+
this.child.stderr?.on("data", (chunk) => {
|
|
231
|
+
this.lastStdError = chunk;
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
return new Promise((resolve, reject) => {
|
|
235
|
+
this.child.on("error", reject).on("close", (exitCode) => {
|
|
236
|
+
const result = this.resolveExitCode(exitCode);
|
|
237
|
+
typeof result === "number" ? resolve(result) : reject(result);
|
|
238
|
+
});
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
exports.AsyncProcess = AsyncProcess;
|
package/lib/utils/async.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
};
|
|
4
|
-
type ItemBuffer<T> = Map<T, ControllerItem>;
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
type ItemBuffer<T> = Map<T, AbortController>;
|
|
5
3
|
export declare function runParallel<T>(options: {
|
|
6
4
|
items: T[];
|
|
7
5
|
concurrency: number;
|
|
@@ -13,7 +11,7 @@ export declare function runParallel<T>(options: {
|
|
|
13
11
|
onItem: (data: {
|
|
14
12
|
item: T;
|
|
15
13
|
index: number;
|
|
16
|
-
controller:
|
|
14
|
+
controller: AbortController;
|
|
17
15
|
}) => Promise<void> | void;
|
|
18
16
|
onFinished?: () => Promise<void> | void;
|
|
19
17
|
}): Promise<void>;
|