@datatruck/cli 0.39.0 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +51 -16
- package/lib/commands/RunCommand.js +2 -3
- package/lib/tasks/MariadbTask.js +1 -1
- package/lib/tasks/MongoDumpTask.d.ts +9 -7
- package/lib/tasks/MongoDumpTask.js +62 -11
- package/lib/tasks/MssqlTask.js +1 -1
- package/lib/tasks/MysqlDumpTask.js +1 -1
- package/lib/tasks/SqlDumpTaskAbstract.js +1 -1
- package/lib/utils/datatruck/cron-server.js +1 -1
- package/lib/utils/datatruck/job.d.ts +4 -2
- package/lib/utils/datatruck/job.js +57 -42
- package/lib/utils/fs.d.ts +2 -1
- package/lib/utils/fs.js +16 -7
- package/lib/utils/git.js +1 -1
- package/lib/utils/logs.js +2 -1
- package/lib/utils/mongodb.d.ts +11 -0
- package/lib/utils/mongodb.js +39 -0
- package/package.json +6 -4
package/config.schema.json
CHANGED
|
@@ -409,30 +409,47 @@
|
|
|
409
409
|
"config": {
|
|
410
410
|
"type": "object",
|
|
411
411
|
"properties": {
|
|
412
|
-
"
|
|
413
|
-
"type": "string"
|
|
414
|
-
},
|
|
415
|
-
"hostname": {
|
|
416
|
-
"type": "string"
|
|
417
|
-
},
|
|
418
|
-
"port": {
|
|
419
|
-
"type": "number"
|
|
420
|
-
},
|
|
421
|
-
"username": {
|
|
422
|
-
"type": "string"
|
|
423
|
-
},
|
|
424
|
-
"password": {
|
|
412
|
+
"uri": {
|
|
425
413
|
"anyOf": [
|
|
426
414
|
{
|
|
427
415
|
"type": "object",
|
|
428
416
|
"properties": {
|
|
429
|
-
"
|
|
417
|
+
"host": {
|
|
418
|
+
"type": "string"
|
|
419
|
+
},
|
|
420
|
+
"username": {
|
|
421
|
+
"type": "string"
|
|
422
|
+
},
|
|
423
|
+
"password": {
|
|
424
|
+
"anyOf": [
|
|
425
|
+
{
|
|
426
|
+
"type": "object",
|
|
427
|
+
"properties": {
|
|
428
|
+
"path": {
|
|
429
|
+
"type": "string"
|
|
430
|
+
}
|
|
431
|
+
},
|
|
432
|
+
"additionalProperties": false,
|
|
433
|
+
"required": [
|
|
434
|
+
"path"
|
|
435
|
+
]
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
"type": "string"
|
|
439
|
+
}
|
|
440
|
+
]
|
|
441
|
+
},
|
|
442
|
+
"port": {
|
|
443
|
+
"type": "number"
|
|
444
|
+
},
|
|
445
|
+
"database": {
|
|
430
446
|
"type": "string"
|
|
431
447
|
}
|
|
432
448
|
},
|
|
433
449
|
"additionalProperties": false,
|
|
434
450
|
"required": [
|
|
435
|
-
"
|
|
451
|
+
"database",
|
|
452
|
+
"host"
|
|
436
453
|
]
|
|
437
454
|
},
|
|
438
455
|
{
|
|
@@ -440,14 +457,32 @@
|
|
|
440
457
|
}
|
|
441
458
|
]
|
|
442
459
|
},
|
|
460
|
+
"command": {
|
|
461
|
+
"type": "string"
|
|
462
|
+
},
|
|
443
463
|
"compress": {
|
|
444
464
|
"type": "boolean"
|
|
445
465
|
},
|
|
446
466
|
"concurrency": {
|
|
447
467
|
"type": "number"
|
|
468
|
+
},
|
|
469
|
+
"targetDatabase": {
|
|
470
|
+
"type": "object",
|
|
471
|
+
"properties": {
|
|
472
|
+
"name": {
|
|
473
|
+
"type": "string"
|
|
474
|
+
}
|
|
475
|
+
},
|
|
476
|
+
"additionalProperties": false,
|
|
477
|
+
"required": [
|
|
478
|
+
"name"
|
|
479
|
+
]
|
|
448
480
|
}
|
|
449
481
|
},
|
|
450
|
-
"additionalProperties": false
|
|
482
|
+
"additionalProperties": false,
|
|
483
|
+
"required": [
|
|
484
|
+
"uri"
|
|
485
|
+
]
|
|
451
486
|
}
|
|
452
487
|
},
|
|
453
488
|
"additionalProperties": false,
|
|
@@ -31,12 +31,11 @@ class RunCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
31
31
|
console.error(`error: missing required argument 'jobName' (values: ${jobNames.join(", ")})`);
|
|
32
32
|
return { exitCode: 1 };
|
|
33
33
|
}
|
|
34
|
-
await (0, job_1.runJob)(job, jobName, {
|
|
34
|
+
const exitCode = await (0, job_1.runJob)(job, jobName, {
|
|
35
35
|
verbose: verbose,
|
|
36
36
|
configPath: config.path,
|
|
37
|
-
log: "inherit",
|
|
38
37
|
});
|
|
39
|
-
return { exitCode
|
|
38
|
+
return { exitCode };
|
|
40
39
|
}
|
|
41
40
|
}
|
|
42
41
|
exports.RunCommand = RunCommand;
|
package/lib/tasks/MariadbTask.js
CHANGED
|
@@ -160,7 +160,7 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
160
160
|
const reloadFiles = async (data = {}) => {
|
|
161
161
|
if (data.removeFile)
|
|
162
162
|
removeFiles.push(data.removeFile);
|
|
163
|
-
return (files = (await (0, fs_1.
|
|
163
|
+
return (files = (await (0, fs_1.safeReaddir)(snapshotPath)).filter((v) => !removeFiles.includes(v)));
|
|
164
164
|
};
|
|
165
165
|
await reloadFiles();
|
|
166
166
|
const absolute = {
|
|
@@ -1,14 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { MongoUriObject } from "../utils/mongodb";
|
|
2
|
+
import { TaskBackupData, TaskRestoreData, TaskAbstract, TaskPrepareRestoreData } from "./TaskAbstract";
|
|
2
3
|
export type MongoDumpTaskConfig = {
|
|
4
|
+
uri: string | MongoUriObject;
|
|
3
5
|
command?: string;
|
|
4
|
-
hostname?: string;
|
|
5
|
-
port?: number;
|
|
6
|
-
username?: string;
|
|
7
|
-
password?: string | {
|
|
8
|
-
path: string;
|
|
9
|
-
};
|
|
10
6
|
compress?: boolean;
|
|
11
7
|
concurrency?: number;
|
|
8
|
+
targetDatabase?: {
|
|
9
|
+
name: string;
|
|
10
|
+
};
|
|
12
11
|
};
|
|
13
12
|
export declare const mongodumpTaskName = "mongo-dump";
|
|
14
13
|
export declare class MongoDumpTask extends TaskAbstract<MongoDumpTaskConfig> {
|
|
@@ -17,5 +16,8 @@ export declare class MongoDumpTask extends TaskAbstract<MongoDumpTaskConfig> {
|
|
|
17
16
|
backup(data: TaskBackupData): Promise<{
|
|
18
17
|
snapshotPath: string;
|
|
19
18
|
}>;
|
|
19
|
+
prepareRestore(data: TaskPrepareRestoreData): Promise<{
|
|
20
|
+
snapshotPath: string;
|
|
21
|
+
}>;
|
|
20
22
|
restore(data: TaskRestoreData): Promise<void>;
|
|
21
23
|
}
|
|
@@ -2,9 +2,15 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MongoDumpTask = exports.mongodumpTaskName = void 0;
|
|
4
4
|
const async_process_1 = require("../utils/async-process");
|
|
5
|
+
const config_1 = require("../utils/datatruck/config");
|
|
6
|
+
const error_1 = require("../utils/error");
|
|
5
7
|
const fs_1 = require("../utils/fs");
|
|
8
|
+
const mongodb_1 = require("../utils/mongodb");
|
|
6
9
|
const temp_1 = require("../utils/temp");
|
|
7
10
|
const TaskAbstract_1 = require("./TaskAbstract");
|
|
11
|
+
const promises_1 = require("fs/promises");
|
|
12
|
+
const mongodb_2 = require("mongodb");
|
|
13
|
+
const path_1 = require("path");
|
|
8
14
|
exports.mongodumpTaskName = "mongo-dump";
|
|
9
15
|
class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
10
16
|
verbose;
|
|
@@ -17,21 +23,19 @@ class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
17
23
|
(await (0, temp_1.mkTmpDir)(exports.mongodumpTaskName, "task", "backup", "snapshot"));
|
|
18
24
|
await (0, fs_1.mkdirIfNotExists)(snapshotPath);
|
|
19
25
|
await (0, fs_1.ensureEmptyDir)(snapshotPath);
|
|
26
|
+
const config = await (0, mongodb_1.resolveMongoUri)(this.config.uri);
|
|
20
27
|
const p = new async_process_1.AsyncProcess(this.command, [
|
|
21
|
-
...(
|
|
22
|
-
...(
|
|
23
|
-
...
|
|
28
|
+
...(config.host ? ["/h", config.host] : []),
|
|
29
|
+
...(config.port ? [`/port:${config.port}`] : []),
|
|
30
|
+
...["/authenticationDatabase:admin"],
|
|
31
|
+
...["/d", config.database],
|
|
32
|
+
...(config.username ? ["/u", config.username] : []),
|
|
24
33
|
...(this.config.compress ? ["/gzip"] : []),
|
|
25
34
|
...(this.config.concurrency ? ["/j", this.config.concurrency] : []),
|
|
26
35
|
"/o",
|
|
27
36
|
snapshotPath,
|
|
28
|
-
], {
|
|
29
|
-
|
|
30
|
-
});
|
|
31
|
-
const password = this.config.password !== undefined
|
|
32
|
-
? (await (0, fs_1.fetchData)(this.config.password, (p) => p.path)) ?? ""
|
|
33
|
-
: "";
|
|
34
|
-
p.stdin.writable.write(`${password}\n`);
|
|
37
|
+
], { $log: this.verbose });
|
|
38
|
+
p.stdin.writable.write(`${config.password ?? ""}\n`);
|
|
35
39
|
await p.stderr.parseLines((line) => {
|
|
36
40
|
data.onProgress({
|
|
37
41
|
absolute: {
|
|
@@ -39,10 +43,57 @@ class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
39
43
|
},
|
|
40
44
|
});
|
|
41
45
|
});
|
|
46
|
+
const tmpDir = (0, path_1.join)(snapshotPath, config.database);
|
|
47
|
+
for (const file of await (0, promises_1.readdir)(tmpDir))
|
|
48
|
+
await (0, promises_1.rename)((0, path_1.join)(tmpDir, file), (0, path_1.join)(snapshotPath, file));
|
|
49
|
+
await (0, promises_1.rmdir)(tmpDir);
|
|
42
50
|
return { snapshotPath };
|
|
43
51
|
}
|
|
52
|
+
async prepareRestore(data) {
|
|
53
|
+
return {
|
|
54
|
+
snapshotPath: data.package.restorePath ??
|
|
55
|
+
(await (0, temp_1.mkTmpDir)(exports.mongodumpTaskName, "task", "restore", "snapshot")),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
44
58
|
async restore(data) {
|
|
45
|
-
|
|
59
|
+
this.verbose = data.options.verbose;
|
|
60
|
+
const config = await (0, mongodb_1.resolveMongoUri)(this.config.uri);
|
|
61
|
+
const uri = (0, mongodb_1.toMongoUri)(config);
|
|
62
|
+
const client = new mongodb_2.MongoClient(`${uri}?authSource=admin`);
|
|
63
|
+
const params = {
|
|
64
|
+
packageName: data.package.name,
|
|
65
|
+
snapshotId: data.options.id,
|
|
66
|
+
snapshotDate: data.snapshot.date,
|
|
67
|
+
action: "restore",
|
|
68
|
+
database: undefined,
|
|
69
|
+
};
|
|
70
|
+
const database = {
|
|
71
|
+
name: (0, config_1.resolveDatabaseName)(config.database, params),
|
|
72
|
+
};
|
|
73
|
+
if (this.config.targetDatabase && !data.options.initial)
|
|
74
|
+
database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
|
|
75
|
+
...params,
|
|
76
|
+
database: database.name,
|
|
77
|
+
});
|
|
78
|
+
const restoreCollections = (await (0, promises_1.readdir)(data.snapshotPath))
|
|
79
|
+
.filter((name) => name.endsWith(".bson"))
|
|
80
|
+
.map((name) => name.replace(/\.bson$/, ""));
|
|
81
|
+
const collections = (await client.db(database.name).collections()).map((v) => v.collectionName);
|
|
82
|
+
const duplicatedCollections = restoreCollections.filter((v) => collections.includes(v));
|
|
83
|
+
if (duplicatedCollections.length)
|
|
84
|
+
throw new error_1.AppError(`Target collections already exists: ${duplicatedCollections.join(", ")}`);
|
|
85
|
+
const p = new async_process_1.AsyncProcess("mongorestore", [
|
|
86
|
+
...(config.host ? ["/h", config.host] : []),
|
|
87
|
+
...(config.port ? [`/port:${config.port}`] : []),
|
|
88
|
+
...["/authenticationDatabase:admin"],
|
|
89
|
+
...["/d", database.name],
|
|
90
|
+
...(config.username ? ["/u", config.username] : []),
|
|
91
|
+
...(this.config.compress ? ["/gzip"] : []),
|
|
92
|
+
...(this.config.concurrency ? ["/j", this.config.concurrency] : []),
|
|
93
|
+
data.snapshotPath,
|
|
94
|
+
], { $log: this.verbose });
|
|
95
|
+
p.stdin.writable.write(`${config.password ?? ""}\n`);
|
|
96
|
+
await p.waitForClose();
|
|
46
97
|
}
|
|
47
98
|
}
|
|
48
99
|
exports.MongoDumpTask = MongoDumpTask;
|
package/lib/tasks/MssqlTask.js
CHANGED
|
@@ -66,7 +66,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
66
66
|
async restore(data) {
|
|
67
67
|
this.verbose = data.options.verbose;
|
|
68
68
|
const snapshotPath = data.snapshotPath;
|
|
69
|
-
const files = await (0, fs_1.
|
|
69
|
+
const files = await (0, fs_1.safeReaddir)(snapshotPath);
|
|
70
70
|
for (const file of files) {
|
|
71
71
|
if (!file.endsWith(MssqlTask.SUFFIX))
|
|
72
72
|
continue;
|
|
@@ -255,7 +255,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
255
255
|
...params,
|
|
256
256
|
database: database.name,
|
|
257
257
|
});
|
|
258
|
-
const [files, compressed] = (0, fs_1.groupFiles)(await (0, fs_1.
|
|
258
|
+
const [files, compressed] = (0, fs_1.groupFiles)(await (0, fs_1.safeReaddir)(snapshotPath), Object.values(suffix));
|
|
259
259
|
// Database check
|
|
260
260
|
if (files.some((f) => f.endsWith(suffix.database)) &&
|
|
261
261
|
!(await sql.isDatabaseEmpty(database.name)))
|
|
@@ -168,7 +168,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
|
|
|
168
168
|
database: database.name,
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
-
const items = (await (0, fs_1.
|
|
171
|
+
const items = (await (0, fs_1.safeReaddir)(snapshotPath))
|
|
172
172
|
.map(parseSqlFile)
|
|
173
173
|
.filter((v) => !!v);
|
|
174
174
|
// Database check
|
|
@@ -34,8 +34,10 @@ export type Job = JobAction & {
|
|
|
34
34
|
schedule?: JobSchedule;
|
|
35
35
|
};
|
|
36
36
|
export type JobConfig = {
|
|
37
|
-
log: DatatruckCronServerOptions["log"]
|
|
37
|
+
log: DatatruckCronServerOptions["log"];
|
|
38
38
|
verbose: boolean;
|
|
39
39
|
configPath: string;
|
|
40
40
|
};
|
|
41
|
-
export declare function
|
|
41
|
+
export declare function getJobCliOptions(job: Job): string[];
|
|
42
|
+
export declare function runJob(job: Job, name: string, config: Omit<JobConfig, "log">): Promise<number>;
|
|
43
|
+
export declare function runCronJob(job: Job, name: string, config: JobConfig): Promise<void>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.runJob = void 0;
|
|
3
|
+
exports.runCronJob = exports.runJob = exports.getJobCliOptions = void 0;
|
|
4
4
|
const async_process_1 = require("../async-process");
|
|
5
5
|
const cli_1 = require("../cli");
|
|
6
6
|
const fs_1 = require("../fs");
|
|
@@ -11,32 +11,59 @@ const fs_2 = require("fs");
|
|
|
11
11
|
const promises_1 = require("fs/promises");
|
|
12
12
|
const path_1 = require("path");
|
|
13
13
|
async function createJobLog(config, name) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
14
|
+
const dt = new Date().toISOString().replaceAll(":", "-");
|
|
15
|
+
const dir = config?.path ?? logs_1.defaultsLogPath;
|
|
16
|
+
const tmpPath = (0, path_1.join)(dir, `${dt}_${name}.tmp.log`);
|
|
17
|
+
await (0, promises_1.mkdir)(dir, { recursive: true });
|
|
18
|
+
const stream = (0, fs_2.createWriteStream)(tmpPath);
|
|
19
|
+
return {
|
|
20
|
+
stream,
|
|
21
|
+
write(data) {
|
|
22
|
+
stream.write(data);
|
|
23
|
+
},
|
|
24
|
+
async finish(pid) {
|
|
25
|
+
const path = (0, path_1.join)(dir, `${dt}_${name}_${pid}.log`);
|
|
26
|
+
await (0, fs_1.safeRename)(tmpPath, path);
|
|
27
|
+
return path;
|
|
28
|
+
},
|
|
29
|
+
};
|
|
31
30
|
}
|
|
31
|
+
function getJobCliOptions(job) {
|
|
32
|
+
const Command = command_1.datatruckCommands[job.action];
|
|
33
|
+
const command = new Command({ config: { packages: [], repositories: [] } }, {});
|
|
34
|
+
return (0, options_1.stringifyOptions)(command.optionsConfig, job.action === "prune"
|
|
35
|
+
? ({ ...job.options, confirm: true })
|
|
36
|
+
: job.options);
|
|
37
|
+
}
|
|
38
|
+
exports.getJobCliOptions = getJobCliOptions;
|
|
32
39
|
async function runJob(job, name, config) {
|
|
40
|
+
const cliOptions = getJobCliOptions(job);
|
|
41
|
+
const [node, bin] = process.argv;
|
|
42
|
+
return await async_process_1.AsyncProcess.exec(node, [
|
|
43
|
+
process.env.DTT_BIN_SCRIPT ?? process.env.pm_exec_path ?? bin,
|
|
44
|
+
"-c",
|
|
45
|
+
config.configPath,
|
|
46
|
+
job.action,
|
|
47
|
+
...cliOptions,
|
|
48
|
+
...(config.verbose ? ["-v"] : []),
|
|
49
|
+
], {
|
|
50
|
+
$exitCode: false,
|
|
51
|
+
$log: { exec: config.verbose },
|
|
52
|
+
env: {
|
|
53
|
+
...process.env,
|
|
54
|
+
JOB_NAME: name,
|
|
55
|
+
},
|
|
56
|
+
stdio: "inherit",
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
exports.runJob = runJob;
|
|
60
|
+
async function runCronJob(job, name, config) {
|
|
33
61
|
let pid = 0;
|
|
34
62
|
try {
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
: job.options);
|
|
63
|
+
const log = config.log?.enabled ?? true
|
|
64
|
+
? await createJobLog(config.log, name)
|
|
65
|
+
: undefined;
|
|
66
|
+
const cliOptions = getJobCliOptions(job);
|
|
40
67
|
const [node, bin] = process.argv;
|
|
41
68
|
const argv = [
|
|
42
69
|
process.env.DTT_BIN_SCRIPT ?? process.env.pm_exec_path ?? bin,
|
|
@@ -49,41 +76,29 @@ async function runJob(job, name, config) {
|
|
|
49
76
|
job.action,
|
|
50
77
|
...cliOptions,
|
|
51
78
|
];
|
|
52
|
-
|
|
53
|
-
if (log?.type === "file")
|
|
54
|
-
log.stream.write(`+ dtt ${argv.join(" ")}\n`);
|
|
79
|
+
log?.write(`+ dtt ${argv.slice(1).join(" ")}\n`);
|
|
55
80
|
const p = new async_process_1.AsyncProcess(node, argv, {
|
|
56
81
|
$log: config.verbose,
|
|
57
82
|
$exitCode: false,
|
|
58
83
|
env: {
|
|
59
84
|
...process.env,
|
|
85
|
+
JOB_NAME: name,
|
|
60
86
|
COLUMNS: "160",
|
|
61
87
|
NO_COLOR: "1",
|
|
62
|
-
JOB_NAME: name,
|
|
63
88
|
},
|
|
64
|
-
...(log?.type === "inherit" && {
|
|
65
|
-
stdio: "inherit",
|
|
66
|
-
}),
|
|
67
89
|
});
|
|
68
90
|
pid = p.child.pid || 0;
|
|
69
91
|
(0, cli_1.logJson)("job", `'${name}' started`, { pid });
|
|
70
92
|
const [exitCode] = await Promise.all([
|
|
71
93
|
p.waitForClose(),
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
: []),
|
|
94
|
+
log && p.stderr.pipe(log.stream),
|
|
95
|
+
log && p.stdout.pipe(log.stream),
|
|
75
96
|
]);
|
|
76
|
-
|
|
77
|
-
if (log?.stream) {
|
|
78
|
-
const base = (0, path_1.dirname)(log?.stream.path.toString());
|
|
79
|
-
const logPath = (0, path_1.join)(base, `${log.dt}_${name}_${pid}.log`);
|
|
80
|
-
await (0, fs_1.safeRename)(log?.stream.path.toString(), logPath);
|
|
81
|
-
logData["log"] = logPath;
|
|
82
|
-
}
|
|
97
|
+
const logPath = await log?.finish(pid);
|
|
83
98
|
(0, cli_1.logJson)("job", `'${name}' finished`, {
|
|
84
99
|
pid,
|
|
85
100
|
exitCode,
|
|
86
|
-
...
|
|
101
|
+
...(logPath && { log: logPath }),
|
|
87
102
|
});
|
|
88
103
|
}
|
|
89
104
|
catch (error) {
|
|
@@ -91,4 +106,4 @@ async function runJob(job, name, config) {
|
|
|
91
106
|
console.error(error);
|
|
92
107
|
}
|
|
93
108
|
}
|
|
94
|
-
exports.
|
|
109
|
+
exports.runCronJob = runCronJob;
|
package/lib/utils/fs.d.ts
CHANGED
|
@@ -28,7 +28,8 @@ export declare function parsePackageFile(): {
|
|
|
28
28
|
export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
|
|
29
29
|
export declare function fastFolderSizeAsync(path: string): Promise<number>;
|
|
30
30
|
export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
|
|
31
|
-
export declare function
|
|
31
|
+
export declare function safeReaddir(path: string): Promise<string[]>;
|
|
32
|
+
export declare function tryReaddir(path: string): Promise<string[]>;
|
|
32
33
|
export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
|
|
33
34
|
/**
|
|
34
35
|
* @experimental
|
package/lib/utils/fs.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.asFile = exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.readTextFile = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.
|
|
6
|
+
exports.asFile = exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.readTextFile = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.tryReaddir = exports.safeReaddir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.include = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
|
|
7
7
|
const pkg_1 = require("../pkg");
|
|
8
8
|
const bytes_1 = require("./bytes");
|
|
9
9
|
const error_1 = require("./error");
|
|
@@ -58,7 +58,7 @@ async function ensureEmptyDir(path) {
|
|
|
58
58
|
}
|
|
59
59
|
exports.ensureEmptyDir = ensureEmptyDir;
|
|
60
60
|
async function ensureSingleFile(path) {
|
|
61
|
-
const files = await
|
|
61
|
+
const files = await safeReaddir(path);
|
|
62
62
|
if (files.length !== 1)
|
|
63
63
|
throw new error_1.AppError(`Dir has not one file: ${files.length}`);
|
|
64
64
|
const [file] = files;
|
|
@@ -171,15 +171,13 @@ async function readPartialFile(path, positions) {
|
|
|
171
171
|
});
|
|
172
172
|
}
|
|
173
173
|
exports.readPartialFile = readPartialFile;
|
|
174
|
-
async function
|
|
174
|
+
async function safeReaddir(path) {
|
|
175
175
|
try {
|
|
176
176
|
return await (0, promises_1.readdir)(path);
|
|
177
177
|
}
|
|
178
178
|
catch (anyError) {
|
|
179
179
|
const nodeError = anyError;
|
|
180
180
|
if (nodeError.code === "ENOENT") {
|
|
181
|
-
if (optional)
|
|
182
|
-
return [];
|
|
183
181
|
const error = new Error(nodeError.message);
|
|
184
182
|
error.code = nodeError.code;
|
|
185
183
|
error.errno = nodeError.errno;
|
|
@@ -189,9 +187,20 @@ async function readDir(path, optional) {
|
|
|
189
187
|
throw anyError;
|
|
190
188
|
}
|
|
191
189
|
}
|
|
192
|
-
exports.
|
|
190
|
+
exports.safeReaddir = safeReaddir;
|
|
191
|
+
async function tryReaddir(path) {
|
|
192
|
+
try {
|
|
193
|
+
return await safeReaddir(path);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (error.code === "ENOENT")
|
|
197
|
+
return [];
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
exports.tryReaddir = tryReaddir;
|
|
193
202
|
async function forEachFile(dirPath, cb, includeDir) {
|
|
194
|
-
const files = await
|
|
203
|
+
const files = await safeReaddir(dirPath);
|
|
195
204
|
for (const file of files) {
|
|
196
205
|
const filePath = (0, path_1.join)(dirPath, file);
|
|
197
206
|
if ((await (0, promises_1.stat)(filePath)).isDirectory()) {
|
package/lib/utils/git.js
CHANGED
|
@@ -23,7 +23,7 @@ class Git {
|
|
|
23
23
|
}
|
|
24
24
|
async canBeInit(repo) {
|
|
25
25
|
return ((0, fs_1.isLocalDir)(repo) &&
|
|
26
|
-
(!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.
|
|
26
|
+
(!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.safeReaddir)(repo)).length));
|
|
27
27
|
}
|
|
28
28
|
async clone(options) {
|
|
29
29
|
await this.exec([
|
package/lib/utils/logs.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.removeOldLogs = exports.defaultsLogPath = exports.maxAgeToMS = void 0;
|
|
4
|
+
const fs_1 = require("./fs");
|
|
4
5
|
const promises_1 = require("fs/promises");
|
|
5
6
|
const os_1 = require("os");
|
|
6
7
|
const path_1 = require("path");
|
|
@@ -29,7 +30,7 @@ async function removeOldLogs(path, inMaxAge) {
|
|
|
29
30
|
const hasMaxAge = Object.values(inMaxAge).some((v) => typeof v === "number");
|
|
30
31
|
if (!hasMaxAge)
|
|
31
32
|
return [];
|
|
32
|
-
const files = (await (0,
|
|
33
|
+
const files = (await (0, fs_1.tryReaddir)(path)).filter((file) => file.endsWith(".log"));
|
|
33
34
|
const now = Date.now();
|
|
34
35
|
const paths = [];
|
|
35
36
|
for (const file of files) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type MongoUriObject<Resolved = false> = {
|
|
2
|
+
host: string;
|
|
3
|
+
username?: string;
|
|
4
|
+
password?: [Resolved] extends [true] ? string : string | {
|
|
5
|
+
path: string;
|
|
6
|
+
};
|
|
7
|
+
port?: number;
|
|
8
|
+
database: string;
|
|
9
|
+
};
|
|
10
|
+
export declare function toMongoUri(object: MongoUriObject<true>): string;
|
|
11
|
+
export declare function resolveMongoUri(input: string | MongoUriObject): Promise<MongoUriObject<true>>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resolveMongoUri = exports.toMongoUri = void 0;
|
|
4
|
+
const fs_1 = require("./fs");
|
|
5
|
+
function toMongoUri(object) {
|
|
6
|
+
const url = new URL(`mongodb://${object.host}`);
|
|
7
|
+
if (typeof object.username === "string")
|
|
8
|
+
url.username = object.username;
|
|
9
|
+
if (typeof object.password === "string")
|
|
10
|
+
url.password = object.password;
|
|
11
|
+
if (typeof object.port === "number")
|
|
12
|
+
url.port = object.port.toString();
|
|
13
|
+
url.pathname = `/${object.database}`;
|
|
14
|
+
return url.href;
|
|
15
|
+
}
|
|
16
|
+
exports.toMongoUri = toMongoUri;
|
|
17
|
+
async function resolveMongoUri(input) {
|
|
18
|
+
let object;
|
|
19
|
+
if (typeof input === "string") {
|
|
20
|
+
const url = new URL(input);
|
|
21
|
+
object = {
|
|
22
|
+
host: url.hostname,
|
|
23
|
+
password: url.password,
|
|
24
|
+
port: url.port ? Number(url.port) : undefined,
|
|
25
|
+
username: url.username,
|
|
26
|
+
database: url.pathname.slice(1),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
object = input;
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
...object,
|
|
34
|
+
password: object.password !== undefined
|
|
35
|
+
? (await (0, fs_1.fetchData)(object.password, (p) => p.path)) ?? ""
|
|
36
|
+
: "",
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
exports.resolveMongoUri = resolveMongoUri;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@datatruck/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.40.0",
|
|
4
4
|
"description": "Tool for creating and managing backups",
|
|
5
5
|
"homepage": "https://github.com/swordev/datatruck#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -31,19 +31,21 @@
|
|
|
31
31
|
"async": "^3.2.5",
|
|
32
32
|
"chalk": "^4.1.2",
|
|
33
33
|
"commander": "^12.0.0",
|
|
34
|
-
"croner": "^8.0.
|
|
34
|
+
"croner": "^8.0.2",
|
|
35
35
|
"dayjs": "^1.11.10",
|
|
36
36
|
"fast-folder-size": "^2.2.0",
|
|
37
37
|
"fast-glob": "^3.3.2",
|
|
38
38
|
"listr2": "^8.2.1",
|
|
39
39
|
"micromatch": "^4.0.5",
|
|
40
|
-
"
|
|
40
|
+
"mongodb": "^6.5.0",
|
|
41
|
+
"mysql2": "^3.9.4",
|
|
41
42
|
"tty-table": "^4.2.3",
|
|
42
43
|
"yaml": "^2.4.1"
|
|
43
44
|
},
|
|
44
45
|
"devDependencies": {
|
|
45
46
|
"@types/async": "^3.2.24",
|
|
46
|
-
"@types/micromatch": "^4.0.6"
|
|
47
|
+
"@types/micromatch": "^4.0.6",
|
|
48
|
+
"mongodb-memory-server": "^9.1.8"
|
|
47
49
|
},
|
|
48
50
|
"optionalDependencies": {
|
|
49
51
|
"ts-node": "^10.9.2"
|