@datatruck/cli 0.38.2 → 0.39.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.schema.json +38 -9
- package/lib/commands/RunCommand.js +2 -5
- package/lib/commands/StartServerCommand.js +1 -6
- package/lib/tasks/MariadbTask.js +1 -1
- package/lib/tasks/MssqlTask.js +1 -1
- package/lib/tasks/MysqlDumpTask.js +1 -1
- package/lib/tasks/SqlDumpTaskAbstract.js +1 -1
- package/lib/utils/datatruck/config-type.d.ts +0 -1
- package/lib/utils/datatruck/cron-server.d.ts +19 -4
- package/lib/utils/datatruck/cron-server.js +37 -17
- package/lib/utils/datatruck/job.d.ts +5 -3
- package/lib/utils/datatruck/job.js +61 -37
- package/lib/utils/datatruck/repository-server.d.ts +0 -1
- package/lib/utils/datatruck/repository-server.js +4 -8
- package/lib/utils/fs.d.ts +2 -1
- package/lib/utils/fs.js +16 -7
- package/lib/utils/git.js +1 -1
- package/lib/utils/logs.d.ts +8 -0
- package/lib/utils/logs.js +47 -0
- package/lib/utils/mysql.d.ts +5 -5
- package/lib/utils/options.js +4 -4
- package/package.json +2 -2
package/config.schema.json
CHANGED
|
@@ -645,9 +645,6 @@
|
|
|
645
645
|
"server": {
|
|
646
646
|
"type": "object",
|
|
647
647
|
"properties": {
|
|
648
|
-
"log": {
|
|
649
|
-
"type": "boolean"
|
|
650
|
-
},
|
|
651
648
|
"repository": {
|
|
652
649
|
"type": "object",
|
|
653
650
|
"properties": {
|
|
@@ -757,14 +754,46 @@
|
|
|
757
754
|
"type": "object",
|
|
758
755
|
"properties": {
|
|
759
756
|
"enabled": {
|
|
757
|
+
"default": true,
|
|
760
758
|
"type": "boolean"
|
|
761
759
|
},
|
|
762
|
-
"
|
|
763
|
-
"
|
|
764
|
-
"
|
|
765
|
-
"
|
|
766
|
-
|
|
767
|
-
|
|
760
|
+
"log": {
|
|
761
|
+
"type": "object",
|
|
762
|
+
"properties": {
|
|
763
|
+
"enabled": {
|
|
764
|
+
"default": true,
|
|
765
|
+
"type": "boolean"
|
|
766
|
+
},
|
|
767
|
+
"path": {
|
|
768
|
+
"default": "/var/logs/datatruck",
|
|
769
|
+
"type": "string"
|
|
770
|
+
},
|
|
771
|
+
"rotate": {
|
|
772
|
+
"type": "object",
|
|
773
|
+
"properties": {
|
|
774
|
+
"maxAge": {
|
|
775
|
+
"default": {
|
|
776
|
+
"days": 14
|
|
777
|
+
},
|
|
778
|
+
"type": "object",
|
|
779
|
+
"properties": {
|
|
780
|
+
"days": {
|
|
781
|
+
"type": "number"
|
|
782
|
+
},
|
|
783
|
+
"hours": {
|
|
784
|
+
"type": "number"
|
|
785
|
+
},
|
|
786
|
+
"minutes": {
|
|
787
|
+
"type": "number"
|
|
788
|
+
}
|
|
789
|
+
},
|
|
790
|
+
"additionalProperties": false
|
|
791
|
+
}
|
|
792
|
+
},
|
|
793
|
+
"additionalProperties": false
|
|
794
|
+
}
|
|
795
|
+
},
|
|
796
|
+
"additionalProperties": false
|
|
768
797
|
}
|
|
769
798
|
},
|
|
770
799
|
"additionalProperties": false
|
|
@@ -23,7 +23,6 @@ class RunCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
23
23
|
async exec() {
|
|
24
24
|
const config = await ConfigAction_1.ConfigAction.fromGlobalOptionsWithPath(this.globalOptions);
|
|
25
25
|
const verbose = !!this.globalOptions.verbose;
|
|
26
|
-
const log = config.data.server?.log ?? true;
|
|
27
26
|
const jobs = config.data.jobs || {};
|
|
28
27
|
const jobName = this.options.jobName;
|
|
29
28
|
const job = jobName ? jobs[jobName] : undefined;
|
|
@@ -32,13 +31,11 @@ class RunCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
32
31
|
console.error(`error: missing required argument 'jobName' (values: ${jobNames.join(", ")})`);
|
|
33
32
|
return { exitCode: 1 };
|
|
34
33
|
}
|
|
35
|
-
await (0, job_1.runJob)(job, jobName, {
|
|
36
|
-
log,
|
|
34
|
+
const exitCode = await (0, job_1.runJob)(job, jobName, {
|
|
37
35
|
verbose: verbose,
|
|
38
36
|
configPath: config.path,
|
|
39
|
-
logPath: config.data.server?.cron?.logPath,
|
|
40
37
|
});
|
|
41
|
-
return { exitCode
|
|
38
|
+
return { exitCode };
|
|
42
39
|
}
|
|
43
40
|
}
|
|
44
41
|
exports.RunCommand = RunCommand;
|
|
@@ -23,12 +23,10 @@ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
23
23
|
const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
|
|
24
24
|
const configPath = this.configPath;
|
|
25
25
|
const verbose = !!this.globalOptions.verbose;
|
|
26
|
-
const log = config.server?.log ?? true;
|
|
27
26
|
const repositoryOptions = config.server?.repository || {};
|
|
28
27
|
if (repositoryOptions.enabled ?? true) {
|
|
29
28
|
const server = (0, repository_server_1.createDatatruckRepositoryServer)(repositoryOptions, {
|
|
30
29
|
configPath,
|
|
31
|
-
log,
|
|
32
30
|
});
|
|
33
31
|
const port = repositoryOptions.listen?.port ?? 8888;
|
|
34
32
|
const address = repositoryOptions.listen?.address ?? "127.0.0.1";
|
|
@@ -40,18 +38,15 @@ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
|
|
|
40
38
|
server.listen(port, address);
|
|
41
39
|
}
|
|
42
40
|
const cronOptions = config.server?.cron || {};
|
|
43
|
-
const logPath = config.server?.cron?.logPath;
|
|
44
41
|
if (cronOptions.enabled ?? true) {
|
|
45
42
|
if (typeof configPath !== "string")
|
|
46
43
|
throw new error_1.AppError(`Config path is required by cron server`);
|
|
47
44
|
const server = await (0, cron_server_1.createCronServer)({
|
|
48
45
|
configPath,
|
|
49
46
|
verbose,
|
|
50
|
-
log,
|
|
51
|
-
logPath,
|
|
47
|
+
log: config.server?.cron?.log,
|
|
52
48
|
});
|
|
53
49
|
server.start();
|
|
54
|
-
(0, cli_1.logJson)("cron-server", `server started`);
|
|
55
50
|
}
|
|
56
51
|
const exitCode = await new Promise((resolve) => {
|
|
57
52
|
process.on("SIGINT", () => resolve(1)).on("SIGTERM", () => resolve(1));
|
package/lib/tasks/MariadbTask.js
CHANGED
|
@@ -160,7 +160,7 @@ class MariadbTask extends TaskAbstract_1.TaskAbstract {
|
|
|
160
160
|
const reloadFiles = async (data = {}) => {
|
|
161
161
|
if (data.removeFile)
|
|
162
162
|
removeFiles.push(data.removeFile);
|
|
163
|
-
return (files = (await (0, fs_1.
|
|
163
|
+
return (files = (await (0, fs_1.safeReaddir)(snapshotPath)).filter((v) => !removeFiles.includes(v)));
|
|
164
164
|
};
|
|
165
165
|
await reloadFiles();
|
|
166
166
|
const absolute = {
|
package/lib/tasks/MssqlTask.js
CHANGED
|
@@ -66,7 +66,7 @@ class MssqlTask extends TaskAbstract_1.TaskAbstract {
|
|
|
66
66
|
async restore(data) {
|
|
67
67
|
this.verbose = data.options.verbose;
|
|
68
68
|
const snapshotPath = data.snapshotPath;
|
|
69
|
-
const files = await (0, fs_1.
|
|
69
|
+
const files = await (0, fs_1.safeReaddir)(snapshotPath);
|
|
70
70
|
for (const file of files) {
|
|
71
71
|
if (!file.endsWith(MssqlTask.SUFFIX))
|
|
72
72
|
continue;
|
|
@@ -255,7 +255,7 @@ class MysqlDumpTask extends TaskAbstract_1.TaskAbstract {
|
|
|
255
255
|
...params,
|
|
256
256
|
database: database.name,
|
|
257
257
|
});
|
|
258
|
-
const [files, compressed] = (0, fs_1.groupFiles)(await (0, fs_1.
|
|
258
|
+
const [files, compressed] = (0, fs_1.groupFiles)(await (0, fs_1.safeReaddir)(snapshotPath), Object.values(suffix));
|
|
259
259
|
// Database check
|
|
260
260
|
if (files.some((f) => f.endsWith(suffix.database)) &&
|
|
261
261
|
!(await sql.isDatabaseEmpty(database.name)))
|
|
@@ -168,7 +168,7 @@ class SqlDumpTaskAbstract extends TaskAbstract_1.TaskAbstract {
|
|
|
168
168
|
database: database.name,
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
-
const items = (await (0, fs_1.
|
|
171
|
+
const items = (await (0, fs_1.safeReaddir)(snapshotPath))
|
|
172
172
|
.map(parseSqlFile)
|
|
173
173
|
.filter((v) => !!v);
|
|
174
174
|
// Database check
|
|
@@ -1,11 +1,26 @@
|
|
|
1
|
+
import { MaxAge } from "../logs";
|
|
1
2
|
import { JobConfig } from "./job";
|
|
2
|
-
export declare const defaultsLogPath: string;
|
|
3
3
|
export type DatatruckCronServerOptions = {
|
|
4
|
-
enabled?: boolean;
|
|
5
4
|
/**
|
|
6
|
-
* @default
|
|
5
|
+
* @default true
|
|
7
6
|
*/
|
|
8
|
-
|
|
7
|
+
enabled?: boolean;
|
|
8
|
+
log?: {
|
|
9
|
+
/**
|
|
10
|
+
* @default true
|
|
11
|
+
*/
|
|
12
|
+
enabled?: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* @default "/var/logs/datatruck"
|
|
15
|
+
*/
|
|
16
|
+
path?: string;
|
|
17
|
+
rotate?: {
|
|
18
|
+
/**
|
|
19
|
+
* @default {"days":14}
|
|
20
|
+
*/
|
|
21
|
+
maxAge?: MaxAge;
|
|
22
|
+
};
|
|
23
|
+
};
|
|
9
24
|
};
|
|
10
25
|
export declare function createCronServer(options: JobConfig): Promise<{
|
|
11
26
|
start: () => void;
|
|
@@ -1,18 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createCronServer =
|
|
3
|
+
exports.createCronServer = void 0;
|
|
4
4
|
const ConfigAction_1 = require("../../actions/ConfigAction");
|
|
5
5
|
const cli_1 = require("../cli");
|
|
6
6
|
const cron_1 = require("../cron");
|
|
7
|
+
const logs_1 = require("../logs");
|
|
7
8
|
const string_1 = require("../string");
|
|
8
9
|
const watcher_1 = require("../watcher");
|
|
9
10
|
const job_1 = require("./job");
|
|
10
11
|
const croner_1 = require("croner");
|
|
11
|
-
const os_1 = require("os");
|
|
12
|
-
const path_1 = require("path");
|
|
13
|
-
exports.defaultsLogPath = (0, os_1.platform)() === "win32"
|
|
14
|
-
? (0, path_1.join)(process.env.APPDATA ?? `${process.env.HOMEDRIVE ?? "C:"}\\ProgramData`, "datatruck\\logs")
|
|
15
|
-
: "/var/logs/datatruck";
|
|
16
12
|
function createCrons(jobs, options) {
|
|
17
13
|
const crons = [];
|
|
18
14
|
for (const name in jobs) {
|
|
@@ -25,7 +21,7 @@ function createCrons(jobs, options) {
|
|
|
25
21
|
context: name,
|
|
26
22
|
catch: true,
|
|
27
23
|
protect: true,
|
|
28
|
-
}, () => (0, job_1.
|
|
24
|
+
}, () => (0, job_1.runCronJob)(job, name, options)));
|
|
29
25
|
}
|
|
30
26
|
return crons;
|
|
31
27
|
}
|
|
@@ -34,31 +30,55 @@ async function createCronServer(options) {
|
|
|
34
30
|
config: options.configPath,
|
|
35
31
|
});
|
|
36
32
|
let crons = createCrons(config.jobs || {}, options);
|
|
33
|
+
let log = config.server?.cron?.log;
|
|
34
|
+
const getLogPath = () => log?.path ?? logs_1.defaultsLogPath;
|
|
35
|
+
let reloading = false;
|
|
37
36
|
const watcher = (0, watcher_1.createWatcher)({
|
|
38
37
|
onRead: () => ConfigAction_1.ConfigAction.findAndParseFile(options.configPath),
|
|
39
38
|
onCheck: (prev, current) => (0, string_1.compareJsons)(prev, current),
|
|
40
39
|
onError: (error) => {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
console.error(error);
|
|
44
|
-
}
|
|
40
|
+
(0, cli_1.logJson)("cron-server", "job update error");
|
|
41
|
+
console.error(error);
|
|
45
42
|
},
|
|
46
43
|
onChange: (data) => {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
44
|
+
(0, cli_1.logJson)("cron-server", "jobs updated");
|
|
45
|
+
try {
|
|
46
|
+
reloading = true;
|
|
47
|
+
handler.stop();
|
|
48
|
+
const enabled = data?.server?.cron?.enabled ?? true;
|
|
49
|
+
crons = enabled ? createCrons(data?.jobs || {}, options) : [];
|
|
50
|
+
log = data?.server?.cron?.log;
|
|
51
|
+
handler.start();
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
reloading = false;
|
|
55
|
+
}
|
|
53
56
|
},
|
|
54
57
|
});
|
|
58
|
+
let rotateLogsInterval;
|
|
59
|
+
const rotateLogs = async () => {
|
|
60
|
+
const removed = await (0, logs_1.removeOldLogs)(getLogPath(), log?.rotate?.maxAge ?? { days: 14 });
|
|
61
|
+
if (removed.length)
|
|
62
|
+
(0, cli_1.logJson)("cron-server", "old logs removed", {
|
|
63
|
+
amount: removed.length,
|
|
64
|
+
});
|
|
65
|
+
};
|
|
55
66
|
const handler = {
|
|
56
67
|
start: () => {
|
|
68
|
+
if (!reloading) {
|
|
69
|
+
(0, cli_1.logJson)("cron-server", `server started`, {
|
|
70
|
+
"log.path": getLogPath(),
|
|
71
|
+
});
|
|
72
|
+
clearInterval(rotateLogsInterval);
|
|
73
|
+
rotateLogsInterval = setInterval(rotateLogs, 60_000);
|
|
74
|
+
}
|
|
57
75
|
for (const cron of crons)
|
|
58
76
|
cron.resume();
|
|
59
77
|
watcher.start();
|
|
60
78
|
},
|
|
61
79
|
stop: () => {
|
|
80
|
+
if (!reloading)
|
|
81
|
+
clearInterval(rotateLogsInterval);
|
|
62
82
|
watcher.stop();
|
|
63
83
|
for (const cron of crons)
|
|
64
84
|
cron.stop();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { BackupCommandOptions } from "../../commands/BackupCommand";
|
|
2
2
|
import { CopyCommandOptions } from "../../commands/CopyCommand";
|
|
3
3
|
import { PruneCommandOptions } from "../../commands/PruneCommand";
|
|
4
|
+
import { DatatruckCronServerOptions } from "./cron-server";
|
|
4
5
|
export type JobScheduleObject = {
|
|
5
6
|
minute?: number | {
|
|
6
7
|
each: number;
|
|
@@ -33,9 +34,10 @@ export type Job = JobAction & {
|
|
|
33
34
|
schedule?: JobSchedule;
|
|
34
35
|
};
|
|
35
36
|
export type JobConfig = {
|
|
36
|
-
log:
|
|
37
|
-
logPath: string | boolean | undefined;
|
|
37
|
+
log: DatatruckCronServerOptions["log"];
|
|
38
38
|
verbose: boolean;
|
|
39
39
|
configPath: string;
|
|
40
40
|
};
|
|
41
|
-
export declare function
|
|
41
|
+
export declare function getJobCliOptions(job: Job): string[];
|
|
42
|
+
export declare function runJob(job: Job, name: string, config: Omit<JobConfig, "log">): Promise<number>;
|
|
43
|
+
export declare function runCronJob(job: Job, name: string, config: JobConfig): Promise<void>;
|
|
@@ -1,32 +1,63 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.runJob = void 0;
|
|
3
|
+
exports.runCronJob = exports.runJob = exports.getJobCliOptions = void 0;
|
|
4
4
|
const async_process_1 = require("../async-process");
|
|
5
5
|
const cli_1 = require("../cli");
|
|
6
6
|
const fs_1 = require("../fs");
|
|
7
|
+
const logs_1 = require("../logs");
|
|
7
8
|
const options_1 = require("../options");
|
|
8
9
|
const command_1 = require("./command");
|
|
9
|
-
const cron_server_1 = require("./cron-server");
|
|
10
10
|
const fs_2 = require("fs");
|
|
11
11
|
const promises_1 = require("fs/promises");
|
|
12
12
|
const path_1 = require("path");
|
|
13
|
+
async function createJobLog(config, name) {
|
|
14
|
+
if (typeof config === "object" && (config?.enabled ?? true)) {
|
|
15
|
+
const dt = new Date().toISOString().replaceAll(":", "-");
|
|
16
|
+
const dir = config?.path ?? logs_1.defaultsLogPath;
|
|
17
|
+
const tmpLogPath = (0, path_1.join)(dir, `${dt}_${name}.tmp.log`);
|
|
18
|
+
await (0, promises_1.mkdir)(dir, { recursive: true });
|
|
19
|
+
return {
|
|
20
|
+
dt,
|
|
21
|
+
dir,
|
|
22
|
+
stream: (0, fs_2.createWriteStream)(tmpLogPath),
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function getJobCliOptions(job) {
|
|
27
|
+
const Command = command_1.datatruckCommands[job.action];
|
|
28
|
+
const command = new Command({ config: { packages: [], repositories: [] } }, {});
|
|
29
|
+
return (0, options_1.stringifyOptions)(command.optionsConfig, job.action === "prune"
|
|
30
|
+
? ({ ...job.options, confirm: true })
|
|
31
|
+
: job.options);
|
|
32
|
+
}
|
|
33
|
+
exports.getJobCliOptions = getJobCliOptions;
|
|
13
34
|
async function runJob(job, name, config) {
|
|
35
|
+
const cliOptions = getJobCliOptions(job);
|
|
36
|
+
const [node, bin] = process.argv;
|
|
37
|
+
return await async_process_1.AsyncProcess.exec(node, [
|
|
38
|
+
process.env.DTT_BIN_SCRIPT ?? process.env.pm_exec_path ?? bin,
|
|
39
|
+
"-c",
|
|
40
|
+
config.configPath,
|
|
41
|
+
job.action,
|
|
42
|
+
...cliOptions,
|
|
43
|
+
...(config.verbose ? ["-v"] : []),
|
|
44
|
+
], {
|
|
45
|
+
$exitCode: false,
|
|
46
|
+
$log: { exec: config.verbose },
|
|
47
|
+
env: {
|
|
48
|
+
...process.env,
|
|
49
|
+
JOB_NAME: name,
|
|
50
|
+
},
|
|
51
|
+
stdio: "inherit",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
exports.runJob = runJob;
|
|
55
|
+
async function runCronJob(job, name, config) {
|
|
14
56
|
let pid = 0;
|
|
15
57
|
try {
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const cliOptions = (0, options_1.stringifyOptions)(command.optionsConfig, job.action === "prune"
|
|
19
|
-
? ({ ...job.options, confirm: true })
|
|
20
|
-
: job.options);
|
|
58
|
+
const log = await createJobLog(config.log, name);
|
|
59
|
+
const cliOptions = getJobCliOptions(job);
|
|
21
60
|
const [node, bin] = process.argv;
|
|
22
|
-
const baseLogPath = typeof config.logPath === "string"
|
|
23
|
-
? config.logPath
|
|
24
|
-
: config.logPath === true || config.logPath === undefined
|
|
25
|
-
? cron_server_1.defaultsLogPath
|
|
26
|
-
: config.logPath;
|
|
27
|
-
let stream;
|
|
28
|
-
let logPath;
|
|
29
|
-
const dt = new Date().toISOString().replaceAll(":", "-");
|
|
30
61
|
const argv = [
|
|
31
62
|
process.env.DTT_BIN_SCRIPT ?? process.env.pm_exec_path ?? bin,
|
|
32
63
|
"--tty",
|
|
@@ -38,45 +69,38 @@ async function runJob(job, name, config) {
|
|
|
38
69
|
job.action,
|
|
39
70
|
...cliOptions,
|
|
40
71
|
];
|
|
41
|
-
|
|
42
|
-
const tmpLogPath = (0, path_1.join)(baseLogPath, dt) + ".log";
|
|
43
|
-
await (0, promises_1.mkdir)(baseLogPath, { recursive: true });
|
|
44
|
-
stream = (0, fs_2.createWriteStream)(tmpLogPath);
|
|
45
|
-
stream.write(`+ dtt ${argv.join(" ")}\n`);
|
|
46
|
-
}
|
|
72
|
+
log?.stream.write(`+ dtt ${argv.slice(1).join(" ")}\n`);
|
|
47
73
|
const p = new async_process_1.AsyncProcess(node, argv, {
|
|
48
74
|
$log: config.verbose,
|
|
49
75
|
$exitCode: false,
|
|
50
76
|
env: {
|
|
51
77
|
...process.env,
|
|
78
|
+
JOB_NAME: name,
|
|
52
79
|
COLUMNS: "160",
|
|
53
80
|
NO_COLOR: "1",
|
|
54
|
-
JOB_NAME: name,
|
|
55
81
|
},
|
|
56
82
|
});
|
|
57
83
|
pid = p.child.pid || 0;
|
|
58
|
-
if (
|
|
84
|
+
if (log)
|
|
59
85
|
(0, cli_1.logJson)("job", `'${name}' started`, { pid });
|
|
60
86
|
const [exitCode] = await Promise.all([
|
|
61
87
|
p.waitForClose(),
|
|
62
|
-
|
|
63
|
-
|
|
88
|
+
...(log?.stream
|
|
89
|
+
? [p.stderr.pipe(log?.stream), p.stdout.pipe(log?.stream)]
|
|
90
|
+
: []),
|
|
64
91
|
]);
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
92
|
+
let logData = {};
|
|
93
|
+
if (log) {
|
|
94
|
+
const base = (0, path_1.dirname)(log.stream.path.toString());
|
|
95
|
+
const logPath = (0, path_1.join)(base, `${log.dt}_${name}_${pid}.log`);
|
|
96
|
+
await (0, fs_1.safeRename)(log?.stream.path.toString(), logPath);
|
|
97
|
+
logData["log"] = logPath;
|
|
68
98
|
}
|
|
69
|
-
|
|
70
|
-
(0, cli_1.logJson)("job", `'${name}' finished`, {
|
|
71
|
-
pid,
|
|
72
|
-
exitCode,
|
|
73
|
-
...(logPath && { log: logPath }),
|
|
74
|
-
});
|
|
99
|
+
(0, cli_1.logJson)("job", `'${name}' finished`, { pid, exitCode, ...logData });
|
|
75
100
|
}
|
|
76
101
|
catch (error) {
|
|
77
|
-
|
|
78
|
-
(0, cli_1.logJson)("job", `'${name}' failed`, { pid });
|
|
102
|
+
(0, cli_1.logJson)("job", `'${name}' failed`, { pid });
|
|
79
103
|
console.error(error);
|
|
80
104
|
}
|
|
81
105
|
}
|
|
82
|
-
exports.
|
|
106
|
+
exports.runCronJob = runCronJob;
|
|
@@ -37,7 +37,6 @@ export declare const headerKey: {
|
|
|
37
37
|
password: string;
|
|
38
38
|
};
|
|
39
39
|
export declare function createDatatruckRepositoryServer(inOptions: Omit<DatatruckRepositoryServerOptions, "listen">, config?: {
|
|
40
|
-
log?: boolean;
|
|
41
40
|
configPath?: string;
|
|
42
41
|
}): import("http").Server<typeof IncomingMessage, typeof import("http").ServerResponse>;
|
|
43
42
|
export {};
|
|
@@ -80,8 +80,7 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
|
|
|
80
80
|
const backend = findRepositoryBackend(req, repository, options);
|
|
81
81
|
if (!backend)
|
|
82
82
|
return res.writeHead(401);
|
|
83
|
-
|
|
84
|
-
(0, cli_1.logJson)("repository-server", "request", { id, repository, url });
|
|
83
|
+
(0, cli_1.logJson)("repository-server", "request", { id, repository, url });
|
|
85
84
|
const fs = new virtual_fs_1.LocalFs({ backend: backend.path });
|
|
86
85
|
if (action === "comcheck") {
|
|
87
86
|
res.write(JSON.stringify({ success: true }));
|
|
@@ -111,14 +110,11 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
|
|
|
111
110
|
if (json !== undefined)
|
|
112
111
|
res.write(JSON.stringify(json));
|
|
113
112
|
}
|
|
114
|
-
|
|
115
|
-
(0, cli_1.logJson)("repository-server", "request finished", { id });
|
|
113
|
+
(0, cli_1.logJson)("repository-server", "request finished", { id });
|
|
116
114
|
}
|
|
117
115
|
catch (error) {
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
console.error(error);
|
|
121
|
-
}
|
|
116
|
+
(0, cli_1.logJson)("repository-server", "request failed", { id });
|
|
117
|
+
console.error(error);
|
|
122
118
|
if (!res.writableEnded && !res.headersSent)
|
|
123
119
|
res.writeHead(500, error.message);
|
|
124
120
|
}
|
package/lib/utils/fs.d.ts
CHANGED
|
@@ -28,7 +28,8 @@ export declare function parsePackageFile(): {
|
|
|
28
28
|
export declare function findFile(sourcePath: string, baseName: string, extensions: string[], errorMessage?: string): Promise<string>;
|
|
29
29
|
export declare function fastFolderSizeAsync(path: string): Promise<number>;
|
|
30
30
|
export declare function readPartialFile(path: string, positions: [number, number?]): Promise<string>;
|
|
31
|
-
export declare function
|
|
31
|
+
export declare function safeReaddir(path: string): Promise<string[]>;
|
|
32
|
+
export declare function tryReaddir(path: string): Promise<string[]>;
|
|
32
33
|
export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
|
|
33
34
|
/**
|
|
34
35
|
* @experimental
|
package/lib/utils/fs.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.asFile = exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.readTextFile = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.
|
|
6
|
+
exports.asFile = exports.groupFiles = exports.ensureFreeDiskSpace = exports.checkFreeDiskSpace = exports.fetchDiskStats = exports.initEmptyDir = exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.readTextFile = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.tryReaddir = exports.safeReaddir = exports.readPartialFile = exports.fastFolderSizeAsync = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.include = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureExistsDir = exports.ensureSingleFile = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
|
|
7
7
|
const pkg_1 = require("../pkg");
|
|
8
8
|
const bytes_1 = require("./bytes");
|
|
9
9
|
const error_1 = require("./error");
|
|
@@ -58,7 +58,7 @@ async function ensureEmptyDir(path) {
|
|
|
58
58
|
}
|
|
59
59
|
exports.ensureEmptyDir = ensureEmptyDir;
|
|
60
60
|
async function ensureSingleFile(path) {
|
|
61
|
-
const files = await
|
|
61
|
+
const files = await safeReaddir(path);
|
|
62
62
|
if (files.length !== 1)
|
|
63
63
|
throw new error_1.AppError(`Dir has not one file: ${files.length}`);
|
|
64
64
|
const [file] = files;
|
|
@@ -171,15 +171,13 @@ async function readPartialFile(path, positions) {
|
|
|
171
171
|
});
|
|
172
172
|
}
|
|
173
173
|
exports.readPartialFile = readPartialFile;
|
|
174
|
-
async function
|
|
174
|
+
async function safeReaddir(path) {
|
|
175
175
|
try {
|
|
176
176
|
return await (0, promises_1.readdir)(path);
|
|
177
177
|
}
|
|
178
178
|
catch (anyError) {
|
|
179
179
|
const nodeError = anyError;
|
|
180
180
|
if (nodeError.code === "ENOENT") {
|
|
181
|
-
if (optional)
|
|
182
|
-
return [];
|
|
183
181
|
const error = new Error(nodeError.message);
|
|
184
182
|
error.code = nodeError.code;
|
|
185
183
|
error.errno = nodeError.errno;
|
|
@@ -189,9 +187,20 @@ async function readDir(path, optional) {
|
|
|
189
187
|
throw anyError;
|
|
190
188
|
}
|
|
191
189
|
}
|
|
192
|
-
exports.
|
|
190
|
+
exports.safeReaddir = safeReaddir;
|
|
191
|
+
async function tryReaddir(path) {
|
|
192
|
+
try {
|
|
193
|
+
return await safeReaddir(path);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (error.code === "ENOENT")
|
|
197
|
+
return [];
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
exports.tryReaddir = tryReaddir;
|
|
193
202
|
async function forEachFile(dirPath, cb, includeDir) {
|
|
194
|
-
const files = await
|
|
203
|
+
const files = await safeReaddir(dirPath);
|
|
195
204
|
for (const file of files) {
|
|
196
205
|
const filePath = (0, path_1.join)(dirPath, file);
|
|
197
206
|
if ((await (0, promises_1.stat)(filePath)).isDirectory()) {
|
package/lib/utils/git.js
CHANGED
|
@@ -23,7 +23,7 @@ class Git {
|
|
|
23
23
|
}
|
|
24
24
|
async canBeInit(repo) {
|
|
25
25
|
return ((0, fs_1.isLocalDir)(repo) &&
|
|
26
|
-
(!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.
|
|
26
|
+
(!(await (0, fs_1.existsDir)(repo)) || !(await (0, fs_1.safeReaddir)(repo)).length));
|
|
27
27
|
}
|
|
28
28
|
async clone(options) {
|
|
29
29
|
await this.exec([
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type MaxAge = {
|
|
2
|
+
days?: number;
|
|
3
|
+
hours?: number;
|
|
4
|
+
minutes?: number;
|
|
5
|
+
};
|
|
6
|
+
export declare function maxAgeToMS(input: MaxAge): number;
|
|
7
|
+
export declare const defaultsLogPath: string;
|
|
8
|
+
export declare function removeOldLogs(path: string, inMaxAge: MaxAge): Promise<string[]>;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.removeOldLogs = exports.defaultsLogPath = exports.maxAgeToMS = void 0;
|
|
4
|
+
const fs_1 = require("./fs");
|
|
5
|
+
const promises_1 = require("fs/promises");
|
|
6
|
+
const os_1 = require("os");
|
|
7
|
+
const path_1 = require("path");
|
|
8
|
+
const maxAgeUnits = {
|
|
9
|
+
minutes: () => 60 * 1000,
|
|
10
|
+
hours: () => 60 * maxAgeUnits.minutes(),
|
|
11
|
+
days: () => 24 * maxAgeUnits.hours(),
|
|
12
|
+
};
|
|
13
|
+
function maxAgeToMS(input) {
|
|
14
|
+
let ms = 0;
|
|
15
|
+
for (const key in input) {
|
|
16
|
+
const units = input[key];
|
|
17
|
+
if (units && units >= 0) {
|
|
18
|
+
const factor = maxAgeUnits[key]();
|
|
19
|
+
ms += units * factor;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return ms;
|
|
23
|
+
}
|
|
24
|
+
exports.maxAgeToMS = maxAgeToMS;
|
|
25
|
+
exports.defaultsLogPath = (0, os_1.platform)() === "win32"
|
|
26
|
+
? (0, path_1.join)(process.env.APPDATA ?? `${process.env.HOMEDRIVE ?? "C:"}\\ProgramData`, "datatruck\\logs")
|
|
27
|
+
: "/var/logs/datatruck";
|
|
28
|
+
async function removeOldLogs(path, inMaxAge) {
|
|
29
|
+
const maxAge = maxAgeToMS(inMaxAge);
|
|
30
|
+
const hasMaxAge = Object.values(inMaxAge).some((v) => typeof v === "number");
|
|
31
|
+
if (!hasMaxAge)
|
|
32
|
+
return [];
|
|
33
|
+
const files = (await (0, fs_1.tryReaddir)(path)).filter((file) => file.endsWith(".log"));
|
|
34
|
+
const now = Date.now();
|
|
35
|
+
const paths = [];
|
|
36
|
+
for (const file of files) {
|
|
37
|
+
const filePath = (0, path_1.join)(path, file);
|
|
38
|
+
const { mtimeMs } = await (0, promises_1.stat)((0, path_1.join)(path, file));
|
|
39
|
+
const ms = now - mtimeMs;
|
|
40
|
+
if (ms > maxAge) {
|
|
41
|
+
await (0, promises_1.rm)(filePath);
|
|
42
|
+
paths.push(filePath);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return paths;
|
|
46
|
+
}
|
|
47
|
+
exports.removeOldLogs = removeOldLogs;
|
package/lib/utils/mysql.d.ts
CHANGED
|
@@ -22,12 +22,12 @@ export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
|
|
|
22
22
|
dump: (input: {
|
|
23
23
|
output: string;
|
|
24
24
|
database: string;
|
|
25
|
-
items?: string[]
|
|
26
|
-
onlyStoredPrograms?: boolean
|
|
27
|
-
controller?: AbortController
|
|
28
|
-
onProgress?: (
|
|
25
|
+
items?: string[];
|
|
26
|
+
onlyStoredPrograms?: boolean;
|
|
27
|
+
controller?: AbortController;
|
|
28
|
+
onProgress?: (data: {
|
|
29
29
|
totalBytes: number;
|
|
30
|
-
}) => void
|
|
30
|
+
}) => void;
|
|
31
31
|
}) => Promise<void>;
|
|
32
32
|
assertDumpFile: typeof assertDumpFile;
|
|
33
33
|
fetchTableNames: (database: string, include?: string[], exclude?: string[]) => Promise<string[]>;
|
package/lib/utils/options.js
CHANGED
|
@@ -68,9 +68,9 @@ exports.createCommand = createCommand;
|
|
|
68
68
|
function stringifyOptions(options, object) {
|
|
69
69
|
const result = [];
|
|
70
70
|
const prepend = [];
|
|
71
|
-
for (const
|
|
72
|
-
const option = options[
|
|
73
|
-
const value = object[
|
|
71
|
+
for (const name in options) {
|
|
72
|
+
const option = options[name];
|
|
73
|
+
const value = object[name];
|
|
74
74
|
if (value === undefined)
|
|
75
75
|
continue;
|
|
76
76
|
if (option.flag === false) {
|
|
@@ -79,7 +79,7 @@ function stringifyOptions(options, object) {
|
|
|
79
79
|
else {
|
|
80
80
|
const flag = option.shortFlag
|
|
81
81
|
? `-${option.shortFlag}`
|
|
82
|
-
: `--${option.flag ??
|
|
82
|
+
: `--${option.flag ?? name}`;
|
|
83
83
|
if (option.value === "boolean") {
|
|
84
84
|
if (option.value)
|
|
85
85
|
result.push(flag);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@datatruck/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.39.1",
|
|
4
4
|
"description": "Tool for creating and managing backups",
|
|
5
5
|
"homepage": "https://github.com/swordev/datatruck#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"dayjs": "^1.11.10",
|
|
36
36
|
"fast-folder-size": "^2.2.0",
|
|
37
37
|
"fast-glob": "^3.3.2",
|
|
38
|
-
"listr2": "^8.
|
|
38
|
+
"listr2": "^8.2.1",
|
|
39
39
|
"micromatch": "^4.0.5",
|
|
40
40
|
"mysql2": "^3.9.3",
|
|
41
41
|
"tty-table": "^4.2.3",
|