@datatruck/cli 0.36.2 → 0.36.4

package/lib/actions/BackupAction.js

@@ -149,7 +149,8 @@ class BackupAction {
             let title = item.key.slice(0, 1).toUpperCase() + item.key.slice(1);
             return item.key === "backup" && color ? chalk_1.default.cyan(title) : title;
         };
-        const renderData = (item, color, result = []) => {
+        const renderData = (item, result, format) => {
+            const color = format !== "list";
             const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
             return (0, cli_1.renderListTaskItem)(item, color, {
                 snapshot: (data) => data.id,
@@ -176,6 +177,9 @@ class BackupAction {
                             result += item.data.pruned;
                         return result;
                     }, 0),
+                    ...(format === "list" && {
+                        duration: (0, date_1.duration)(item.elapsed),
+                    }),
                 }),
             });
         };
@@ -187,7 +191,7 @@ class BackupAction {
                 .map((item) => {
                 const icon = (0, cli_1.renderResult)(item.error, false);
                 const title = renderTitle(item);
-                const data = renderData(item, false, result);
+                const data = renderData(item, result, "list");
                 return `${icon} ${title}: ${data}`;
             }),
             table: {
@@ -203,7 +207,7 @@ class BackupAction {
                     .map((item) => [
                     (0, cli_1.renderResult)(item.error),
                     renderTitle(item, true),
-                    renderData(item, true, result),
+                    renderData(item, result, "table"),
                     (0, date_1.duration)(item.elapsed),
                     (0, cli_1.renderError)(item.error, options.verbose),
                 ]),

package/lib/actions/CopyAction.js

@@ -76,7 +76,8 @@ class CopyAction {
             let title = item.key.slice(0, 1).toUpperCase() + item.key.slice(1);
             return item.key === "copy" && color ? chalk_1.default.cyan(title) : title;
         };
-        const renderData = (item, color, items = []) => {
+        const renderData = (item, items = [], format) => {
+            const color = format !== "list";
             const g = (v) => (color ? `${chalk_1.default.gray(`(${v})`)}` : `(${v})`);
             return (0, cli_1.renderListTaskItem)(item, color, {
                 snapshots: (data) => data.snapshots.length,
@@ -93,6 +94,9 @@ class CopyAction {
                     errors: data.errors,
                     copied: items.filter((i) => i.key === "copy" && !i.error && !i.data.skipped).length,
                     skipped: items.filter((i) => i.key === "copy" && !i.error && i.data.skipped).length,
+                    ...(format === "list" && {
+                        duration: (0, date_1.duration)(item.elapsed),
+                    }),
                 }),
             });
         };
@@ -102,7 +106,7 @@ class CopyAction {
             list: () => result.map((item) => {
                 const icon = (0, cli_1.renderResult)(item.error, false);
                 const title = renderTitle(item);
-                const data = renderData(item, false, result);
+                const data = renderData(item, result, "list");
                 return `${icon} ${title}: ${data}`;
             }),
             table: {
@@ -116,7 +120,7 @@ class CopyAction {
                 rows: () => result.map((item) => [
                     (0, cli_1.renderResult)(item.error),
                     renderTitle(item, true),
-                    renderData(item, true, result),
+                    renderData(item, result, "table"),
                     (0, date_1.duration)(item.elapsed),
                     (0, cli_1.renderError)(item.error, options.verbose),
                 ]),
@@ -147,9 +151,8 @@ class CopyAction {
     async copyCrossRepository(options) {
         const env_1 = { stack: [], error: void 0, hasError: false };
         try {
-            const { repo, repoConfig, mirrorRepo, mirrorConfig, snapshot } = options;
+            const { repo, mirrorRepo, mirrorConfig, snapshot } = options;
             const tmp = __addDisposableResource(env_1, await (0, temp_1.useTempDir)("copy", "restore"), true);
-            const pkg = (0, config_1.findPackageOrFail)(this.config, snapshot.packageName);
             await repo.restore({
                 options: {
                     verbose: this.options.verbose,
@@ -157,7 +160,7 @@ class CopyAction {
                 },
                 snapshot: { id: snapshot.id, date: snapshot.date },
                 package: { name: snapshot.packageName },
-                packageConfig: (0, config_1.findPackageRepositoryConfig)(pkg, repoConfig),
+                packageConfig: undefined,
                 snapshotPath: tmp.path,
                 onProgress: options.onProgress,
             });
@@ -174,7 +177,7 @@ class CopyAction {
                     name: snapshot.packageName,
                     path: tmp.path,
                 },
-                packageConfig: (0, config_1.findPackageRepositoryConfig)(pkg, mirrorConfig),
+                packageConfig: undefined,
                 onProgress: options.onProgress,
             });
         }

package/lib/cli.js

@@ -76,7 +76,8 @@ function makeCommandAction(command) {
                 }
             }
         }
-        process.stdout.write("", () => process.exit(exitCode));
+        await (0, cli_1.waitForStdDrain)(5000);
+        process.exit(exitCode);
     };
 }
 const program = new commander_1.Command();

package/lib/commands/StartServerCommand.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StartServerCommand = void 0;
 const ConfigAction_1 = require("../actions/ConfigAction");
+const cli_1 = require("../utils/cli");
 const cron_server_1 = require("../utils/datatruck/cron-server");
 const repository_server_1 = require("../utils/datatruck/repository-server");
 const error_1 = require("../utils/error");
@@ -22,7 +23,7 @@ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
             });
             const port = repositoryOptions.listen?.port ?? 8888;
             const address = repositoryOptions.listen?.address ?? "127.0.0.1";
-            console.info(`Listening datatruck repository on http://${address}:${port}`);
+            (0, cli_1.logJson)("datatruck-server", `listening server on http://${address}:${port}`);
             server.on("error", (error) => {
                 console.error(`SERVER ERROR`, error);
                 process.exit(1);
@@ -30,6 +31,7 @@ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
             server.listen(port, address);
         }
         const cronOptions = config.server?.cron || {};
+        const cronJobs = cronOptions.actions || [];
         if (cronOptions.enabled ?? true) {
             if (typeof this.configPath !== "string")
                 throw new error_1.AppError(`Config path is required by cron server`);
@@ -39,7 +41,9 @@ class StartServerCommand extends CommandAbstract_1.CommandAbstract {
                 configPath: this.configPath,
             });
             server.start();
-            console.info(`Cron server started`);
+            (0, cli_1.logJson)("cron-server", `server started`, {
+                jobs: cronJobs.length,
+            });
         }
         process.on("SIGINT", () => process.exit(1));
         process.on("SIGTERM", () => process.exit(1));

package/lib/repositories/RepositoryAbstract.d.ts

@@ -27,7 +27,7 @@ export type RepoFetchSnapshotsData = {
 export type RepoCopyData<TRepositoryConfig> = {
     options: BackupActionOptions;
     snapshot: PreSnapshot;
-    package: PackageConfig;
+    package: Pick<PackageConfig, "name">;
     mirrorRepositoryConfig: TRepositoryConfig;
     onProgress: (data: Progress) => void;
 };
@@ -35,8 +35,9 @@ export type RepoBackupData<TPackageConfig> = {
     options: BackupActionOptions;
     snapshot: PreSnapshot;
     hostname: string;
-    package: Omit<PackageConfig, "path"> & {
-        path: string;
+    package: Pick<PackageConfig, "name" | "include" | "exclude"> & {
+        path: NonNullable<PackageConfig["path"]>;
+        task?: Pick<NonNullable<PackageConfig["task"]>, "name">;
     };
     packageConfig: TPackageConfig | undefined;
     onProgress: (data: Progress) => void;
@@ -44,9 +45,9 @@ export type RepoBackupData<TPackageConfig> = {
 export type RepoRestoreData<TPackageConfig> = {
     options: RestoreActionOptions;
     snapshot: PreSnapshot;
-    package: PackageConfig;
+    package: Pick<PackageConfig, "name">;
     snapshotPath: string;
-    packageConfig: TPackageConfig;
+    packageConfig: TPackageConfig | undefined;
     onProgress: (data: Progress) => void;
 };
 export type RepoPruneData = {

package/lib/utils/cli.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="node" />
 import { Listr3TaskResultEnd } from "./list";
+import chalk from "chalk";
 export declare const showCursorCommand = "\u001B[?25h";
 export declare function renderProgressBar(progress: number, size?: number, subprogress?: number): string;
 export declare function logExec(command: string, argv?: string[], env?: NodeJS.ProcessEnv, logToStderr?: boolean): void;
@@ -29,3 +30,7 @@ export declare function stringifyOptions<T1, T2 extends {
     [K in keyof T1]: unknown;
 }>(options: OptionsConfig<T1, T2>, object: any): string[];
 export declare function confirm(message: string): Promise<unknown>;
+export declare function waitForStdDrain(ms?: number): Promise<void>;
+export declare function colorizeValue(value: unknown, color?: typeof chalk.ForegroundColor): string;
+export declare function colorizeObject(input: Record<string, any>): string;
+export declare function logJson(ctx: string, msg: string, data?: any): void;

package/lib/utils/cli.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.confirm = exports.stringifyOptions = exports.parseOptions = exports.renderObject = exports.renderListTaskItem = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
+exports.logJson = exports.colorizeObject = exports.colorizeValue = exports.waitForStdDrain = exports.confirm = exports.stringifyOptions = exports.parseOptions = exports.renderObject = exports.renderListTaskItem = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
 const error_1 = require("./error");
 const chalk_1 = __importDefault(require("chalk"));
 const chalk_2 = require("chalk");
@@ -149,3 +149,48 @@ function confirm(message) {
     });
 }
 exports.confirm = confirm;
+async function waitForStdDrain(ms) {
+    await Promise.all([process.stdout, process.stderr].map((stream) => new Promise((resolve) => {
+        {
+            const finish = () => {
+                clearTimeout(timeout);
+                resolve();
+            };
+            const timeout = ms ? setTimeout(finish, ms) : undefined;
+            const drained = stream.write("", finish);
+            if (drained)
+                finish();
+        }
+    })));
+}
+exports.waitForStdDrain = waitForStdDrain;
+function colorizeValue(value, color) {
+    const json = JSON.stringify(value);
+    return color ? chalk_1.default[color](json) : json;
+}
+exports.colorizeValue = colorizeValue;
+function colorizeObject(input) {
+    const object = {};
+    for (const key in input) {
+        const value = input[key];
+        if (value !== undefined)
+            object[colorizeValue(key)] =
+                typeof value === "object" && !!value && !Array.isArray(value)
+                    ? colorizeObject(value)
+                    : colorizeValue(value, "green");
+    }
+    const values = Object.entries(object)
+        .map(([key, value]) => `${key}: ${value}`)
+        .join(", ");
+    return `{ ${values} }`;
+}
+exports.colorizeObject = colorizeObject;
+function logJson(ctx, msg, data) {
+    const json = colorizeObject({
+        ctx,
+        msg,
+        data,
+    });
+    console.log(json);
+}
+exports.logJson = logJson;

package/lib/utils/datatruck/cron-server.js

@@ -26,8 +26,7 @@ function createJobs(actions, worker) {
 }
 function createCronServer(options, config) {
     const worker = async (action, index) => {
-        if (config.log)
-            console.info(`> [job] ${index} - ${action.name}`);
+        let pid = 0;
         try {
             const Command = command_1.datatruckCommandMap[action.name];
             const command = new Command({ config: { packages: [], repositories: [] } }, {});
@@ -35,19 +34,24 @@ function createCronServer(options, config) {
                 ? ({ ...action.options, confirm: true })
                 : action.options);
             const [node, bin] = process.argv;
-            await async_process_1.AsyncProcess.exec(node, [
+            const p = new async_process_1.AsyncProcess(node, [
                 process.env.pm_exec_path ?? bin,
                 "-c",
                 config.configPath,
                 action.name,
                 ...cliOptions,
-            ], { $log: config.verbose });
+            ], { $log: config.verbose, $exitCode: false });
+            pid = p.child.pid || 0;
             if (config.log)
-                console.info(`< [job] ${index} - ${action.name}`);
+                (0, cli_1.logJson)("cron-server", `${action.name} started`, { pid });
+            const exitCode = await p.waitForClose();
+            if (config.log)
+                (0, cli_1.logJson)("cron-server", `${action.name} finished`, { pid, exitCode });
         }
         catch (error) {
             if (config.log)
-                console.error(`< [job] ${index} - ${action.name}`, error);
+                (0, cli_1.logJson)("cron-server", `${action.name} failed`, { pid });
+            console.error(error);
         }
     };
     let jobs = createJobs(options.actions || [], worker);
@@ -55,12 +59,14 @@ function createCronServer(options, config) {
         onRead: () => ConfigAction_1.ConfigAction.findAndParseFile(config.configPath),
         onCheck: (prev, current) => (0, string_1.compareJsons)(prev, current),
         onError: (error) => {
-            if (config.log)
-                console.error(`< [jobs] update error`, error);
+            if (config.log) {
+                (0, cli_1.logJson)("cron-server", "job update error");
+                console.error(error);
+            }
         },
         onChange: (data) => {
             if (config.log)
-                console.info(`[jobs] updated`);
+                (0, cli_1.logJson)("cron-server", "jobs updated");
             handler.stop();
             const cron = data?.server?.cron;
             const enabled = cron?.enabled ?? true;

package/lib/utils/datatruck/paths.d.ts

@@ -9,7 +9,7 @@ export type ParsePathsOptions = {
 };
 export declare function parsePaths(values: (string | SpawnStep)[], options: ParsePathsOptions): Promise<string[]>;
 export type BackupPathsOptions = {
-    package: PackageConfig;
+    package: Pick<PackageConfig, "name" | "path" | "include" | "exclude">;
     snapshot: PreSnapshot;
     path: string;
     verbose?: boolean;

package/lib/utils/datatruck/paths.js

@@ -12,6 +12,7 @@ async function parsePaths(values, options) {
             await (0, spawnSteps_1.runSpawnSteps)(value, {
                 tempDir: options.tempDir,
                 verbose: options.verbose,
+                data: options.data,
                 onLine: (path) => paths.push(path),
             });
         }

package/lib/utils/datatruck/repository-server.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDatatruckRepositoryServer = exports.headerKey = void 0;
 const ConfigAction_1 = require("../../actions/ConfigAction");
+const cli_1 = require("../cli");
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
 const fs_1 = require("fs");
@@ -77,7 +78,10 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
                 return res.end();
             }
             if (config.log)
-                console.info(`> [repository] ${repository} - ${req.url}`);
+                (0, cli_1.logJson)("repository-server", "request", {
+                    repository,
+                    url: req.url,
+                });
             const fs = new virtual_fs_1.LocalFs({
                 backend: backend.path,
             });
@@ -123,12 +127,18 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
                     res.write(JSON.stringify(json));
             }
             if (config.log)
-                console.info(`< [repository] ${repository} - ${action}`);
+                (0, cli_1.logJson)("repository-server", "request finished", {
+                    url: req.url,
+                });
             res.end();
         }
         catch (error) {
-            if (config.log)
-                console.error(`< [repository] ${req.url}`, error);
+            if (config.log) {
+                (0, cli_1.logJson)("repository-server", "request failed", {
+                    url: req.url,
+                });
+                console.error(error);
+            }
             res.statusCode = 500;
             res.statusMessage = error.message;
             res.end();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.36.2",
+  "version": "0.36.4",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {
@@ -26,18 +26,18 @@
     "config.schema.json"
   ],
   "dependencies": {
-    "@supercharge/promise-pool": "^3.1.0",
+    "@supercharge/promise-pool": "^3.1.1",
     "ajv": "^8.12.0",
     "async": "^3.2.5",
     "chalk": "^4.1.2",
-    "commander": "^11.1.0",
-    "croner": "^8.0.0",
+    "commander": "^12.0.0",
+    "croner": "^8.0.1",
     "dayjs": "^1.11.10",
     "fast-folder-size": "^2.2.0",
     "fast-glob": "^3.3.2",
-    "listr2": "^8.0.1",
+    "listr2": "^8.0.2",
     "micromatch": "^4.0.5",
-    "mysql2": "^3.7.0",
+    "mysql2": "^3.9.1",
     "tty-table": "^4.2.3",
     "yaml": "^2.3.4"
   },