@datatruck/cli 0.33.0 → 0.34.1

package/lib/utils/Restic.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Restic = void 0;
+const async_process_1 = require("./async-process");
 const fs_1 = require("./fs");
-const process_1 = require("./process");
 const string_1 = require("./string");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
@@ -28,14 +28,11 @@ class Restic {
         }
         return `${input.backend}:${(0, string_1.formatUri)({ ...input, password: input.password }, hidePassword)}`;
     }
-    async exec(args, settings, options) {
-        return await (0, process_1.exec)("restic", args, {
+    createProcess(args, options) {
+        return new async_process_1.AsyncProcess("restic", args, {
             stdio: ["ignore", "pipe", "pipe"],
             env: { ...process.env, ...this.options.env },
-            cwd: options?.cwd,
-        }, {
-            stderr: { toExitCode: true },
-            log: this.options.log
+            $log: this.options.log
                 ? {
                     exec: true,
                     stdout: true,
@@ -49,17 +46,22 @@ class Restic {
                     ],
                 }
                 : {},
-            ...(settings ?? {}),
+            ...(options ?? {}),
         });
     }
+    async exec(args, options) {
+        return await this.createProcess(args, options).waitForClose();
+    }
+    async stdout(args, options) {
+        return await this.createProcess(args, options).stdout.fetch();
+    }
     async checkRepository() {
-        const result = await this.exec(["cat", "config"], {
-            onExitCodeError: () => false,
-        });
-        return result.exitCode === 0;
+        return ((await this.exec(["cat", "config"], {
+            $exitCode: false,
+        })) === 0);
     }
     async forget(options) {
-        const result = await this.exec([
+        await this.exec([
             "forget",
             ...(options.keepLast ? ["--keep-last", options.keepLast.toString()] : []),
             ...(options.keepHourly
@@ -87,66 +89,46 @@ class Restic {
             ...(options.prune ? ["--prune"] : []),
             ...(options.snapshotId ? [options.snapshotId] : []),
         ]);
-        return result.stdout;
     }
     async snapshots(options) {
-        const result = await this.exec([
+        const json = await this.stdout([
             "snapshots",
             ...(options.tags?.flatMap((tag) => [`--tag`, tag]) ?? []),
             ...(options.json ? ["--json"] : []),
             ...(options.paths?.flatMap((path) => ["--path", path]) ?? []),
             ...(options.latest ? ["--latest", options.latest.toString()] : []),
             ...(options.snapshotIds || []),
-        ], {
-            stdout: { save: true },
-        });
-        return JSON.parse(result.stdout);
-    }
-    async checkBackupSetPathSupport() {
-        /*const result = await this.exec(["backup", "--set-path"], {
-          onExitCodeError: () => false,
-          stderr: { save: true },
-        });
-        return result.stderr.includes("flag needs an argument");*/
-        return false;
+        ]);
+        return JSON.parse(json);
     }
     async backup(options) {
-        const exec = async () => await this.exec([
-            "backup",
-            "--json",
-            ...(options.exclude?.flatMap((v) => ["-e", v]) ?? []),
-            ...(options.excludeFile?.flatMap((v) => ["--exclude-file", v]) ?? []),
-            ...(options.tags?.flatMap((v) => ["--tag", v]) ?? []),
-            ...(options.setPaths?.flatMap((v) => ["--set-path", v]) ?? []),
-            ...(options.parent ? ["--parent", options.parent] : []),
-            ...options.paths,
-        ], {
-            stderr: {
-                toExitCode: true,
-            },
-            stdout: {
-                ...(options.onStream && {
-                    onData: (data) => {
-                        for (const rawLine of data.split("\n")) {
-                            const line = rawLine.trim();
-                            if (line.startsWith("{") && line.endsWith("}")) {
-                                let parsedLine;
-                                try {
-                                    parsedLine = JSON.parse(line);
-                                }
-                                catch (error) { }
-                                if (parsedLine)
-                                    options.onStream?.(parsedLine);
-                            }
-                        }
-                    },
-                }),
-            },
-        }, {
-            cwd: options.cwd,
-        });
         try {
-            return await exec();
+            const backup = this.createProcess([
+                "backup",
+                "--json",
+                ...(options.exclude?.flatMap((v) => ["-e", v]) ?? []),
+                ...(options.excludeFile?.flatMap((v) => ["--exclude-file", v]) ?? []),
+                ...(options.tags?.flatMap((v) => ["--tag", v]) ?? []),
+                ...(options.setPaths?.flatMap((v) => ["--set-path", v]) ?? []),
+                ...(options.parent ? ["--parent", options.parent] : []),
+                ...options.paths,
+            ], { cwd: options.cwd });
+            if (options.onStream) {
+                await backup.stdout.parseLines((line) => {
+                    if (line.startsWith("{") && line.endsWith("}")) {
+                        let parsedLine;
+                        try {
+                            parsedLine = JSON.parse(line);
+                        }
+                        catch (error) { }
+                        if (parsedLine)
+                            options.onStream?.(parsedLine);
+                    }
+                });
+            }
+            else {
+                await backup.waitForClose();
+            }
         }
         catch (error) {
             if (options.allowEmptySnapshot &&
@@ -169,20 +151,17 @@ class Restic {
         }
     }
     async copy(options) {
-        return await this.exec(["copy", "--json", options.id], {
-            stderr: {
-                toExitCode: true,
-            },
-            stdout: {
-                ...(options.onStream && {
-                    onData: async (data) => {
-                        if (data.startsWith("{") && data.endsWith("}")) {
-                            await options.onStream?.(JSON.parse(data));
-                        }
-                    },
-                }),
-            },
-        });
+        const copy = this.createProcess(["copy", "--json", options.id]);
+        if (options.onStream) {
+            await copy.stdout.parseLines((line) => {
+                if (line.startsWith("{") && line.endsWith("}")) {
+                    options.onStream?.(JSON.parse(line));
+                }
+            });
+        }
+        else {
+            await copy.waitForClose();
+        }
     }
     async restore(options) {
         let progressTimeout;
@@ -207,20 +186,23 @@ class Restic {
         if (typeof progressInterval === "number")
             progressRutine();
         try {
-            const result = await this.exec(["restore", "--json", options.id, "--target", options.target], {
-                stderr: {
-                    toExitCode: true,
-                },
-                stdout: {
-                    ...(options.onStream && {
-                        onData: async (data) => {
-                            if (data.startsWith("{") && data.endsWith("}")) {
-                                await options.onStream?.(JSON.parse(data));
-                            }
-                        },
-                    }),
-                },
-            });
+            const result = this.createProcess([
+                "restore",
+                "--json",
+                options.id,
+                "--target",
+                options.target,
+            ]);
+            if (options.onStream) {
+                await result.stdout.parseLines((line) => {
+                    if (line.startsWith("{") && line.endsWith("}")) {
+                        options.onStream?.(JSON.parse(line));
+                    }
+                });
+            }
+            else {
+                await result.waitForClose();
+            }
             if (snapshots.at(0)?.tags?.includes(emptySnapshotTag))
                 await (0, promises_1.rm)((0, path_1.join)(options.target, `.${emptySnapshotTag}`));
             return result;

package/lib/utils/async-process.d.ts

@@ -0,0 +1,66 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import { ChildProcess, SpawnOptions } from "child_process";
+import { ReadStream } from "fs";
+import { Readable, Writable } from "stream";
+export type AsyncProcessOptions = SpawnOptions & {
+    $log?: AsyncProcessLog | boolean;
+    $controller?: AbortController;
+    $exitCode?: ExitCode;
+};
+type ExitCode = ExitCodeValue | ((code: number) => ExitCodeValue | void | undefined);
+type ExitCodeValue = Error | string | number | boolean;
+type AsyncProcessArgv = (string | number)[] | undefined;
+declare class StdIn {
+    readonly process: AsyncProcess;
+    readonly writable: Writable;
+    constructor(process: AsyncProcess, writable: Writable);
+    pipe(source: string | ReadStream, onProgress?: (data: {
+        totalBytes: number;
+        currentBytes: number;
+        progress: number;
+    }) => void): Promise<void>;
+}
+declare class Std {
+    protected type: "stdout" | "stderr";
+    readonly process: AsyncProcess;
+    readonly readable: Readable;
+    constructor(type: "stdout" | "stderr", process: AsyncProcess, readable: Readable);
+    onData(cb: (chunk: Buffer) => void): void;
+    fetch(): Promise<string>;
+    parseLines(cb: (line: string, total: number) => void): Promise<number>;
+    pipe(out: string | NodeJS.WritableStream | StdIn, onProgress?: (data: {
+        totalBytes: number;
+    }) => void): Promise<void>;
+}
+export type AsyncProcessLog = {
+    colorize?: boolean;
+    exec?: boolean;
+    stdout?: boolean;
+    stderr?: boolean;
+    allToStderr?: boolean;
+    envNames?: string[];
+};
+export declare class AsyncProcess {
+    protected command: string;
+    protected argv: AsyncProcessArgv;
+    protected options: AsyncProcessOptions;
+    readonly child: ChildProcess;
+    readonly stdout: Std;
+    readonly stderr: Std;
+    readonly stdin: StdIn;
+    protected log: AsyncProcessLog | undefined;
+    protected controller: AbortController;
+    private lastStdError;
+    constructor(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions);
+    static exec(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions): Promise<number>;
+    static stdout(command: string, argv: AsyncProcessArgv, options?: AsyncProcessOptions): Promise<string>;
+    private installLog;
+    private installAbortController;
+    private resolveExitCode;
+    waitForClose(): Promise<number>;
+}
+export {};

package/lib/utils/async-process.js

@@ -0,0 +1,237 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AsyncProcess = void 0;
+const cli_1 = require("./cli");
+const fs_1 = require("./fs");
+const math_1 = require("./math");
+const process_1 = require("./process");
+const child_process_1 = require("child_process");
+const fs_2 = require("fs");
+const promises_1 = require("fs/promises");
+const readline_1 = require("readline");
+function resolveLogOptions(log) {
+    return log === true
+        ? { exec: {}, stdout: {}, stderr: {} }
+        : log || undefined;
+}
+function ensureDir(cwd) {
+    if (typeof cwd === "string") {
+        let isDir = false;
+        try {
+            isDir = (0, fs_2.statSync)(cwd).isDirectory();
+        }
+        catch (error) { }
+        if (!isDir)
+            throw new Error(`Current working directory does not exist: ${cwd}`);
+    }
+}
+class StdIn {
+    process;
+    writable;
+    constructor(process, writable) {
+        this.process = process;
+        this.writable = writable;
+    }
+    async pipe(source, onProgress) {
+        if (this.process["log"]?.exec) {
+            const path = source instanceof fs_2.ReadStream
+                ? "path" in source
+                    ? source.path
+                    : "&readableStream"
+                : source;
+            (0, cli_1.logExec)(`[${this.process.child.pid || 0}] < ${path}`);
+        }
+        const stream = typeof source === "string" ? (0, fs_2.createReadStream)(source) : source;
+        const streamPath = stream.path.toString();
+        if (onProgress) {
+            const fileInfo = await (0, promises_1.stat)(streamPath);
+            const totalBytes = fileInfo.size;
+            let currentBytes = 0;
+            stream.on("data", (data) => {
+                currentBytes += data.length;
+                onProgress?.({
+                    totalBytes: totalBytes,
+                    currentBytes: currentBytes,
+                    progress: (0, math_1.progressPercent)(totalBytes, currentBytes),
+                });
+            });
+        }
+        stream.pipe(this.writable);
+        await Promise.all([
+            this.process.waitForClose(),
+            (0, fs_1.waitForClose)(stream),
+            (0, fs_1.waitForClose)(this.writable),
+        ]);
+    }
+}
+class Std {
+    type;
+    process;
+    readable;
+    constructor(type, process, readable) {
+        this.type = type;
+        this.process = process;
+        this.readable = readable;
+    }
+    onData(cb) {
+        this.readable.on("data", cb);
+    }
+    async fetch() {
+        let data = "";
+        this.onData((chunk) => (data += chunk));
+        await this.process.waitForClose();
+        return data.trim();
+    }
+    async parseLines(cb) {
+        let total = 0;
+        const parser = (0, readline_1.createInterface)({ input: this.readable });
+        parser.on("line", (inLine) => {
+            const line = inLine.toString().trim();
+            if (line.length)
+                cb(line, ++total);
+        });
+        await Promise.all([this.process.waitForClose(), (0, fs_1.waitForClose)(parser)]);
+        return total;
+    }
+    async pipe(out, onProgress) {
+        if (this.process["log"]?.exec) {
+            const path = out instanceof StdIn
+                ? `${[out.process.child.pid || 0]}`
+                : typeof out === "string"
+                    ? out
+                    : "path" in out
+                        ? out.path
+                        : "&writableStream";
+            (0, cli_1.logExec)(`[${this.process.child.pid || 0}] ${this.type === "stderr" ? 2 : ""}> ${path}`);
+        }
+        if (onProgress) {
+            let totalBytes = 0;
+            this.readable.on("data", (chunk) => {
+                totalBytes += chunk.length;
+                onProgress({ totalBytes });
+            });
+        }
+        const stream = out instanceof StdIn
+            ? out.writable
+            : typeof out === "string"
+                ? (0, fs_2.createWriteStream)(out)
+                : out;
+        this.readable.pipe(stream);
+        await Promise.all([
+            this.process.waitForClose().catch((error) => {
+                if ("destroy" in stream)
+                    stream.destroy();
+                return Promise.reject(error);
+            }),
+            (0, fs_1.waitForClose)(stream),
+            (0, fs_1.waitForClose)(this.readable),
+        ]);
+    }
+}
+class AsyncProcess {
+    command;
+    argv;
+    options;
+    child;
+    stdout;
+    stderr;
+    stdin;
+    log;
+    controller;
+    lastStdError;
+    constructor(command, argv, options = {}) {
+        this.command = command;
+        this.argv = argv;
+        this.options = options;
+        const { $log, $controller, ...otherOptions } = options;
+        this.log = resolveLogOptions($log);
+        this.controller = $controller || new AbortController();
+        if (this.log?.exec)
+            (0, process_1.logProcess)(command, argv || [], this.log.exec === true ? {} : this.log.exec);
+        if (typeof options.cwd === "string")
+            ensureDir(options.cwd);
+        this.child = (0, child_process_1.spawn)(command, argv?.map(String) || [], otherOptions ?? {});
+        if (this.log)
+            this.installLog(this.log);
+        this.installAbortController(this.controller);
+        this.stdout = new Std("stdout", this, this.child.stdout);
+        this.stderr = new Std("stderr", this, this.child.stderr);
+        this.stdin = new StdIn(this, this.child.stdin);
+    }
+    static async exec(command, argv, options = {}) {
+        const p = new AsyncProcess(command, argv, options);
+        return await p.waitForClose();
+    }
+    static async stdout(command, argv, options = {}) {
+        const p = new AsyncProcess(command, argv, options);
+        return await p.stdout.fetch();
+    }
+    installLog(log) {
+        if (log.stdout)
+            this.child.stdout?.on("data", (chunk) => (0, process_1.logStdout)({
+                data: chunk.toString(),
+                colorize: log.colorize,
+                stderr: log.allToStderr,
+            }));
+        if (log.stderr)
+            this.child.stderr?.on("data", (chunk) => (0, process_1.logStdout)({
+                data: chunk.toString(),
+                colorize: log.colorize,
+                stderr: log.allToStderr,
+            }));
+    }
+    installAbortController(controller) {
+        if (controller.signal.aborted) {
+            this.child.kill();
+        }
+        else {
+            controller.signal.addEventListener("abort", () => {
+                this.child.kill();
+            });
+        }
+    }
+    resolveExitCode(inExitCode) {
+        let exitCode = inExitCode ?? 32;
+        if (!exitCode)
+            return exitCode;
+        let result = this.options.$exitCode ?? true;
+        let message = inExitCode === null ? "Process killed" : `Process exit code: ${exitCode}`;
+        if (typeof result === "function")
+            result = result(exitCode);
+        if (typeof result === "string") {
+            message = result;
+        }
+        else if (typeof result === "number") {
+            exitCode = result;
+        }
+        else if (result instanceof Error) {
+            return result;
+        }
+        else if (result === false) {
+            return exitCode;
+        }
+        return new Error(message, {
+            cause: {
+                command: this.command,
+                argv: this.argv,
+                exitCode,
+                lastStdError: this.lastStdError?.toString().trim().slice(0, 255),
+            },
+        });
+    }
+    async waitForClose() {
+        if (!this.lastStdError) {
+            this.lastStdError = Buffer.from([]);
+            this.child.stderr?.on("data", (chunk) => {
+                this.lastStdError = chunk;
+            });
+        }
+        return new Promise((resolve, reject) => {
+            this.child.on("error", reject).on("close", (exitCode) => {
+                const result = this.resolveExitCode(exitCode);
+                typeof result === "number" ? resolve(result) : reject(result);
+            });
+        });
+    }
+}
+exports.AsyncProcess = AsyncProcess;

package/lib/utils/async.d.ts

@@ -1,7 +1,5 @@
-type ControllerItem = {
-    stop?: () => void;
-};
-type ItemBuffer<T> = Map<T, ControllerItem>;
+/// <reference types="node" />
+type ItemBuffer<T> = Map<T, AbortController>;
 export declare function runParallel<T>(options: {
     items: T[];
     concurrency: number;
@@ -13,7 +11,7 @@ export declare function runParallel<T>(options: {
     onItem: (data: {
         item: T;
         index: number;
-        controller: ControllerItem;
+        controller: AbortController;
     }) => Promise<void> | void;
     onFinished?: () => Promise<void> | void;
 }): Promise<void>;

package/lib/utils/async.js

@@ -9,7 +9,7 @@ async function runParallel(options) {
     await promise_pool_1.PromisePool.for(options.items)
         .withConcurrency(options.concurrency)
         .process(async (item, index, pool) => {
-        const controller = {};
+        const controller = new AbortController();
         buffer.set(item, controller);
         try {
             await options.onChange({
@@ -29,7 +29,7 @@ async function runParallel(options) {
             pool.stop();
             for (const [, $controller] of buffer.entries())
                 try {
-                    $controller.stop?.();
+                    $controller.abort();
                 }
                 catch (_) { }
         }

package/lib/utils/datatruck/client.d.ts

@@ -23,7 +23,9 @@ export declare class RemoteFs extends AbstractFs {
     download(source: string, target: string, options?: {
         timeout?: number;
         onProgress?: (progress: BasicProgress) => void;
-    }): Promise<void>;
+    }): Promise<{
+        bytes: number;
+    }>;
 }
 export declare function isRemoteBackend(backend: string): boolean;
 export declare function createFs(backend: string): AbstractFs;

package/lib/utils/datatruck/client.js

@@ -80,7 +80,7 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         });
     }
     async download(source, target, options = {}) {
-        await (0, http_1.downloadFile)(`${this.url}/download`, target, {
+        return await (0, http_1.downloadFile)(`${this.url}/download`, target, {
             ...options,
             headers: this.headers,
             query: { params: JSON.stringify([source]) },

package/lib/utils/datatruck/config-type.d.ts

@@ -9,6 +9,7 @@ import type { DatatruckRepositoryServerOptions } from "./repository-server";
 export { RepositoryConfig, RepositoryConfigEnabledAction, TaskConfig };
 export type Config = {
     $schema?: string;
+    hostname?: string;
     tempDir?: string;
     minFreeDiskSpace?: string | number;
     repositories: RepositoryConfig[];

package/lib/utils/datatruck/cron-server.js

@@ -2,9 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createCronServer = void 0;
 const ConfigAction_1 = require("../../actions/ConfigAction");
+const async_process_1 = require("../async-process");
 const cli_1 = require("../cli");
 const cron_1 = require("../cron");
-const process_1 = require("../process");
 const string_1 = require("../string");
 const watcher_1 = require("../watcher");
 const command_1 = require("./command");
@@ -33,7 +33,7 @@ function createCronServer(options, config) {
             const command = new Command({ config: { packages: [], repositories: [] } }, {});
             const cliOptions = (0, cli_1.stringifyOptions)(command.optionsConfig(), action.options);
             const [node, bin] = process.argv;
-            await (0, process_1.exec)(node, [bin, "-c", config.configPath, action.name, ...cliOptions], {}, { log: config.verbose });
+            await async_process_1.AsyncProcess.exec(node, [bin, "-c", config.configPath, action.name, ...cliOptions], { $log: config.verbose });
             if (config.log)
                 console.info(`< [job] ${index} - ${action.name}`);
         }

package/lib/utils/datatruck/report-list.d.ts

@@ -6,6 +6,8 @@ export type ReportListTaskContext = {
     };
 };
 export declare function createReportListTasks<T extends ReportListTaskContext>(list: Listr3<T>, options: {
+    action: string;
+    hostname: string;
     reports: DatatruckReportConfig[];
     onMessage: (result: (List3SummaryResult | Listr3TaskResult<T>)[], report: DatatruckReportConfig) => string;
     verbose?: boolean;

package/lib/utils/datatruck/report-list.js

@@ -41,7 +41,7 @@ function createReportListTasks(list, options) {
                 else if ((0, reportSteps_1.isReportStep)(report.run)) {
                     await (0, reportSteps_1.runReportSteps)(report.run, {
                         data: {
-                            title: "DTT Backup",
+                            title: `[${options.hostname}] DTT ${options.action}`,
                             message,
                             success,
                         },

package/lib/utils/fs.d.ts

@@ -3,7 +3,7 @@
 /// <reference types="node" />
 import { Progress } from "./progress";
 import { Entry, Options } from "fast-glob";
-import { ReadStream, Stats } from "fs";
+import { Stats } from "fs";
 import { WriteStream } from "fs";
 import { Interface } from "readline";
 export declare const isWSLSystem: boolean;
@@ -38,7 +38,12 @@ export declare function writeGitIgnoreList(options: {
     paths: NodeJS.ReadableStream | string[];
     outDir: string;
 }): Promise<string>;
-export declare function waitForClose(stream: WriteStream | ReadStream): Promise<void>;
+export declare function waitForClose(stream: {
+    on(event: "close", cb: (...args: any[]) => any): any;
+} | {
+    on(event: "close", cb: (...args: any[]) => any): any;
+    on(event: "error", cb: (...args: any[]) => any): any;
+}): Promise<void>;
 export declare function copyFileWithStreams(source: string, target: string): Promise<unknown>;
 export declare function updateFileStats(path: string, fileInfo: Stats): Promise<void>;
 export declare function isNotFoundError(error: unknown): boolean;

package/lib/utils/fs.js

@@ -250,7 +250,6 @@ async function waitForClose(stream) {
     return new Promise((resolve, reject) => {
         stream.on("close", resolve);
         stream.on("error", reject);
-        return stream;
     });
 }
 exports.waitForClose = waitForClose;

package/lib/utils/http.d.ts

@@ -22,7 +22,9 @@ export declare function downloadFile(url: string, output: string, options?: {
     query?: Record<string, string>;
     timeout?: number;
     onProgress?: (progress: BasicProgress) => void;
-}): Promise<void>;
+}): Promise<{
+    bytes: number;
+}>;
 export declare function uploadFile(url: string, path: string, options?: {
     headers?: Record<string, string>;
     query?: Record<string, string>;