@datatruck/cli 0.26.2 → 0.28.0

package/utils/list.js

@@ -0,0 +1,145 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Listr3 = exports.List3Logger = void 0;
+const date_1 = require("./date");
+const stream_1 = require("./stream");
+const listr2_1 = require("listr2");
+class List3Logger extends listr2_1.ListrLogger {
+    constructor(options = {}) {
+        const streams = (0, stream_1.createStreams)(options.streams);
+        super({
+            processOutput: new listr2_1.ProcessOutput(streams.stdout, streams.stderr),
+        });
+    }
+}
+exports.List3Logger = List3Logger;
+class Listr3 extends listr2_1.Listr {
+    $options;
+    resultMap = {};
+    resultList = [];
+    execTimer;
+    constructor($options) {
+        super([], {
+            renderer: "default",
+            collectErrors: "minimal",
+            ...($options.progressManager && {
+                fallbackRendererCondition: () => !$options.progressManager.tty,
+            }),
+            fallbackRenderer: "simple",
+            fallbackRendererOptions: {
+                logger: new List3Logger(),
+                timestamp: listr2_1.PRESET_TIMESTAMP,
+                timer: listr2_1.PRESET_TIMER,
+            },
+            rendererOptions: {
+                logger: new List3Logger(),
+                collapseSubtasks: false,
+                collapseErrors: false,
+                timer: listr2_1.PRESET_TIMER,
+            },
+        });
+        this.$options = $options;
+        this.execTimer = (0, date_1.createTimer)();
+    }
+    serializeKeyIndex(keyIndex) {
+        return keyIndex
+            ? Array.isArray(keyIndex)
+                ? keyIndex.map((k) => k.toString())
+                : [keyIndex.toString()]
+            : [];
+    }
+    createResultIndex(key, keyIndex) {
+        return [key, ...this.serializeKeyIndex(keyIndex)].join(".");
+    }
+    result(key, keyIndex) {
+        const index = this.createResultIndex(key, keyIndex);
+        const result = this.resultMap[index];
+        if (!result)
+            throw new Error(`Task result not found: ${index}`);
+        return result;
+    }
+    $task(item) {
+        const index = this.createResultIndex(item.key, item.keyIndex);
+        if (this.resultMap[index])
+            throw new Error(`Duplicated task index: ${index}`);
+        this.resultMap[index] = {
+            key: item.key,
+            keyIndex: item.keyIndex
+                ? this.serializeKeyIndex(item.keyIndex).join(".")
+                : undefined,
+            elapsed: 0,
+            error: undefined,
+            data: item.data,
+        };
+        this.resultList.push(this.resultMap[index]);
+        const title = typeof item.title === "string" ? { initial: item.title } : item.title;
+        return {
+            title: title.initial,
+            exitOnError: item.exitOnError,
+            enabled: item.enabled,
+            skip: item.skip,
+            task: async (_, task) => {
+                const result = this.result(item.key, item.keyIndex);
+                if (title.started)
+                    task.title = title.started;
+                const timer = (0, date_1.createTimer)();
+                if (title)
+                    try {
+                        const runResult = item.runWrapper
+                            ? await item.runWrapper(async () => await item.run(task, result.data))
+                            : await item.run(task, result.data);
+                        if (title.completed)
+                            task.title = title.completed;
+                        return Array.isArray(runResult)
+                            ? task.newListr(runResult)
+                            : runResult;
+                    }
+                    catch (error) {
+                        result.error = error;
+                        if (title.failed)
+                            task.title = title.failed;
+                        throw error;
+                    }
+                    finally {
+                        result.elapsed = timer.elapsed();
+                    }
+            },
+        };
+    }
+    $tasks(...items) {
+        return items
+            .map((item) => (item ? ("key" in item ? this.$task(item) : item) : null))
+            .filter(Boolean);
+    }
+    add(tasks) {
+        super.add(tasks);
+        return this;
+    }
+    getSummaryResult() {
+        return {
+            key: "summary",
+            elapsed: this.execTimer.elapsed(),
+            data: {
+                errors: this.resultList.filter((i) => i.error).length,
+            },
+        };
+    }
+    getResult() {
+        return [...this.resultList, this.getSummaryResult()];
+    }
+    async exec() {
+        try {
+            this.$options.progressManager?.start();
+            this.execTimer.reset();
+            await super.run();
+            return this.getResult();
+        }
+        catch (error) {
+            throw error;
+        }
+        finally {
+            this.$options.progressManager?.dispose();
+        }
+    }
+}
+exports.Listr3 = Listr3;

package/utils/mysql.d.ts

@@ -9,6 +9,7 @@ export type MysqlCliOptions = {
     verbose?: boolean;
     database?: string;
 };
+export declare function assertDumpFile(path: string): Promise<void>;
 export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
     options: MysqlCliOptions;
     initSharedDir: (sharedDir?: string) => Promise<string>;
@@ -28,6 +29,7 @@ export declare function createMysqlCli(options: MysqlCliOptions): Promise<{
             totalBytes: number;
         }) => void) | undefined;
     }) => Promise<[void, import("./process").ExecResultType]>;
+    assertDumpFile: typeof assertDumpFile;
     fetchTableNames: (database: string, include?: string[], exclude?: string[]) => Promise<string[]>;
     importFile: (input: {
         path: string;

package/utils/mysql.js

@@ -1,11 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createMysqlCli = void 0;
+exports.createMysqlCli = exports.assertDumpFile = void 0;
 const AppError_1 = require("../Error/AppError");
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const process_1 = require("./process");
 const string_1 = require("./string");
+const temp_1 = require("./temp");
 const crypto_1 = require("crypto");
 const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
@@ -22,6 +23,23 @@ function flatQuery(query, params) {
         })
         : query;
 }
+async function assertDumpFile(path) {
+    const headerContents = await (0, fs_1.readPartialFile)(path, [0, 100]);
+    const footerContents = await (0, fs_1.readPartialFile)(path, [-100]);
+    const successHeader = headerContents.split(/\r?\n/).some((line) => {
+        const firstLine = line.trim().toLowerCase();
+        return (firstLine.startsWith("-- mysql dump") ||
+            firstLine.startsWith("-- mariadb dump"));
+    });
+    if (!successHeader)
+        throw new AppError_1.AppError("No start line found");
+    const successFooter = footerContents
+        .split(/\r?\n/)
+        .some((line) => line.trim().toLowerCase().startsWith("-- dump completed"));
+    if (!successFooter)
+        throw new AppError_1.AppError("No end line found (incomplete backup)");
+}
+exports.assertDumpFile = assertDumpFile;
 async function createMysqlCli(options) {
     let sqlConfigPath;
     const password = (await (0, fs_1.fetchData)(options.password, (p) => p.path)) ?? "";
@@ -35,7 +53,7 @@ async function createMysqlCli(options) {
     async function createSqlConfig() {
         if (sqlConfigPath)
             return sqlConfigPath;
-        const dir = await (0, fs_1.mkTmpDir)("mysql-cli");
+        const dir = await (0, temp_1.mkTmpDir)("mysql", "config");
         const password = await (0, fs_1.fetchData)(options.password, (p) => p.path);
         const data = [
             `[client]`,
@@ -266,6 +284,7 @@ async function createMysqlCli(options) {
         changeDatabase,
         fetchAll,
         dump,
+        assertDumpFile,
         fetchTableNames,
         importFile,
         isDatabaseEmpty,

package/utils/process.d.ts

@@ -38,6 +38,7 @@ export interface ExecSettingsInterface {
     };
     stderr?: {
         save?: boolean;
+        parseLines?: boolean | "skip-empty";
         onData?: (data: string) => void;
         toExitCode?: boolean;
     };

package/utils/process.js

@@ -189,7 +189,7 @@ async function exec(command, argv = [], options = null, settings = {}) {
             });
         }
         if (typeof options?.cwd === "string" && !(await (0, fs_1.existsDir)(options.cwd)))
-            throw new Error(`Current working directory does not exist: ${options.cwd}`);
+            return reject(new Error(`Current working directory does not exist: ${options.cwd}`));
         if (pipe?.stream instanceof fs_2.ReadStream && "onReadProgress" in pipe) {
             const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
             const totalBytes = fileInfo.size;
@@ -216,6 +216,8 @@ async function exec(command, argv = [], options = null, settings = {}) {
             finishListeners++;
         if (settings.stdout?.parseLines)
             finishListeners++;
+        if (settings.stderr?.parseLines)
+            finishListeners++;
         let streamError;
         const tryFinish = () => {
             if (!--finishListeners)
@@ -282,10 +284,18 @@ async function exec(command, argv = [], options = null, settings = {}) {
                 pipe.stream.pipe(p.stdin);
             }
         }
-        if (log.stdout || settings.stdout) {
-            if (!p.stdout)
-                throw new Error(`stdout is not defined`);
-            const parseLines = settings.stdout?.parseLines;
+        const stdConfig = {
+            stdout: { log: log.stdout, settings: settings.stdout },
+            stderr: { log: log.stderr, settings: settings.stderr },
+        };
+        for (const inType in stdConfig) {
+            const type = inType;
+            const enabled = log[type] || settings[type];
+            if (!enabled)
+                continue;
+            if (!p[type])
+                throw new Error(`${type} is not defined`);
+            const parseLines = settings[type]?.parseLines;
             const skipEmptyLines = parseLines === "skip-empty";
             const onData = (inData) => {
                 let data = inData.toString();
@@ -294,46 +304,29 @@ async function exec(command, argv = [], options = null, settings = {}) {
                         return;
                     data = `${inData}\n`;
                 }
-                if (log.stdout)
+                if (log[type])
                     logExecStdout({
                         data,
                         stderr: log.allToStderr,
                         colorize: log.colorize,
                     });
-                if (settings.stdout?.save)
-                    spawnData.stdout += data;
-                if (settings.stdout?.onData)
-                    settings.stdout.onData(inData.toString());
+                if (settings[type]?.save ||
+                    (type === "stderr" && settings[type]?.toExitCode))
+                    spawnData[type] += data;
+                if (settings[type]?.onData)
+                    settings[type].onData(inData.toString());
             };
             if (parseLines) {
                 const rl = (0, readline_1.createInterface)({
-                    input: p.stdout,
+                    input: p[type],
                 });
                 rl.on("line", onData);
                 rl.on("close", tryFinish);
             }
-            else if (log.stdout ||
-                settings.stdout?.save ||
-                settings.stdout?.onData) {
-                p.stdout.on("data", onData);
+            else if (log[type] || settings[type]?.save || settings[type]?.onData) {
+                p[type].on("data", onData);
             }
         }
-        if (log.stderr || settings.stderr) {
-            if (!p.stderr)
-                throw new Error(`stderr is not defined`);
-            p.stderr.on("data", (data) => {
-                if (log.stderr)
-                    logExecStdout({
-                        data: data.toString(),
-                        stderr: log.allToStderr,
-                        colorize: log.colorize,
-                    });
-                if (settings.stderr?.save || settings.stderr?.toExitCode)
-                    spawnData.stderr += data.toString();
-                if (settings.stderr?.onData)
-                    settings.stderr.onData(data.toString());
-            });
-        }
         p.on("error", (error) => (spawnError = error)).on("close", (exitCode) => {
             spawnData.exitCode = exitCode ?? 0;
             if (pipe?.stream instanceof fs_2.WriteStream)

package/utils/progress.d.ts

@@ -1,3 +1,5 @@
+/// <reference types="node" />
+import { Timer } from "./date";
 export type ProgressStats = {
     percent?: number;
     total?: number;
@@ -10,3 +12,34 @@ export type Progress = {
     absolute?: ProgressStats;
     relative?: ProgressStats;
 };
+export declare class ProgressManager {
+    readonly options: {
+        verbose?: boolean;
+        /**
+         * @default true
+         */
+        tty?: boolean | "auto";
+        enabled?: boolean | "auto" | "interval";
+        interval?: number;
+    };
+    protected timer: Timer;
+    protected interval: Timer | undefined;
+    protected keydownListener: ((data: Buffer) => void) | undefined;
+    readonly tty: boolean;
+    readonly enabled: boolean | "interval";
+    constructor(options: {
+        verbose?: boolean;
+        /**
+         * @default true
+         */
+        tty?: boolean | "auto";
+        enabled?: boolean | "auto" | "interval";
+        interval?: number;
+    });
+    elapsed(): number;
+    start(): void;
+    dispose(): void;
+    update(progress: Progress, cb: (text: string) => void): void;
+}
+export declare function renderProgress(progress: Progress, bar?: boolean): string[];
+export declare function renderProgressStats(stats: ProgressStats, progressBar?: boolean): string;

package/utils/progress.js

@@ -1,2 +1,115 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderProgressStats = exports.renderProgress = exports.ProgressManager = void 0;
+const cli_1 = require("./cli");
+const date_1 = require("./date");
+const bytes_1 = __importDefault(require("bytes"));
+const chalk_1 = require("chalk");
+const readline_1 = require("readline");
+class ProgressManager {
+    options;
+    timer = (0, date_1.createTimer)();
+    interval = (0, date_1.createTimer)();
+    keydownListener;
+    tty;
+    enabled;
+    constructor(options) {
+        this.options = options;
+        this.tty =
+            options.tty === "auto"
+                ? options.verbose
+                    ? false
+                    : process.stdout.isTTY
+                : !!options.tty;
+        this.enabled =
+            options.enabled === "auto"
+                ? this.tty
+                    ? true
+                    : "interval"
+                : !!options.enabled;
+    }
+    elapsed() {
+        return this.timer.elapsed();
+    }
+    start() {
+        this.timer.reset();
+        (0, readline_1.emitKeypressEvents)(process.stdin);
+        process.stdin?.setRawMode?.(true);
+        process.stdin?.resume();
+        process.stdin?.setEncoding("utf8");
+        process.stdin?.on("keypress", (this.keydownListener = (inKey) => {
+            const key = inKey.toString();
+            if (key === "\u0003") {
+                process.stdin.setRawMode?.(false);
+                process.emit("SIGINT");
+            }
+            else if (/^(\r\n)|\r|\n$/.test(key)) {
+                this.interval = undefined;
+            }
+        }));
+    }
+    dispose() {
+        this.timer.stop();
+        if (this.keydownListener) {
+            process.stdin?.off("keypress", this.keydownListener);
+            this.keydownListener = undefined;
+        }
+    }
+    update(progress, cb) {
+        if (!this.enabled)
+            return;
+        if (this.enabled === "interval") {
+            if (this.interval) {
+                if (!this.interval.reset(this.options.interval ?? 5000))
+                    return;
+            }
+            else {
+                this.interval = (0, date_1.createTimer)();
+            }
+        }
+        cb(renderProgress(progress, this.tty).join("\n"));
+    }
+}
+exports.ProgressManager = ProgressManager;
+function renderProgress(progress, bar) {
+    return [
+        progress.absolute && renderProgressStats(progress.absolute, bar),
+        progress.relative && renderProgressStats(progress.relative, bar),
+    ].filter((v) => !!v);
+}
+exports.renderProgress = renderProgress;
+function renderProgressStats(stats, progressBar) {
+    const text = [];
+    if (typeof stats.percent === "number") {
+        if (progressBar)
+            text.push((0, cli_1.renderProgressBar)(stats.percent));
+        text.push(`${stats.percent.toFixed(2)}%`);
+    }
+    if (typeof stats.current === "number" || typeof stats.total === "number") {
+        const format = (value) => stats.format === "size" ? (0, bytes_1.default)(value) : value;
+        if (typeof stats.current === "number" && typeof stats.total === "number") {
+            text.push(`${format(stats.current)}/${format(stats.total)}`);
+        }
+        else if (typeof stats.current === "number") {
+            text.push(`${format(stats.current)}`);
+        }
+        else if (typeof stats.total === "number") {
+            text.push(`?/${format(stats.total)}`);
+        }
+    }
+    if (stats.description && stats.payload) {
+        text.push(`${stats.description}: ${stats.payload}`);
+    }
+    else if (stats.description) {
+        text.push(stats.description);
+    }
+    else if (stats.payload) {
+        text.push(stats.payload);
+    }
+    const sep = (0, chalk_1.grey)(`|`);
+    return text.join(` ${sep} `);
+}
+exports.renderProgressStats = renderProgressStats;

package/utils/steps.d.ts

@@ -9,12 +9,20 @@ export type NodeStepConfig = {
     vars?: Record<string, any>;
     code: string | string[];
 };
+export type TelegramMessageStepConfig = {
+    bot: string;
+    chatId: number;
+    text?: string;
+};
 export type Step = {
     type: "process";
     config: ProcessStepConfig;
 } | {
     type: "node";
     config: NodeStepConfig;
+} | {
+    type: "telegram-message";
+    config: TelegramMessageStepConfig;
 };
 export type StepOptions = {
     env?: Record<string, string | undefined>;
@@ -25,6 +33,9 @@ export type StepOptions = {
         vars?: Record<string, any>;
         tempDir?: () => Promise<string>;
     };
+    telegram?: {
+        vars?: Record<string, string>;
+    };
     cwd?: string;
     onLine?: (p: string) => any;
     verbose?: boolean;

package/utils/steps.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runSteps = void 0;
-const fs_1 = require("./fs");
+const http_1 = require("./http");
 const process_1 = require("./process");
 const string_1 = require("./string");
+const temp_1 = require("./temp");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 async function runSteps(input, options) {
@@ -33,19 +34,19 @@ async function runSteps(input, options) {
                 tempDir = await options.node.tempDir();
             }
             else {
-                tempDir = await (0, fs_1.mkTmpDir)("node-step");
+                tempDir = await (0, temp_1.mkTmpDir)("node-step");
             }
             const scriptPath = (0, path_1.join)(tempDir, "script.js");
-            const vars = Object.entries({
+            const vars = {
                 ...step.config.vars,
                 ...options.node?.vars,
-            }).reduce((items, [name, value]) => {
-                items.push(`let ${name} = ${JSON.stringify(value)}`);
-                return items;
-            }, []);
-            await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
-                ? [...vars, ...step.config.code].join(";\n")
-                : `${vars.join(";\n")}\n${step.config.code}`);
+            };
+            const varKeys = Object.keys(vars);
+            const varJson = JSON.stringify(vars);
+            const code = Array.isArray(step.config.code)
+                ? [...step.config.code].join(";\n")
+                : step.config.code;
+            await (0, promises_1.writeFile)(scriptPath, `(async function({ ${varKeys} }) {\n${code};\n})(${varJson});`);
             await (0, process_1.exec)("node", [scriptPath], {
                 cwd: options.cwd,
                 env: {
@@ -63,6 +64,17 @@ async function runSteps(input, options) {
                 }),
             });
         }
+        else if (step.type === "telegram-message") {
+            await (0, http_1.post)(`https://api.telegram.org/bot${step.config.bot}/sendMessage`, JSON.stringify({
+                chat_id: step.config.chatId.toString(),
+                text: (0, string_1.render)(step.config.text ?? `{TEXT}`, options.telegram?.vars || {}),
+                disable_notification: true,
+            }), {
+                headers: {
+                    "Content-Type": "application/json",
+                },
+            });
+        }
         else {
             throw new Error(`Invalid step type: ${step.type}`);
         }

package/utils/stream.d.ts

@@ -0,0 +1,7 @@
+/// <reference types="node" />
+import { Writable } from "stream";
+export type Streams = {
+    stdout: Writable;
+    stderr: Writable;
+};
+export declare function createStreams(options?: Partial<Streams>): Streams;

package/utils/stream.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createStreams = void 0;
+function createStreams(options = {}) {
+    return {
+        stdout: options.stdout ?? process.stdout,
+        stderr: options.stderr ?? process.stderr,
+    };
+}
+exports.createStreams = createStreams;

package/utils/string.d.ts

@@ -19,6 +19,5 @@ export declare function endsWith(input: string, patterns: string[]): boolean;
 export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;
-export declare function splitLines(input: string, satinize?: boolean): string[];
 export declare function undefIfEmpty(input: string): string | undefined;
 export {};

package/utils/string.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.undefIfEmpty = exports.splitLines = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
+exports.undefIfEmpty = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -121,18 +121,6 @@ function formatDateTime(datetime) {
     return result;
 }
 exports.formatDateTime = formatDateTime;
-function splitLines(input, satinize = true) {
-    const lines = input.split(/\r?\n/);
-    return satinize
-        ? input.split(/\r?\n/).reduce((result, value) => {
-            value = value.trim();
-            if (value.length)
-                result.push(value);
-            return result;
-        }, [])
-        : lines;
-}
-exports.splitLines = splitLines;
 function undefIfEmpty(input) {
     return input.length ? input : undefined;
 }

package/utils/tar.d.ts

@@ -24,14 +24,17 @@ export type DecompressOptions = {
      */
     cores?: CoresOptions;
 };
-export interface CreateTarOptions {
+export type CreateTarOptions = {
     path: string;
     verbose?: boolean;
     output: string;
-    includeList: string;
     compress?: boolean | CompressOptions;
     onEntry?: (entry: TarEntry) => void;
-}
+} & ({
+    includeList: string;
+} | {
+    include: string[];
+});
 export interface ExtractOptions {
     input: string;
     output: string;
@@ -51,4 +54,8 @@ export type ListTarOptions = {
 export declare function listTar(options: ListTarOptions): Promise<number>;
 export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
 export declare function createTar(options: CreateTarOptions): Promise<void>;
+/**
+ * bsdtar (only windows?) fails if path ends with slash
+ */
+export declare function normalizeTarPath(path: string): string;
 export declare function extractTar(options: ExtractOptions): Promise<void>;