@datatruck/cli 0.27.0 → 0.28.0

package/utils/progress.js

@@ -1,2 +1,115 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderProgressStats = exports.renderProgress = exports.ProgressManager = void 0;
+const cli_1 = require("./cli");
+const date_1 = require("./date");
+const bytes_1 = __importDefault(require("bytes"));
+const chalk_1 = require("chalk");
+const readline_1 = require("readline");
+class ProgressManager {
+    options;
+    timer = (0, date_1.createTimer)();
+    interval = (0, date_1.createTimer)();
+    keydownListener;
+    tty;
+    enabled;
+    constructor(options) {
+        this.options = options;
+        this.tty =
+            options.tty === "auto"
+                ? options.verbose
+                    ? false
+                    : process.stdout.isTTY
+                : !!options.tty;
+        this.enabled =
+            options.enabled === "auto"
+                ? this.tty
+                    ? true
+                    : "interval"
+                : !!options.enabled;
+    }
+    elapsed() {
+        return this.timer.elapsed();
+    }
+    start() {
+        this.timer.reset();
+        (0, readline_1.emitKeypressEvents)(process.stdin);
+        process.stdin?.setRawMode?.(true);
+        process.stdin?.resume();
+        process.stdin?.setEncoding("utf8");
+        process.stdin?.on("keypress", (this.keydownListener = (inKey) => {
+            const key = inKey.toString();
+            if (key === "\u0003") {
+                process.stdin.setRawMode?.(false);
+                process.emit("SIGINT");
+            }
+            else if (/^(\r\n)|\r|\n$/.test(key)) {
+                this.interval = undefined;
+            }
+        }));
+    }
+    dispose() {
+        this.timer.stop();
+        if (this.keydownListener) {
+            process.stdin?.off("keypress", this.keydownListener);
+            this.keydownListener = undefined;
+        }
+    }
+    update(progress, cb) {
+        if (!this.enabled)
+            return;
+        if (this.enabled === "interval") {
+            if (this.interval) {
+                if (!this.interval.reset(this.options.interval ?? 5000))
+                    return;
+            }
+            else {
+                this.interval = (0, date_1.createTimer)();
+            }
+        }
+        cb(renderProgress(progress, this.tty).join("\n"));
+    }
+}
+exports.ProgressManager = ProgressManager;
+function renderProgress(progress, bar) {
+    return [
+        progress.absolute && renderProgressStats(progress.absolute, bar),
+        progress.relative && renderProgressStats(progress.relative, bar),
+    ].filter((v) => !!v);
+}
+exports.renderProgress = renderProgress;
+function renderProgressStats(stats, progressBar) {
+    const text = [];
+    if (typeof stats.percent === "number") {
+        if (progressBar)
+            text.push((0, cli_1.renderProgressBar)(stats.percent));
+        text.push(`${stats.percent.toFixed(2)}%`);
+    }
+    if (typeof stats.current === "number" || typeof stats.total === "number") {
+        const format = (value) => stats.format === "size" ? (0, bytes_1.default)(value) : value;
+        if (typeof stats.current === "number" && typeof stats.total === "number") {
+            text.push(`${format(stats.current)}/${format(stats.total)}`);
+        }
+        else if (typeof stats.current === "number") {
+            text.push(`${format(stats.current)}`);
+        }
+        else if (typeof stats.total === "number") {
+            text.push(`?/${format(stats.total)}`);
+        }
+    }
+    if (stats.description && stats.payload) {
+        text.push(`${stats.description}: ${stats.payload}`);
+    }
+    else if (stats.description) {
+        text.push(stats.description);
+    }
+    else if (stats.payload) {
+        text.push(stats.payload);
+    }
+    const sep = (0, chalk_1.grey)(`|`);
+    return text.join(` ${sep} `);
+}
+exports.renderProgressStats = renderProgressStats;

package/utils/steps.d.ts

@@ -9,12 +9,20 @@ export type NodeStepConfig = {
     vars?: Record<string, any>;
     code: string | string[];
 };
+export type TelegramMessageStepConfig = {
+    bot: string;
+    chatId: number;
+    text?: string;
+};
 export type Step = {
     type: "process";
     config: ProcessStepConfig;
 } | {
     type: "node";
     config: NodeStepConfig;
+} | {
+    type: "telegram-message";
+    config: TelegramMessageStepConfig;
 };
 export type StepOptions = {
     env?: Record<string, string | undefined>;
@@ -25,6 +33,9 @@ export type StepOptions = {
         vars?: Record<string, any>;
         tempDir?: () => Promise<string>;
     };
+    telegram?: {
+        vars?: Record<string, string>;
+    };
     cwd?: string;
     onLine?: (p: string) => any;
     verbose?: boolean;

package/utils/steps.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runSteps = void 0;
-const fs_1 = require("./fs");
+const http_1 = require("./http");
 const process_1 = require("./process");
 const string_1 = require("./string");
+const temp_1 = require("./temp");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 async function runSteps(input, options) {
@@ -33,19 +34,19 @@ async function runSteps(input, options) {
                 tempDir = await options.node.tempDir();
             }
             else {
-                tempDir = await (0, fs_1.mkTmpDir)("node-step");
+                tempDir = await (0, temp_1.mkTmpDir)("node-step");
             }
             const scriptPath = (0, path_1.join)(tempDir, "script.js");
-            const vars = Object.entries({
+            const vars = {
                 ...step.config.vars,
                 ...options.node?.vars,
-            }).reduce((items, [name, value]) => {
-                items.push(`let ${name} = ${JSON.stringify(value)}`);
-                return items;
-            }, []);
-            await (0, promises_1.writeFile)(scriptPath, Array.isArray(step.config.code)
-                ? [...vars, ...step.config.code].join(";\n")
-                : `${vars.join(";\n")}\n${step.config.code}`);
+            };
+            const varKeys = Object.keys(vars);
+            const varJson = JSON.stringify(vars);
+            const code = Array.isArray(step.config.code)
+                ? [...step.config.code].join(";\n")
+                : step.config.code;
+            await (0, promises_1.writeFile)(scriptPath, `(async function({ ${varKeys} }) {\n${code};\n})(${varJson});`);
             await (0, process_1.exec)("node", [scriptPath], {
                 cwd: options.cwd,
                 env: {
@@ -63,6 +64,17 @@ async function runSteps(input, options) {
                 }),
             });
         }
+        else if (step.type === "telegram-message") {
+            await (0, http_1.post)(`https://api.telegram.org/bot${step.config.bot}/sendMessage`, JSON.stringify({
+                chat_id: step.config.chatId.toString(),
+                text: (0, string_1.render)(step.config.text ?? `{TEXT}`, options.telegram?.vars || {}),
+                disable_notification: true,
+            }), {
+                headers: {
+                    "Content-Type": "application/json",
+                },
+            });
+        }
         else {
             throw new Error(`Invalid step type: ${step.type}`);
         }

package/utils/stream.d.ts

@@ -0,0 +1,7 @@
+/// <reference types="node" />
+import { Writable } from "stream";
+export type Streams = {
+    stdout: Writable;
+    stderr: Writable;
+};
+export declare function createStreams(options?: Partial<Streams>): Streams;

package/utils/stream.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createStreams = void 0;
+function createStreams(options = {}) {
+    return {
+        stdout: options.stdout ?? process.stdout,
+        stderr: options.stderr ?? process.stderr,
+    };
+}
+exports.createStreams = createStreams;

package/utils/string.d.ts

@@ -19,6 +19,5 @@ export declare function endsWith(input: string, patterns: string[]): boolean;
 export declare function createMatchFilter(include?: string[], exclude?: string[]): (input: string) => boolean;
 export declare function checkMatch(subject: string | undefined, patterns: string[]): boolean;
 export declare function formatDateTime(datetime: string): string;
-export declare function splitLines(input: string, satinize?: boolean): string[];
 export declare function undefIfEmpty(input: string): string | undefined;
 export {};

package/utils/string.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.undefIfEmpty = exports.splitLines = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
+exports.undefIfEmpty = exports.formatDateTime = exports.checkMatch = exports.createMatchFilter = exports.endsWith = exports.match = exports.makePathPatterns = exports.formatSeconds = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.serialize = void 0;
 const AppError_1 = require("../Error/AppError");
 const micromatch_1 = require("micromatch");
 function serialize(message, data) {
@@ -121,18 +121,6 @@ function formatDateTime(datetime) {
     return result;
 }
 exports.formatDateTime = formatDateTime;
-function splitLines(input, satinize = true) {
-    const lines = input.split(/\r?\n/);
-    return satinize
-        ? input.split(/\r?\n/).reduce((result, value) => {
-            value = value.trim();
-            if (value.length)
-                result.push(value);
-            return result;
-        }, [])
-        : lines;
-}
-exports.splitLines = splitLines;
 function undefIfEmpty(input) {
     return input.length ? input : undefined;
 }

package/utils/tar.d.ts

@@ -24,14 +24,17 @@ export type DecompressOptions = {
      */
     cores?: CoresOptions;
 };
-export interface CreateTarOptions {
+export type CreateTarOptions = {
     path: string;
     verbose?: boolean;
     output: string;
-    includeList: string;
     compress?: boolean | CompressOptions;
     onEntry?: (entry: TarEntry) => void;
-}
+} & ({
+    includeList: string;
+} | {
+    include: string[];
+});
 export interface ExtractOptions {
     input: string;
     output: string;
@@ -51,4 +54,8 @@ export type ListTarOptions = {
 export declare function listTar(options: ListTarOptions): Promise<number>;
 export declare function checkPigzLib(cache?: boolean): Promise<boolean>;
 export declare function createTar(options: CreateTarOptions): Promise<void>;
+/**
+ * bsdtar (only windows?) fails if path ends with slash
+ */
+export declare function normalizeTarPath(path: string): string;
 export declare function extractTar(options: ExtractOptions): Promise<void>;

package/utils/tar.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
+exports.extractTar = exports.normalizeTarPath = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
@@ -52,13 +52,18 @@ async function getTarVendor(cache = true, log = false) {
 }
 exports.getTarVendor = getTarVendor;
 async function listTar(options) {
+    const vendor = await getTarVendor(true, options.verbose);
     let total = 0;
-    await (0, process_1.exec)("tar", ["-tf", toLocalPath(options.input), "--force-local"], {}, {
+    await (0, process_1.exec)("tar", [
+        "-tf",
+        toLocalPath(options.input),
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
+    ], {}, {
         log: options.verbose,
         stdout: {
             parseLines: "skip-empty",
             onData: (path) => {
-                options.onEntry?.({ path });
+                options.onEntry?.({ path: normalizeTarPath(path) });
                 total++;
             },
         },
@@ -91,7 +96,9 @@ async function ifX(input, cb) {
 }
 async function createTar(options) {
     const vendor = await getTarVendor(true, options.verbose);
-    const total = await (0, fs_1.countFileLines)(options.includeList);
+    const total = "include" in options
+        ? options.include.length
+        : await (0, fs_1.countFileLines)(options.includeList);
     const compress = await ifX(options.compress, async (compress) => ({
         ...compress,
         cores: await resolveCores(compress.cores),
@@ -103,16 +110,27 @@ async function createTar(options) {
             GZIP_OPT: compress.level.toString(),
         }),
     };
+    const onData = (line) => {
+        current++;
+        let path = vendor === "bsdtar" ? line.slice(2) : line;
+        options.onEntry?.({
+            path: normalizeTarPath(path),
+            progress: {
+                total,
+                current,
+                percent: (0, math_1.progressPercent)(total, current),
+            },
+        });
+    };
     await (0, process_1.exec)("tar", [
         "--no-recursion",
         "-C",
         toLocalPath(options.path),
         compress?.cores === 1 ? "-czvf" : "-cvf",
         toLocalPath(options.output),
-        "-T",
-        toLocalPath(options.includeList),
-        "--ignore-failed-read",
-        "--force-local",
+        // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=172293
+        ...(vendor === "bsdtar" ? [] : ["--ignore-failed-read"]),
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
         ...(compress && compress.cores > 1
             ? [
                 "-I",
@@ -126,6 +144,9 @@ async function createTar(options) {
                     .join(" ")}"`,
             ]
             : []),
+        ...("includeList" in options
+            ? ["-T", toLocalPath(options.includeList)]
+            : ["--", ...options.include]),
     ], {
         ...(compress &&
             compress.cores > 1 && {
@@ -137,24 +158,16 @@ async function createTar(options) {
         },
     }, {
         log: options.verbose ? { envNames: Object.keys(env) } : false,
-        stderr: { toExitCode: true },
-        stdout: options.onEntry
+        ...(vendor === "bsdtar"
             ? {
-                parseLines: "skip-empty",
-                onData: (line) => {
-                    current++;
-                    const path = vendor === "bsdtar" ? line.slice(2) : line;
-                    options.onEntry?.({
-                        path,
-                        progress: {
-                            total,
-                            current,
-                            percent: (0, math_1.progressPercent)(total, current),
-                        },
-                    });
-                },
+                stderr: options.onEntry
+                    ? { toExitCode: true, parseLines: "skip-empty", onData }
+                    : { toExitCode: true },
             }
-            : undefined,
+            : {
+                stderr: { toExitCode: true },
+                stdout: { parseLines: "skip-empty", onData },
+            }),
     });
 }
 exports.createTar = createTar;
@@ -164,9 +177,18 @@ exports.createTar = createTar;
 function toLocalPath(path) {
     return (0, os_1.platform)() === "win32" ? path.replace(/\\/g, "/") : path;
 }
+/**
+ * bsdtar (only windows?) fails if path ends with slash
+ */
+function normalizeTarPath(path) {
+    return path.endsWith("/") ? path.slice(0, -1) : path;
+}
+exports.normalizeTarPath = normalizeTarPath;
 async function extractTar(options) {
-    let total = options.total ??
-        (await listTar({ input: options.input, verbose: options.verbose }));
+    let total = options.onEntry
+        ? options.total ??
+            (await listTar({ input: options.input, verbose: options.verbose }))
+        : undefined;
     if (!(await (0, fs_1.existsDir)(options.output))) {
         if (options.verbose)
             (0, cli_1.logExec)("mkdir", ["-p", options.output]);
@@ -177,12 +199,25 @@ async function extractTar(options) {
         cores: await resolveCores(decompress.cores),
     }));
     let current = 0;
+    const vendor = await getTarVendor(true, options.verbose);
+    const onData = (line) => {
+        const path = vendor === "bsdtar" ? line.slice(2) : line;
+        current++;
+        options.onEntry?.({
+            path: normalizeTarPath(path),
+            progress: {
+                total: total,
+                current,
+                percent: (0, math_1.progressPercent)(total, current),
+            },
+        });
+    };
     await (0, process_1.exec)("tar", [
         decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
         toLocalPath(options.input),
         "-C",
         toLocalPath(options.output),
-        "--force-local",
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
         ...(decompress && decompress.cores > 1
             ? ["-I", `"pigz -p ${decompress.cores}"`]
             : []),
@@ -193,25 +228,16 @@ async function extractTar(options) {
         }),
     }, {
         log: options.verbose,
-        stderr: {
-            toExitCode: true,
-        },
-        stdout: options.onEntry
+        ...(vendor === "bsdtar"
             ? {
-                parseLines: "skip-empty",
-                onData: (path) => {
-                    current++;
-                    options.onEntry?.({
-                        path,
-                        progress: {
-                            total,
-                            current,
-                            percent: (0, math_1.progressPercent)(total, current),
-                        },
-                    });
-                },
+                stderr: options.onEntry
+                    ? { toExitCode: true, parseLines: "skip-empty", onData }
+                    : { toExitCode: true },
             }
-            : undefined,
+            : {
+                stderr: { toExitCode: true },
+                stdout: { parseLines: "skip-empty", onData },
+            }),
     });
 }
 exports.extractTar = extractTar;

package/utils/temp.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+export declare function parentTmpDir(): string;
+export declare function sessionTmpDir(): string;
+export declare function ensureFreeDiskTempSpace(size: number | string): Promise<void>;
+export declare function isTmpDir(path: string): boolean;
+export declare function rmTmpDir(input: string | string[]): Promise<void>;
+export declare function tmpDir(...keys: [string, ...string[]]): string;
+export declare function mkTmpDir(...keys: [string, ...string[]]): Promise<string>;
+export declare function useTempDir(...keys: [string, ...string[]]): Promise<AsyncDisposable & {
+    path: string;
+}>;
+export declare function useTempFile(path: string): AsyncDisposable & {
+    path: string;
+};
+export declare class CleanupListener {
+    readonly paths: string[];
+    stop(): void;
+    dispose(): Promise<void>;
+}
+export declare class GargabeCollector {
+    protected listeners: Set<CleanupListener>;
+    get pending(): boolean;
+    cleanup(cb?: () => any): Promise<void>;
+    cleanupOnFinish(cb: () => any): Promise<void>;
+    cleanupIfFail(cb: () => any): Promise<CleanupListener>;
+}

package/utils/temp.js

@@ -0,0 +1,133 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GargabeCollector = exports.CleanupListener = exports.useTempFile = exports.useTempDir = exports.mkTmpDir = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.ensureFreeDiskTempSpace = exports.sessionTmpDir = exports.parentTmpDir = void 0;
+const globalData_1 = __importDefault(require("../globalData"));
+const fs_1 = require("./fs");
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+Symbol.dispose ??= Symbol("Symbol.dispose");
+Symbol.asyncDispose ??= Symbol("Symbol.asyncDispose");
+function parentTmpDir() {
+    return (0, path_1.join)(globalData_1.default.tempDir, "datatruck-temp");
+}
+exports.parentTmpDir = parentTmpDir;
+function sessionTmpDir() {
+    return (0, path_1.join)(parentTmpDir(), process.pid.toString());
+}
+exports.sessionTmpDir = sessionTmpDir;
+async function ensureFreeDiskTempSpace(size) {
+    const path = sessionTmpDir();
+    await (0, fs_1.mkdirIfNotExists)(sessionTmpDir());
+    await (0, fs_1.ensureFreeDiskSpace)([path], size);
+}
+exports.ensureFreeDiskTempSpace = ensureFreeDiskTempSpace;
+function isTmpDir(path) {
+    return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
+}
+exports.isTmpDir = isTmpDir;
+async function rmTmpDir(input) {
+    if (typeof input === "string") {
+        if (!isTmpDir(input))
+            throw new Error(`Path is not a temp dir: ${input}`);
+        await (0, promises_1.rm)(input, { recursive: true });
+    }
+    else {
+        for (const path of input)
+            await rmTmpDir(path);
+    }
+}
+exports.rmTmpDir = rmTmpDir;
+function tmpDir(...keys) {
+    const id = (0, crypto_1.randomUUID)().slice(0, 8);
+    const path = (0, path_1.join)(sessionTmpDir(), [...keys, id].map(encodeURIComponent).join("-"));
+    for (const listener of listeners)
+        listener.paths.push(path);
+    return path;
+}
+exports.tmpDir = tmpDir;
+const listeners = new Set();
+async function mkTmpDir(...keys) {
+    const path = tmpDir(...keys);
+    await (0, promises_1.mkdir)(path, { recursive: true });
+    return path;
+}
+exports.mkTmpDir = mkTmpDir;
+async function useTempDir(...keys) {
+    const path = await mkTmpDir(...keys);
+    return {
+        path,
+        async [Symbol.asyncDispose]() {
+            try {
+                await rmTmpDir(path);
+            }
+            catch (_) { }
+        },
+    };
+}
+exports.useTempDir = useTempDir;
+function useTempFile(path) {
+    return {
+        path,
+        async [Symbol.asyncDispose]() {
+            try {
+                await (0, promises_1.rm)(path, { recursive: true });
+            }
+            catch (_) { }
+        },
+    };
+}
+exports.useTempFile = useTempFile;
+class CleanupListener {
+    paths = [];
+    stop() {
+        listeners.delete(this);
+    }
+    async dispose() {
+        this.stop();
+        await rmTmpDir(this.paths);
+    }
+}
+exports.CleanupListener = CleanupListener;
+class GargabeCollector {
+    listeners = new Set();
+    get pending() {
+        return this.listeners.size > 0;
+    }
+    async cleanup(cb) {
+        try {
+            await cb?.();
+        }
+        finally {
+            for (const listener of this.listeners) {
+                this.listeners.delete(listener);
+                await listener.dispose();
+            }
+        }
+    }
+    async cleanupOnFinish(cb) {
+        const cleanup = new CleanupListener();
+        try {
+            await cb();
+        }
+        finally {
+            cleanup.dispose();
+        }
+    }
+    async cleanupIfFail(cb) {
+        const cleanup = new CleanupListener();
+        try {
+            await cb();
+        }
+        catch (error) {
+            await cleanup.dispose();
+            throw error;
+        }
+        this.listeners.add(cleanup);
+        return cleanup;
+    }
+}
+exports.GargabeCollector = GargabeCollector;

package/utils/virtual-fs.d.ts

@@ -1,3 +1,4 @@
+import { DiskStats } from "./fs";
 export declare function resolvePath(path: string): string;
 export type FsOptions = {
     backend: string;
@@ -7,6 +8,7 @@ export declare abstract class AbstractFs {
     constructor(options: FsOptions);
     resolvePath(path: string): string;
     abstract isLocal(): boolean;
+    isRemote(): boolean;
     abstract existsDir(path: string): Promise<boolean>;
     abstract mkdir(path: string): Promise<void>;
     abstract readFile(path: string): Promise<string>;
@@ -15,8 +17,9 @@ export declare abstract class AbstractFs {
     abstract readdir(path: string): Promise<string[]>;
     abstract ensureEmptyDir(path: string): Promise<void>;
     abstract writeFile(path: string, contents: string): Promise<void>;
-    abstract upload(url: string, path: string): Promise<void>;
-    abstract download(url: string, path: string): Promise<void>;
+    abstract upload(source: string, target: string): Promise<void>;
+    abstract download(source: string, target: string): Promise<void>;
+    abstract fetchDiskStats(source: string): Promise<DiskStats>;
 }
 export declare class LocalFs extends AbstractFs {
     isLocal(): boolean;
@@ -28,6 +31,7 @@ export declare class LocalFs extends AbstractFs {
     readdir(path: string): Promise<string[]>;
     writeFile(path: string, contents: string): Promise<void>;
     rmAll(path: string): Promise<void>;
+    fetchDiskStats(source: string): Promise<DiskStats>;
     upload(source: string, target: string): Promise<void>;
     download(source: string, target: string): Promise<void>;
 }