@datatruck/cli 0.0.1

package/lib/util/process-util.d.ts

@@ -0,0 +1,50 @@
+/// <reference types="node" />
+import { SpawnOptions, ChildProcess } from "child_process";
+import { ReadStream, WriteStream } from "fs";
+export declare type ExecLogSettingsType = {
+    colorize?: boolean;
+    exec?: boolean;
+    stdout?: boolean;
+    stderr?: boolean;
+    allToStderr?: boolean;
+    envNames?: string[];
+};
+export interface ExecSettingsInterface {
+    exec?: boolean;
+    pipe?: {
+        stream: WriteStream | ReadStream;
+        onReadProgress?: (data: {
+            totalBytes: number;
+            currentBytes: number;
+            progress: number;
+        }) => void;
+    };
+    log?: ExecLogSettingsType | boolean;
+    onSpawn?: (p: ChildProcess) => void;
+    stdout?: {
+        save?: boolean;
+        onData?: (data: string) => Promise<void>;
+    };
+    stderr?: {
+        save?: boolean;
+        onData?: (data: string) => Promise<void>;
+        toExitCode?: boolean;
+    };
+    onExitCodeError?: (data: ExecResultType, error: Error) => Error | false;
+}
+export declare function logExecStdout(input: {
+    data: string;
+    colorize?: boolean;
+    stderr?: boolean;
+    lineSalt?: boolean;
+}): void;
+export declare function logExecStderr(data: string, colorize?: boolean): void;
+export declare type ExecResultType = {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+};
+export declare function exec(command: string, argv?: string[], options?: SpawnOptions | null, settings?: ExecSettingsInterface): Promise<ExecResultType>;
+declare type EventNameType = "exit" | "SIGINT" | "SIGUSR1" | "SIGUSR2" | "SIGTERM" | "uncaughtException";
+export declare function onExit(cb: (eventName: EventNameType) => void): void;
+export {};

package/lib/util/process-util.js

@@ -0,0 +1,181 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.onExit = exports.exec = exports.logExecStderr = exports.logExecStdout = void 0;
+const cli_util_1 = require("./cli-util");
+const fs_util_1 = require("./fs-util");
+const math_util_1 = require("./math-util");
+const chalk_1 = __importDefault(require("chalk"));
+const child_process_1 = require("child_process");
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+function logExecStdout(input) {
+    let text = input.colorize ? chalk_1.default.grey(input.data) : input.data;
+    if (input.lineSalt)
+        text += "\n";
+    input.stderr ? process.stderr.write(text) : process.stdout.write(text);
+}
+exports.logExecStdout = logExecStdout;
+function logExecStderr(data, colorize) {
+    process.stdout.write(colorize ? chalk_1.default.red(data) : data);
+}
+exports.logExecStderr = logExecStderr;
+async function exec(command, argv = [], options = null, settings = {}) {
+    const pipe = settings.pipe;
+    let log = {};
+    if (settings.log === true) {
+        log.exec = log.stdout = log.stderr = log.allToStderr = log.colorize = true;
+    }
+    else if (settings.log) {
+        log = settings.log;
+    }
+    return new Promise(async (resolve, reject) => {
+        if (log.exec) {
+            const logEnv = log.envNames?.reduce((env, key) => {
+                env[key] = options?.env?.[key] ?? "";
+                return env;
+            }, {});
+            (0, cli_util_1.logExec)(command, pipe
+                ? [
+                    ...argv,
+                    pipe.stream instanceof fs_1.ReadStream ? "<" : ">",
+                    String(pipe.stream.path),
+                ]
+                : argv, logEnv, log.allToStderr);
+        }
+        if (typeof options?.cwd === "string" && !(await (0, fs_util_1.checkDir)(options.cwd)))
+            return reject(new Error(`Current working directory does not exist: ${options.cwd}`));
+        if (pipe?.onReadProgress && pipe.stream instanceof fs_1.ReadStream) {
+            const fileInfo = await (0, promises_1.stat)(pipe.stream.path);
+            const totalBytes = fileInfo.size;
+            let currentBytes = 0;
+            pipe.stream.on("data", (data) => {
+                currentBytes += data.length;
+                pipe.onReadProgress?.({
+                    totalBytes: totalBytes,
+                    currentBytes: currentBytes,
+                    progress: (0, math_util_1.progressPercent)(totalBytes, currentBytes),
+                });
+            });
+        }
+        const p = (0, child_process_1.spawn)(command, argv, options ?? {});
+        settings.onSpawn?.(p);
+        let spawnError;
+        const spawnData = {
+            stdout: "",
+            stderr: "",
+            exitCode: 0,
+        };
+        let finishListens = pipe ? 2 : 1;
+        let streamError;
+        const tryFinish = () => {
+            if (!--finishListens)
+                finish();
+        };
+        const finish = () => {
+            if (spawnData.exitCode) {
+                let exitCodeError;
+                if (settings.stderr?.toExitCode) {
+                    exitCodeError = new Error(`Exit code ${spawnData.exitCode}: ${spawnData.stderr
+                        .split(/\r?\n/g)
+                        .filter((v) => !!v.length)
+                        .join(" | ")}`);
+                }
+                else {
+                    exitCodeError = new Error(`Exit code: ${spawnData.exitCode} (${command} ${argv.join(" ")})`);
+                }
+                const exitCodeErrorResult = settings.onExitCodeError?.(spawnData, exitCodeError);
+                if (exitCodeErrorResult instanceof Error) {
+                    return reject(exitCodeErrorResult);
+                }
+                else if (exitCodeErrorResult !== false) {
+                    return reject(exitCodeError);
+                }
+            }
+            if (streamError) {
+                reject(streamError);
+            }
+            else if (spawnError) {
+                reject(spawnError);
+            }
+            else {
+                resolve(spawnData);
+            }
+        };
+        if (pipe) {
+            pipe.stream.on("error", (error) => {
+                streamError = error;
+                tryFinish();
+            });
+            if (pipe.stream instanceof fs_1.WriteStream) {
+                if (!p.stdout)
+                    throw new Error(`stdout is not defined`);
+                if (!p.stderr)
+                    throw new Error(`stderr is not defined`);
+                p.stdout.pipe(pipe.stream, { end: false });
+                p.stderr.pipe(pipe.stream, { end: false });
+                p.on("close", tryFinish);
+            }
+            else if (pipe.stream instanceof fs_1.ReadStream) {
+                if (!p.stdin)
+                    throw new Error(`stdin is not defined`);
+                pipe.stream.pipe(p.stdin);
+            }
+        }
+        if (log.stdout || settings.stdout) {
+            if (!p.stdout)
+                throw new Error(`stdout is not defined`);
+            p.stdout.on("data", async (data) => {
+                if (log.stdout)
+                    logExecStdout({
+                        data: data.toString(),
+                        stderr: log.allToStderr,
+                        colorize: log.colorize,
+                    });
+                if (settings.stdout?.save)
+                    spawnData.stdout += data.toString();
+                if (settings.stdout?.onData)
+                    await settings.stdout.onData(data.toString());
+            });
+        }
+        if (log.stderr || settings.stderr) {
+            if (!p.stderr)
+                throw new Error(`stderr is not defined`);
+            p.stderr.on("data", async (data) => {
+                if (log.stderr)
+                    logExecStdout({
+                        data: data.toString(),
+                        stderr: log.allToStderr,
+                        colorize: log.colorize,
+                    });
+                if (settings.stderr?.save || settings.stderr?.toExitCode)
+                    spawnData.stderr += data.toString();
+                if (settings.stderr?.onData)
+                    await settings.stderr.onData(data.toString());
+            });
+        }
+        p.on("error", (error) => (spawnError = error)).on("close", (exitCode) => {
+            spawnData.exitCode = exitCode ?? 0;
+            if (pipe?.stream instanceof fs_1.WriteStream)
+                pipe.stream.end();
+            tryFinish();
+        });
+    });
+}
+exports.exec = exec;
+const eventNames = [
+    `exit`,
+    `SIGINT`,
+    `SIGUSR1`,
+    `SIGUSR2`,
+    `uncaughtException`,
+    `SIGTERM`,
+];
+function onExit(cb) {
+    for (const eventName of eventNames) {
+        process.on(eventName, cb.bind(null, eventName));
+    }
+}
+exports.onExit = onExit;

package/lib/util/string-util.d.ts

@@ -0,0 +1,17 @@
+export declare function serialize(message: string, data?: Object): string;
+export declare function ucfirst(value: string): string;
+export declare function lcfirst(value: string): string;
+export declare function snakeCase(value: string, char?: string): string;
+export declare function render(subject: string, vars: Record<string, string | undefined>): string;
+export declare function parseStringList(value: string, validValues?: string[]): string[];
+export declare type UriType = {
+    protocol?: "http" | "https";
+    host?: string;
+    username?: string;
+    password?: string;
+    port?: number;
+    path?: string;
+};
+export declare function formatUri(input: UriType, hidePassword?: boolean): string;
+export declare function makePathPatterns(values: string[] | undefined): string[] | undefined;
+export declare function formatDateTime(datetime: string): string;

package/lib/util/string-util.js

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatDateTime = exports.makePathPatterns = exports.formatUri = exports.parseStringList = exports.render = exports.snakeCase = exports.lcfirst = exports.ucfirst = exports.serialize = void 0;
+const AppError_1 = require("../Error/AppError");
+function serialize(message, data) {
+    if (data)
+        return `${message} (${JSON.stringify(data, null, 2)})`;
+    return message;
+}
+exports.serialize = serialize;
+function ucfirst(value) {
+    return value.charAt(0).toUpperCase() + value.slice(1);
+}
+exports.ucfirst = ucfirst;
+function lcfirst(value) {
+    return value.charAt(0).toLowerCase() + value.slice(1);
+}
+exports.lcfirst = lcfirst;
+function snakeCase(value, char = "_") {
+    return value.replace(/[A-Z]/g, (letter) => `${char}${letter.toLowerCase()}`);
+}
+exports.snakeCase = snakeCase;
+function render(subject, vars) {
+    return subject.replace(/{(\w+)}/g, function (match, name) {
+        const value = vars[name];
+        if (typeof value === "undefined")
+            throw new AppError_1.AppError(`Variable is not defined: '${subject}' (${name})`);
+        return value;
+    });
+}
+exports.render = render;
+function parseStringList(value, validValues) {
+    const result = value
+        ?.split(",")
+        .map((v) => v.trim())
+        .filter((v) => !!v.length) ?? null;
+    if (validValues)
+        for (const v of result)
+            if (!validValues.includes(v))
+                throw new AppError_1.AppError(`Invalid value: ${v}`);
+    return result;
+}
+exports.parseStringList = parseStringList;
+function formatUri(input, hidePassword) {
+    let uri = "";
+    if (input.protocol) {
+        uri = `${input.protocol}://`;
+        if (input.username) {
+            uri += `${input.username}`;
+            if (input.password)
+                uri += `:${hidePassword ? "********" : input.password}`;
+            uri += `@`;
+        }
+        if (input.host)
+            uri += input.host;
+        if (input.port)
+            uri += `:${input.port}`;
+    }
+    if (input.path)
+        uri += input.path;
+    return uri;
+}
+exports.formatUri = formatUri;
+function makePathPatterns(values) {
+    return values?.flatMap((v) => [v, `${v}/**`]);
+}
+exports.makePathPatterns = makePathPatterns;
+function formatDateTime(datetime) {
+    const date = new Date(datetime);
+    const [result] = new Date(date.getTime() - date.getTimezoneOffset() * 60000)
+        .toISOString()
+        .replace("Z", "")
+        .replace("T", " ")
+        .split(".");
+    return result;
+}
+exports.formatDateTime = formatDateTime;

package/lib/util/zip-util.d.ts

@@ -0,0 +1,52 @@
+export interface ZipDataFilterType {
+    recursive?: boolean;
+    exclude?: boolean;
+    patterns: string[];
+}
+export declare type ZipStreamDataType = {
+    type: "progress";
+    data: {
+        progress: number;
+        files: number;
+        path: string;
+    };
+} | {
+    type: "summary";
+    data: {
+        folders: number;
+        files: number;
+    };
+};
+export interface ZipDataType {
+    command?: string;
+    path: string;
+    filter?: (ZipDataFilterType | string)[];
+    output: string;
+    deleteOnZip?: boolean;
+    includeList?: string;
+    excludeList?: string;
+    verbose?: boolean;
+    onStream?: (data: ZipStreamDataType) => Promise<void>;
+}
+export interface UnzipDataType {
+    command?: string;
+    input: string;
+    files?: (ZipDataFilterType | string)[];
+    output: string;
+    verbose?: boolean;
+    onStream?: (data: UnzipStreamDataType) => Promise<void>;
+}
+export declare type UnzipStreamDataType = {
+    type: "progress";
+    data: {
+        progress: number;
+        files: number;
+        path: string;
+    };
+};
+export declare function buildArguments(filters: (ZipDataFilterType | string)[]): string[];
+export declare function zip(data: ZipDataType): Promise<{
+    folders: number;
+    files: number;
+}>;
+export declare function unzip(data: UnzipDataType): Promise<import("./process-util").ExecResultType>;

package/lib/util/zip-util.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unzip = exports.zip = exports.buildArguments = void 0;
+const process_util_1 = require("./process-util");
+const path_1 = require("path");
+function buildArguments(filters) {
+    const args = [];
+    for (const item of filters) {
+        let filter;
+        if (typeof item === "string") {
+            filter = {
+                recursive: false,
+                exclude: false,
+                patterns: [item],
+            };
+        }
+        else {
+            filter = item;
+        }
+        args.push(...filter.patterns.map((value) => {
+            let option = "-";
+            option += filter.exclude ? "x" : "i";
+            if (filter.recursive)
+                option += "r";
+            option += "!";
+            option += (0, path_1.normalize)(value);
+            return option;
+        }));
+    }
+    return args;
+}
+exports.buildArguments = buildArguments;
+async function parseZipStream(chunk, buffer, cb) {
+    const lines = chunk.trim().split(/\r?\n/g);
+    for (const line of lines) {
+        let matches = null;
+        if ((matches = /^(\d+)% (\d+ )?\+/.exec(line))) {
+            const path = line.slice(line.indexOf("+") + 1).trim();
+            const progress = Number(matches[1]);
+            if (!buffer.currentPaths)
+                buffer.currentPaths = 0;
+            if (path !== buffer.lastPath)
+                buffer.currentPaths++;
+            buffer.lastPath = path;
+            await cb({
+                type: "progress",
+                data: { progress, path, files: buffer.currentPaths },
+            });
+        }
+        else if (line.startsWith("Add new data to archive:")) {
+            const [, folders] = /(\d+) folders?/i.exec(line) || [, 0];
+            const [, files] = /(\d+) files?/i.exec(line) || [, 0];
+            await cb({
+                type: "summary",
+                data: {
+                    folders: Number(folders),
+                    files: Number(files),
+                },
+            });
+        }
+    }
+}
+async function zip(data) {
+    let result = {
+        folders: 0,
+        files: 0,
+    };
+    let buffer = {};
+    await (0, process_util_1.exec)(data.command ?? "7z", [
+        "a",
+        "-mmt1",
+        "-bsp1",
+        ...(data.deleteOnZip ? ["-sdel"] : []),
+        (0, path_1.normalize)(data.output),
+        ...buildArguments(data.filter ?? []),
+        ...(data.includeList ? [`@${(0, path_1.normalize)(data.includeList)}`] : []),
+        ...(data.excludeList ? [`-x@${(0, path_1.normalize)(data.excludeList)}`] : []),
+    ], {
+        cwd: data.path,
+    }, {
+        log: data.verbose ?? false,
+        stderr: {
+            toExitCode: true,
+        },
+        stdout: {
+            onData: async (chunk) => {
+                parseZipStream(chunk, buffer, async (stream) => {
+                    await data.onStream?.(stream);
+                    if (stream.type === "summary")
+                        result = stream.data;
+                });
+            },
+        },
+    });
+    return result;
+}
+exports.zip = zip;
+async function parseUnzipStream(chunk, cb) {
+    const lines = chunk.trim().split(/\r?\n/g);
+    for (const line of lines) {
+        let matches = null;
+        if ((matches = /^(\d+)% (\d+) \-/.exec(line))) {
+            const progress = Number(matches[1]);
+            const files = Number(matches[2]);
+            const path = line.slice(line.indexOf("-") + 1).trim();
+            await cb({
+                type: "progress",
+                data: { progress, path, files },
+            });
+        }
+    }
+}
+async function unzip(data) {
+    return await (0, process_util_1.exec)(data.command ?? "7z", [
+        "x",
+        "-mmt1",
+        "-bsp1",
+        (0, path_1.normalize)(data.input),
+        ...buildArguments(data.files ?? []),
+        `-o${(0, path_1.normalize)(data.output)}`,
+        "-r",
+    ], {}, {
+        log: data.verbose ?? false,
+        stderr: { toExitCode: true },
+        stdout: {
+            ...(data.onStream && {
+                onData: async (chunk) => {
+                    if (data.onStream)
+                        parseUnzipStream(chunk, data.onStream);
+                },
+            }),
+        },
+    });
+}
+exports.unzip = unzip;

package/migrations/001-initial.sql

@@ -0,0 +1,122 @@
+--------------------------------------------------------------------------------
+-- Up
+--------------------------------------------------------------------------------
+
+CREATE TABLE "backup_session" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"snapshotId"	TEXT NOT NULL,
+	"packageName"	TEXT NOT NULL,
+	-- "componentName"	TEXT NOT NULL,
+	"tags"	TEXT
+);
+
+CREATE TABLE "backup_session_task" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"sessionId"	INTEGER NOT NULL,
+	"taskName"	TEXT NOT NULL
+);
+
+CREATE TABLE "backup_session_repository" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"sessionId"	INTEGER NOT NULL,
+	"repositoryName"	TEXT NOT NULL,
+	"repositoryType"	TEXT NOT NULL
+);
+
+CREATE TABLE "restore_session" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"snapshotId"	TEXT NOT NULL,
+	"packageName"	TEXT NOT NULL -- ,
+	-- "componentName"	TEXT NOT NULL
+);
+
+CREATE TABLE "restore_session_task" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"sessionId"	INTEGER NOT NULL,
+	"taskName"	TEXT NOT NULL
+);
+
+CREATE TABLE "restore_session_repository" (
+	"id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+	"creationDate"	TEXT NOT NULL,
+	"updatingDate"	TEXT,
+
+	"startDate"	TEXT,
+	"endDate"	TEXT,
+	"state"	TEXT,
+	"error"	TEXT,
+	"progressCurrent" INTEGER,
+	"progressTotal" INTEGER,
+	"progressPercent" INTEGER,
+	"progressStep" TEXT,
+	"progressStepPercent" INTEGER,
+
+	"sessionId"	INTEGER NOT NULL,
+	"repositoryName"	TEXT NOT NULL,
+	"repositoryType"	TEXT NOT NULL
+);

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@datatruck/cli",
+  "version": "0.0.1",
+  "description": "Tool for creating and managing backups",
+  "homepage": "https://github.com/swordev/datatruck#readme",
+  "bugs": {
+    "url": "https://github.com/swordev/datatruck/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/swordev/datatruck"
+  },
+  "license": "MIT",
+  "author": {
+    "name": "Juanra GM",
+    "email": "juanrgm724@gmail.com",
+    "url": "https://github.com/juanrgm"
+  },
+  "exports": {
+    "./*": "./lib/*.js"
+  },
+  "main": "lib/index.js",
+  "bin": {
+    "datatruck": "lib/bin.js",
+    "dtt": "lib/bin.js"
+  },
+  "files": [
+    "lib",
+    "migrations"
+  ],
+  "dependencies": {
+    "ajv": "^8.10.0",
+    "chalk": "^4.1.2",
+    "cli-table3": "^0.6.1",
+    "commander": "^9.0.0",
+    "concurrently": "^7.0.0",
+    "dayjs": "^1.10.7",
+    "fast-glob": "^3.2.11",
+    "fs-extra": "^10.0.1",
+    "micromatch": "^4.0.4",
+    "sqlite": "^4.0.23",
+    "sqlite3": "^5.0.2"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^9.0.13",
+    "@types/json-schema": "^7.0.9",
+    "@types/micromatch": "^4.0.2",
+    "json-schema": "^0.4.0"
+  },
+  "optionalDependencies": {
+    "ts-node": "^10.5.0",
+    "yaml": "^1.10.2"
+  },
+  "engine": {
+    "node": ">=16.0.0"
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "clean": "tsc --build --clean",
+    "watch": "tsc --build -w"
+  }
+}