@datatruck/cli 0.39.1 → 0.40.1

package/config.schema.json

@@ -409,30 +409,47 @@
                   "config": {
                     "type": "object",
                     "properties": {
-                      "command": {
-                        "type": "string"
-                      },
-                      "hostname": {
-                        "type": "string"
-                      },
-                      "port": {
-                        "type": "number"
-                      },
-                      "username": {
-                        "type": "string"
-                      },
-                      "password": {
+                      "uri": {
                         "anyOf": [
                           {
                             "type": "object",
                             "properties": {
-                              "path": {
+                              "host": {
+                                "type": "string"
+                              },
+                              "username": {
+                                "type": "string"
+                              },
+                              "password": {
+                                "anyOf": [
+                                  {
+                                    "type": "object",
+                                    "properties": {
+                                      "path": {
+                                        "type": "string"
+                                      }
+                                    },
+                                    "additionalProperties": false,
+                                    "required": [
+                                      "path"
+                                    ]
+                                  },
+                                  {
+                                    "type": "string"
+                                  }
+                                ]
+                              },
+                              "port": {
+                                "type": "number"
+                              },
+                              "database": {
                                 "type": "string"
                               }
                             },
                             "additionalProperties": false,
                             "required": [
-                              "path"
+                              "database",
+                              "host"
                             ]
                           },
                           {
@@ -440,14 +457,32 @@
                           }
                         ]
                       },
+                      "command": {
+                        "type": "string"
+                      },
                       "compress": {
                         "type": "boolean"
                       },
                       "concurrency": {
                         "type": "number"
+                      },
+                      "targetDatabase": {
+                        "type": "object",
+                        "properties": {
+                          "name": {
+                            "type": "string"
+                          }
+                        },
+                        "additionalProperties": false,
+                        "required": [
+                          "name"
+                        ]
                       }
                     },
-                    "additionalProperties": false
+                    "additionalProperties": false,
+                    "required": [
+                      "uri"
+                    ]
                   }
                 },
                 "additionalProperties": false,

package/lib/tasks/MongoDumpTask.d.ts

@@ -1,14 +1,13 @@
-import { TaskBackupData, TaskRestoreData, TaskAbstract } from "./TaskAbstract";
+import { MongoUriObject } from "../utils/mongodb";
+import { TaskBackupData, TaskRestoreData, TaskAbstract, TaskPrepareRestoreData } from "./TaskAbstract";
 export type MongoDumpTaskConfig = {
+    uri: string | MongoUriObject;
     command?: string;
-    hostname?: string;
-    port?: number;
-    username?: string;
-    password?: string | {
-        path: string;
-    };
     compress?: boolean;
     concurrency?: number;
+    targetDatabase?: {
+        name: string;
+    };
 };
 export declare const mongodumpTaskName = "mongo-dump";
 export declare class MongoDumpTask extends TaskAbstract<MongoDumpTaskConfig> {
@@ -17,5 +16,8 @@ export declare class MongoDumpTask extends TaskAbstract<MongoDumpTaskConfig> {
     backup(data: TaskBackupData): Promise<{
         snapshotPath: string;
     }>;
+    prepareRestore(data: TaskPrepareRestoreData): Promise<{
+        snapshotPath: string;
+    }>;
     restore(data: TaskRestoreData): Promise<void>;
 }

package/lib/tasks/MongoDumpTask.js

@@ -2,9 +2,15 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MongoDumpTask = exports.mongodumpTaskName = void 0;
 const async_process_1 = require("../utils/async-process");
+const config_1 = require("../utils/datatruck/config");
+const error_1 = require("../utils/error");
 const fs_1 = require("../utils/fs");
+const mongodb_1 = require("../utils/mongodb");
 const temp_1 = require("../utils/temp");
 const TaskAbstract_1 = require("./TaskAbstract");
+const promises_1 = require("fs/promises");
+const mongodb_2 = require("mongodb");
+const path_1 = require("path");
 exports.mongodumpTaskName = "mongo-dump";
 class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
     verbose;
@@ -17,21 +23,19 @@ class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
             (await (0, temp_1.mkTmpDir)(exports.mongodumpTaskName, "task", "backup", "snapshot"));
         await (0, fs_1.mkdirIfNotExists)(snapshotPath);
         await (0, fs_1.ensureEmptyDir)(snapshotPath);
+        const config = await (0, mongodb_1.resolveMongoUri)(this.config.uri);
         const p = new async_process_1.AsyncProcess(this.command, [
-            ...(this.config.hostname ? ["/h", this.config.hostname] : []),
-            ...(this.config.port ? ["/p", this.config.port] : []),
-            ...(this.config.username ? ["/u", this.config.username] : []),
+            ...(config.host ? ["/h", config.host] : []),
+            ...(config.port ? [`/port:${config.port}`] : []),
+            ...["/authenticationDatabase:admin"],
+            ...["/d", config.database],
+            ...(config.username ? ["/u", config.username] : []),
             ...(this.config.compress ? ["/gzip"] : []),
             ...(this.config.concurrency ? ["/j", this.config.concurrency] : []),
             "/o",
             snapshotPath,
-        ], {
-            $log: this.verbose,
-        });
-        const password = this.config.password !== undefined
-            ? (await (0, fs_1.fetchData)(this.config.password, (p) => p.path)) ?? ""
-            : "";
-        p.stdin.writable.write(`${password}\n`);
+        ], { $log: this.verbose });
+        p.stdin.writable.write(`${config.password ?? ""}\n`);
         await p.stderr.parseLines((line) => {
             data.onProgress({
                 absolute: {
@@ -39,10 +43,57 @@ class MongoDumpTask extends TaskAbstract_1.TaskAbstract {
                 },
             });
         });
+        const tmpDir = (0, path_1.join)(snapshotPath, config.database);
+        for (const file of await (0, promises_1.readdir)(tmpDir))
+            await (0, promises_1.rename)((0, path_1.join)(tmpDir, file), (0, path_1.join)(snapshotPath, file));
+        await (0, promises_1.rmdir)(tmpDir);
         return { snapshotPath };
     }
+    async prepareRestore(data) {
+        return {
+            snapshotPath: data.package.restorePath ??
+                (await (0, temp_1.mkTmpDir)(exports.mongodumpTaskName, "task", "restore", "snapshot")),
+        };
+    }
     async restore(data) {
-        throw new Error("Not implemented");
+        this.verbose = data.options.verbose;
+        const config = await (0, mongodb_1.resolveMongoUri)(this.config.uri);
+        const uri = (0, mongodb_1.toMongoUri)(config);
+        const client = new mongodb_2.MongoClient(`${uri}?authSource=admin`);
+        const params = {
+            packageName: data.package.name,
+            snapshotId: data.options.id,
+            snapshotDate: data.snapshot.date,
+            action: "restore",
+            database: undefined,
+        };
+        const database = {
+            name: (0, config_1.resolveDatabaseName)(config.database, params),
+        };
+        if (this.config.targetDatabase && !data.options.initial)
+            database.name = (0, config_1.resolveDatabaseName)(this.config.targetDatabase.name, {
+                ...params,
+                database: database.name,
+            });
+        const restoreCollections = (await (0, promises_1.readdir)(data.snapshotPath))
+            .filter((name) => name.endsWith(".bson"))
+            .map((name) => name.replace(/\.bson$/, ""));
+        const collections = (await client.db(database.name).collections()).map((v) => v.collectionName);
+        const duplicatedCollections = restoreCollections.filter((v) => collections.includes(v));
+        if (duplicatedCollections.length)
+            throw new error_1.AppError(`Target collections already exists: ${duplicatedCollections.join(", ")}`);
+        const p = new async_process_1.AsyncProcess("mongorestore", [
+            ...(config.host ? ["/h", config.host] : []),
+            ...(config.port ? [`/port:${config.port}`] : []),
+            ...["/authenticationDatabase:admin"],
+            ...["/d", database.name],
+            ...(config.username ? ["/u", config.username] : []),
+            ...(this.config.compress ? ["/gzip"] : []),
+            ...(this.config.concurrency ? ["/j", this.config.concurrency] : []),
+            data.snapshotPath,
+        ], { $log: this.verbose });
+        p.stdin.writable.write(`${config.password ?? ""}\n`);
+        await p.waitForClose();
     }
 }
 exports.MongoDumpTask = MongoDumpTask;

package/lib/utils/datatruck/client.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { DiskStats } from "../fs";
 import { BasicProgress } from "../progress";
 import { AbstractFs, FsOptions } from "../virtual-fs";
@@ -13,7 +12,7 @@ export declare class RemoteFs extends AbstractFs {
     });
     isLocal(): boolean;
     protected fetchJson(name: string, params: any[]): Promise<any>;
-    protected post(name: string, params: any[], data: string): Promise<Response>;
+    protected post(name: string, params: any[], data: string): Promise<import("undici").Response>;
     existsDir(path: string): Promise<boolean>;
     rename(source: string, target: string): Promise<void>;
     mkdir(path: string): Promise<void>;

package/lib/utils/datatruck/job.js

@@ -11,17 +11,22 @@ const fs_2 = require("fs");
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 async function createJobLog(config, name) {
-    if (typeof config === "object" && (config?.enabled ?? true)) {
-        const dt = new Date().toISOString().replaceAll(":", "-");
-        const dir = config?.path ?? logs_1.defaultsLogPath;
-        const tmpLogPath = (0, path_1.join)(dir, `${dt}_${name}.tmp.log`);
-        await (0, promises_1.mkdir)(dir, { recursive: true });
-        return {
-            dt,
-            dir,
-            stream: (0, fs_2.createWriteStream)(tmpLogPath),
-        };
-    }
+    const dt = new Date().toISOString().replaceAll(":", "-");
+    const dir = config?.path ?? logs_1.defaultsLogPath;
+    const tmpPath = (0, path_1.join)(dir, `${dt}_${name}.tmp.log`);
+    await (0, promises_1.mkdir)(dir, { recursive: true });
+    const stream = (0, fs_2.createWriteStream)(tmpPath);
+    return {
+        stream,
+        write(data) {
+            stream.write(data);
+        },
+        async finish(pid) {
+            const path = (0, path_1.join)(dir, `${dt}_${name}_${pid}.log`);
+            await (0, fs_1.safeRename)(tmpPath, path);
+            return path;
+        },
+    };
 }
 function getJobCliOptions(job) {
     const Command = command_1.datatruckCommands[job.action];
@@ -55,7 +60,9 @@ exports.runJob = runJob;
 async function runCronJob(job, name, config) {
     let pid = 0;
     try {
-        const log = await createJobLog(config.log, name);
+        const log = config.log?.enabled ?? true
+            ? await createJobLog(config.log, name)
+            : undefined;
         const cliOptions = getJobCliOptions(job);
         const [node, bin] = process.argv;
         const argv = [
@@ -69,7 +76,7 @@ async function runCronJob(job, name, config) {
             job.action,
             ...cliOptions,
         ];
-        log?.stream.write(`+ dtt ${argv.slice(1).join(" ")}\n`);
+        log?.write(`+ dtt ${argv.slice(1).join(" ")}\n`);
         const p = new async_process_1.AsyncProcess(node, argv, {
             $log: config.verbose,
             $exitCode: false,
@@ -81,22 +88,18 @@ async function runCronJob(job, name, config) {
             },
         });
         pid = p.child.pid || 0;
-        if (log)
-            (0, cli_1.logJson)("job", `'${name}' started`, { pid });
+        (0, cli_1.logJson)("job", `'${name}' started`, { pid });
         const [exitCode] = await Promise.all([
             p.waitForClose(),
-            ...(log?.stream
-                ? [p.stderr.pipe(log?.stream), p.stdout.pipe(log?.stream)]
-                : []),
+            log && p.stderr.pipe(log.stream),
+            log && p.stdout.pipe(log.stream),
         ]);
-        let logData = {};
-        if (log) {
-            const base = (0, path_1.dirname)(log.stream.path.toString());
-            const logPath = (0, path_1.join)(base, `${log.dt}_${name}_${pid}.log`);
-            await (0, fs_1.safeRename)(log?.stream.path.toString(), logPath);
-            logData["log"] = logPath;
-        }
-        (0, cli_1.logJson)("job", `'${name}' finished`, { pid, exitCode, ...logData });
+        const logPath = await log?.finish(pid);
+        (0, cli_1.logJson)("job", `'${name}' finished`, {
+            pid,
+            exitCode,
+            ...(logPath && { log: logPath }),
+        });
     }
     catch (error) {
         (0, cli_1.logJson)("job", `'${name}' failed`, { pid });

package/lib/utils/datatruck/repository-server.js

@@ -65,7 +65,8 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
         const id = counter.next();
         let requestError;
         let responseError;
-        req.on("error", (error) => (requestError = error));
+        const requestErrorListener = (error) => (requestError = error);
+        req.on("error", requestErrorListener);
         res.on("error", (error) => (responseError = error));
         res.setHeader("X-Accel-Buffering", "no");
         try {
@@ -88,16 +89,19 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
             else if (action === "upload") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
+                req.off("error", requestErrorListener);
                 await (0, http_1.recvFile)(req, res, path);
             }
             else if (action === "download") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
+                req.off("error", requestErrorListener);
                 await (0, http_1.sendFile)(req, res, path, {
                     contentLength: options.contentLength ?? true,
                 });
             }
             else if (action === "writeFile") {
+                req.off("error", requestErrorListener);
                 const data = await (0, http_1.readRequestData)(req);
                 const [target] = params;
                 await fs.writeFile(target, data);

package/lib/utils/http.d.ts

@@ -1,22 +1,19 @@
 /// <reference types="node" />
-/// <reference types="node" />
 import { BasicProgress } from "./progress";
 import { IncomingMessage, Server, ServerResponse } from "http";
+import { fetch, type RequestInit } from "undici";
 export declare function createHref(inUrl: string, query?: Record<string, string>): string;
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
 export declare const safeFetch: typeof fetch;
 export declare function fetchJson<T = any>(url: string, options?: RequestInit): Promise<T | undefined>;
-export declare function post(url: string, data: string, options?: Omit<RequestInit, "method" | "body">): Promise<Response>;
+export declare function post(url: string, data: string, options?: Omit<RequestInit, "method" | "body">): Promise<import("undici").Response>;
 export declare function parseContentLength(value: string | undefined): number;
 export declare function sendFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
     contentLength?: boolean;
-    end?: boolean;
     checksum?: boolean;
 }): Promise<void>;
-export declare function recvFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
-    end?: boolean;
-}): Promise<void>;
+export declare function recvFile(req: IncomingMessage, res: ServerResponse, path: string): Promise<void>;
 export declare function downloadFile(url: string, output: string, options?: Omit<RequestInit, "signal"> & {
     timeout?: number;
     onProgress?: (progress: BasicProgress) => void;

package/lib/utils/http.js

@@ -7,6 +7,7 @@ const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const stream_1 = require("stream");
 const promises_2 = require("stream/promises");
+const undici_1 = require("undici");
 function createHref(inUrl, query) {
     const url = new URL(inUrl);
     for (const key in query || {})
@@ -35,7 +36,7 @@ function readRequestData(req) {
 }
 exports.readRequestData = readRequestData;
 const safeFetch = async (...args) => {
-    const res = await fetch(...args);
+    const res = await (0, undici_1.fetch)(...args);
     if (res.status !== 200)
         throw new Error(`Fetch request failed: ${res.status} ${res.statusText}`);
     return res;
@@ -74,12 +75,12 @@ async function sendFile(req, res, path, options = {}) {
     }
     finally {
         file?.close();
-        if (options.end)
+        if (!res.writableEnded)
             res.end();
     }
 }
 exports.sendFile = sendFile;
-async function recvFile(req, res, path, options = {}) {
+async function recvFile(req, res, path) {
     let file;
     try {
         file = (0, fs_1.createWriteStream)(path);
@@ -95,8 +96,7 @@ async function recvFile(req, res, path, options = {}) {
     }
     finally {
         file?.close();
-        if (options.end)
-            res.end();
+        res.end();
     }
 }
 exports.recvFile = recvFile;
@@ -162,7 +162,7 @@ async function uploadFile(url, path, options = {}) {
     const { size } = await (0, promises_1.stat)(path);
     const file = (0, fs_1.createReadStream)(path);
     try {
-        const res = await fetch(url, {
+        const res = await (0, undici_1.fetch)(url, {
             ...options,
             method: "POST",
             duplex: "half",

package/lib/utils/mongodb.d.ts

@@ -0,0 +1,11 @@
+export type MongoUriObject<Resolved = false> = {
+    host: string;
+    username?: string;
+    password?: [Resolved] extends [true] ? string : string | {
+        path: string;
+    };
+    port?: number;
+    database: string;
+};
+export declare function toMongoUri(object: MongoUriObject<true>): string;
+export declare function resolveMongoUri(input: string | MongoUriObject): Promise<MongoUriObject<true>>;

package/lib/utils/mongodb.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveMongoUri = exports.toMongoUri = void 0;
+const fs_1 = require("./fs");
+function toMongoUri(object) {
+    const url = new URL(`mongodb://${object.host}`);
+    if (typeof object.username === "string")
+        url.username = object.username;
+    if (typeof object.password === "string")
+        url.password = object.password;
+    if (typeof object.port === "number")
+        url.port = object.port.toString();
+    url.pathname = `/${object.database}`;
+    return url.href;
+}
+exports.toMongoUri = toMongoUri;
+async function resolveMongoUri(input) {
+    let object;
+    if (typeof input === "string") {
+        const url = new URL(input);
+        object = {
+            host: url.hostname,
+            password: url.password,
+            port: url.port ? Number(url.port) : undefined,
+            username: url.username,
+            database: url.pathname.slice(1),
+        };
+    }
+    else {
+        object = input;
+    }
+    return {
+        ...object,
+        password: object.password !== undefined
+            ? (await (0, fs_1.fetchData)(object.password, (p) => p.path)) ?? ""
+            : "",
+    };
+}
+exports.resolveMongoUri = resolveMongoUri;

package/lib/utils/options.js

@@ -49,7 +49,9 @@ function createCommand(config, action) {
         for (const name in config.options) {
             const option = config.options[name];
             if (option.flag !== false) {
-                const parse = typeof option.value === "string" ? parsers[option.value] : undefined;
+                const parse = typeof option.value === "string"
+                    ? parsers[option.value]
+                    : option.value;
                 const cliValue = cliOptions[option.flag ?? name] ?? option.defaults;
                 if (cliValue !== undefined)
                     commandOptions[name] = parse ? parse(cliValue) : cliValue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.39.1",
+  "version": "0.40.1",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {
@@ -27,23 +27,26 @@
   ],
   "dependencies": {
     "@supercharge/promise-pool": "^3.2.0",
-    "ajv": "^8.12.0",
+    "ajv": "^8.13.0",
     "async": "^3.2.5",
     "chalk": "^4.1.2",
     "commander": "^12.0.0",
-    "croner": "^8.0.1",
-    "dayjs": "^1.11.10",
+    "croner": "^8.0.2",
+    "dayjs": "^1.11.11",
     "fast-folder-size": "^2.2.0",
     "fast-glob": "^3.3.2",
     "listr2": "^8.2.1",
     "micromatch": "^4.0.5",
-    "mysql2": "^3.9.3",
+    "mongodb": "^6.6.1",
+    "mysql2": "^3.9.7",
     "tty-table": "^4.2.3",
-    "yaml": "^2.4.1"
+    "undici": "^6.15.0",
+    "yaml": "^2.4.2"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
-    "@types/micromatch": "^4.0.6"
+    "@types/micromatch": "^4.0.7",
+    "mongodb-memory-server": "^9.2.0"
   },
   "optionalDependencies": {
     "ts-node": "^10.9.2"