@datatruck/cli 0.30.0 → 0.31.0

package/cli.js

@@ -19,12 +19,11 @@ const fs_2 = require("fs");
 const path_1 = require("path");
 function getGlobalOptions() {
     const result = program.opts();
+    const parseBool = (v) => v === "true" ? true : v === "false" ? false : v;
     return {
         ...result,
-        tty: result.tty === "auto" ? "auto" : result.tty === "true",
-        progress: result.progress === "auto" || result.progress === "interval"
-            ? result.progress
-            : result.progress === "true",
+        tty: parseBool(result.tty),
+        progress: parseBool(result.progress),
     };
 }
 function makeCommand(command) {
@@ -60,7 +59,7 @@ function makeCommandAction(command) {
             exitCode = await (0, CommandFactory_1.CommandFactory)(command, {
                 ...globalOptions,
                 config: config.data,
-            }, options).onExec();
+            }, options, {}, globalOptions.config).onExec();
         }
         catch (e) {
             const error = e;
@@ -89,8 +88,7 @@ program.usage("dtt");
 program.option("-v,--verbose", "Verbose", (_, previous) => previous + 1, 0);
 program.option("-c,--config <path>", "Config path", process.env["DATATRUCK_CONFIG"] ?? (cwd.endsWith(path_1.sep) ? cwd : `${cwd}${path_1.sep}`));
 program.option("--tty <value>", "TTY mode (auto, true, false)", "auto");
-program.option("--progress <value>", "Progress type (auto, true, false, interval)", "auto");
-program.option("--progress-interval <ms>", "Progress interval", Number, 1000);
+program.option("--progress <value>", "Progress type (auto, true, false, interval, interval:[ms])", "auto");
 program.option("-o,--output-format <format>", "Output format (json, pjson, yaml, table, custom=$, tpl=name)", "table");
 makeCommand(CommandFactory_1.CommandEnum.startServer).alias("start");
 makeCommand(CommandFactory_1.CommandEnum.config).alias("c");

package/config.schema.json

@@ -1048,67 +1048,217 @@
           "type": "object",
           "additionalProperties": false,
           "properties": {
-            "path": {
-              "type": "string"
-            },
             "log": {
               "type": "boolean"
             },
-            "users": {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "additionalProperties": false,
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "password": {
-                    "type": "string"
-                  }
-                }
-              }
-            },
-            "listen": {
+            "repository": {
               "type": "object",
               "additionalProperties": false,
               "properties": {
-                "port": {
-                  "type": "integer"
-                },
-                "address": {
-                  "type": "string"
-                }
-              }
-            },
-            "trustProxy": {
-              "anyOf": [
-                {
+                "enabled": {
                   "type": "boolean"
                 },
-                {
+                "listen": {
                   "type": "object",
                   "additionalProperties": false,
-                  "required": [
-                    "remoteAddressHeader"
-                  ],
                   "properties": {
-                    "remoteAddressHeader": {
+                    "port": {
+                      "type": "integer"
+                    },
+                    "address": {
                       "type": "string"
                     }
                   }
+                },
+                "trustProxy": {
+                  "anyOf": [
+                    {
+                      "type": "boolean"
+                    },
+                    {
+                      "type": "object",
+                      "additionalProperties": false,
+                      "required": [
+                        "remoteAddressHeader"
+                      ],
+                      "properties": {
+                        "remoteAddressHeader": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  ]
+                },
+                "allowlist": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "enabled": {
+                      "type": "boolean"
+                    },
+                    "remoteAddresses": {
+                      "$ref": "#/definitions/stringlist-util"
+                    }
+                  }
+                },
+                "backends": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": [
+                      "name",
+                      "path"
+                    ],
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "path": {
+                        "type": "string"
+                      },
+                      "users": {
+                        "type": "array",
+                        "items": {
+                          "type": "object",
+                          "additionalProperties": false,
+                          "required": [
+                            "name",
+                            "password"
+                          ],
+                          "properties": {
+                            "enabled": {
+                              "type": "boolean"
+                            },
+                            "name": {
+                              "type": "string"
+                            },
+                            "password": {
+                              "type": "string"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
                 }
-              ]
+              }
             },
-            "allowlist": {
+            "cron": {
               "type": "object",
               "additionalProperties": false,
               "properties": {
                 "enabled": {
                   "type": "boolean"
                 },
-                "remoteAddreses": {
-                  "$ref": "#/definitions/stringlist-util"
+                "actions": {
+                  "type": "array",
+                  "items": {
+                    "allOf": [
+                      {
+                        "type": "object",
+                        "required": [
+                          "schedule"
+                        ],
+                        "properties": {
+                          "schedule": {
+                            "type": "string"
+                          }
+                        }
+                      },
+                      {
+                        "anyOf": [
+                          {
+                            "if": {
+                              "type": "object",
+                              "properties": {
+                                "type": {
+                                  "const": "backup"
+                                }
+                              }
+                            },
+                            "then": {
+                              "type": "object",
+                              "properties": {
+                                "options": {
+                                  "type": "object",
+                                  "additionalProperties": false,
+                                  "properties": {
+                                    "package": {
+                                      "type": "string"
+                                    },
+                                    "packageTask": {
+                                      "type": "string"
+                                    },
+                                    "repository": {
+                                      "type": "string"
+                                    },
+                                    "repositoryType": {
+                                      "enum": [
+                                        "restic",
+                                        "datatruck",
+                                        "git"
+                                      ]
+                                    },
+                                    "tag": {
+                                      "type": "string"
+                                    },
+                                    "date": {
+                                      "type": "string"
+                                    },
+                                    "prune": {
+                                      "type": "boolean"
+                                    }
+                                  }
+                                }
+                              }
+                            },
+                            "else": false
+                          },
+                          {
+                            "if": {
+                              "type": "object",
+                              "properties": {
+                                "type": {
+                                  "const": "copy"
+                                }
+                              }
+                            },
+                            "then": {
+                              "type": "object",
+                              "properties": {
+                                "options": {
+                                  "type": "object",
+                                  "additionalProperties": false,
+                                  "properties": {
+                                    "id": {
+                                      "type": "string"
+                                    },
+                                    "last": {
+                                      "type": "integer"
+                                    },
+                                    "package": {
+                                      "type": "string"
+                                    },
+                                    "packageTask": {
+                                      "type": "string"
+                                    },
+                                    "repository": {
+                                      "type": "string"
+                                    },
+                                    "repository2": {
+                                      "type": "string"
+                                    }
+                                  }
+                                }
+                              }
+                            },
+                            "else": false
+                          }
+                        ]
+                      }
+                    ]
+                  }
                 }
               }
             }

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.30.0",
+  "version": "0.31.0",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
     "commander": "^11.0.0",
+    "croner": "^7.0.4",
     "dayjs": "^1.11.10",
     "fast-folder-size": "^2.2.0",
     "fast-glob": "^3.3.1",

package/utils/cli.d.ts

@@ -19,4 +19,7 @@ export type OptionsType<T1, T2 extends {
 export declare function parseOptions<T1, T2 extends {
     [K in keyof T1]: unknown;
 }>(object: T1, options: OptionsType<T1, T2>): T2;
+export declare function stringifyOptions<T1, T2 extends {
+    [K in keyof T1]: unknown;
+}>(options: OptionsType<T1, T2>, object: any): string[];
 export declare function confirm(message: string): Promise<unknown>;

package/utils/cli.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.confirm = exports.parseOptions = exports.renderObject = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
+exports.confirm = exports.stringifyOptions = exports.parseOptions = exports.renderObject = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const chalk_2 = require("chalk");
 const readline_1 = require("readline");
@@ -96,6 +96,26 @@ function parseOptions(object, options) {
     return result;
 }
 exports.parseOptions = parseOptions;
+function stringifyOptions(options, object) {
+    const result = [];
+    for (const key in options) {
+        const fullOpt = options[key].option;
+        const [opt] = fullOpt.split(",");
+        const isNegative = fullOpt.startsWith("--no");
+        const isBool = !fullOpt.includes("<") && !fullOpt.includes("[");
+        const defaultsValue = isNegative ? true : options[key].defaults;
+        const value = object?.[key] ?? defaultsValue;
+        if (isBool) {
+            if (object[key])
+                result.push(opt);
+        }
+        else if (value !== undefined) {
+            result.push(opt, `${value}`);
+        }
+    }
+    return result;
+}
+exports.stringifyOptions = stringifyOptions;
 function confirm(message) {
     const rl = (0, readline_1.createInterface)({
         input: process.stdin,

package/utils/datatruck/client.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createFs = exports.isRemoteBackend = exports.RemoteFs = void 0;
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
-const server_1 = require("./server");
+const repository_server_1 = require("./repository-server");
 class RemoteFs extends virtual_fs_1.AbstractFs {
     options;
     url;
@@ -13,8 +13,8 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         this.options = options;
         const url = new URL(options.backend);
         this.headers = {
-            [server_1.headerKey.user]: url.username,
-            [server_1.headerKey.password]: url.password,
+            [repository_server_1.headerKey.user]: url.username,
+            [repository_server_1.headerKey.password]: url.password,
         };
         url.username = "";
         url.password = "";

package/utils/datatruck/cron-server.d.ts

@@ -0,0 +1,23 @@
+import { BackupCommandOptions } from "../../Command/BackupCommand";
+import { CopyCommandOptionsType } from "../../Command/CopyCommand";
+export type CronAction = {
+    schedule: string;
+    name: "backup";
+    options: BackupCommandOptions;
+} | {
+    schedule: string;
+    name: "copy";
+    options: CopyCommandOptionsType;
+};
+export type DatatruckCronServerOptions = {
+    enabled?: boolean;
+    actions?: CronAction[];
+};
+export declare function createCronServer(options: DatatruckCronServerOptions, config: {
+    log: boolean;
+    verbose: boolean;
+    configPath: string;
+}): {
+    start: () => void;
+    stop: () => void;
+};

package/utils/datatruck/cron-server.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCronServer = void 0;
+const CommandFactory_1 = require("../../Factory/CommandFactory");
+const cli_1 = require("../cli");
+const process_1 = require("../process");
+const croner_1 = require("croner");
+function createJobs(actions, currentJobs = [], worker) {
+    const jobs = [];
+    for (const action of actions) {
+        const index = actions.indexOf(action);
+        const context = JSON.stringify({
+            index: actions.indexOf(action),
+            data: action,
+        });
+        const job = currentJobs.at(index);
+        if (!job || job.options.context !== context) {
+            job?.stop();
+            jobs.push((0, croner_1.Cron)(action.schedule, {
+                paused: true,
+                context: JSON.stringify(action),
+                catch: true,
+                protect: true,
+            }, () => worker(action, index)));
+        }
+    }
+    return jobs;
+}
+function createCronServer(options, config) {
+    const worker = async (action, index) => {
+        if (config.log)
+            console.info(`> [job] ${index} - ${action.name}`);
+        try {
+            const Command = (0, CommandFactory_1.CommandConstructorFactory)(action.name);
+            const command = new Command({ config: { packages: [], repositories: [] } }, {});
+            const cliOptions = (0, cli_1.stringifyOptions)(command.onOptions(), action.options);
+            const [node, bin] = process.argv;
+            await (0, process_1.exec)(node, [bin, "-c", config.configPath, action.name, ...cliOptions], {}, { log: config.verbose });
+            if (config.log)
+                console.info(`< [job] ${index} - ${action.name}`);
+        }
+        catch (error) {
+            if (config.log)
+                console.error(`< [job] ${index} - ${action.name}`, error);
+        }
+    };
+    const jobs = createJobs(options.actions || [], [], worker);
+    return {
+        start: () => {
+            for (const job of jobs)
+                job.resume();
+        },
+        stop: () => {
+            for (const job of jobs)
+                job.stop();
+        },
+    };
+}
+exports.createCronServer = createCronServer;

package/utils/datatruck/{server.d.ts → repository-server.d.ts}

@@ -1,17 +1,16 @@
 /// <reference types="node" />
 import { IncomingMessage } from "http";
 type User = {
+    enabled?: boolean;
     name: string;
     password: string;
 };
-export type DatatruckServerOptions = {
-    path?: string;
-    log?: boolean;
+export type DatatruckRepositoryServerOptions = {
+    enabled?: boolean;
     listen?: {
         port?: number;
         address?: string;
     };
-    users?: User[];
     trustProxy?: true | {
         remoteAddressHeader: string;
     };
@@ -20,12 +19,19 @@ export type DatatruckServerOptions = {
          * @default true
          */
         enabled?: boolean;
-        remoteAddreses?: string[];
+        remoteAddresses?: string[];
     };
+    backends?: {
+        name: string;
+        path: string;
+        users?: User[];
+    }[];
 };
 export declare const headerKey: {
     user: string;
     password: string;
 };
-export declare function createDatatruckServer(options: DatatruckServerOptions): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
+export declare function createDatatruckRepositoryServer(options: Omit<DatatruckRepositoryServerOptions, "listen">, config?: {
+    log?: boolean;
+}): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
 export {};

package/utils/datatruck/{server.js → repository-server.js}

@@ -1,41 +1,52 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createDatatruckServer = exports.headerKey = void 0;
+exports.createDatatruckRepositoryServer = exports.headerKey = void 0;
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const http_2 = require("http");
-function parseUrl(inUrl) {
+exports.headerKey = {
+    user: "x-dtt-user",
+    password: "x-dtt-password",
+};
+function parseUrl(inUrl, repositoryPrefix = "repo") {
     const url = new URL(`http://127.0.0.1${inUrl}`);
     const inParams = url.searchParams.get("params");
-    const action = url.pathname.slice(1);
-    if (typeof inParams === "string") {
+    const [prefix, repository, action] = url.pathname.slice(1).split("/");
+    if (prefix !== repositoryPrefix) {
+        return { repository: undefined, action: undefined, params: [] };
+    }
+    else if (typeof inParams === "string") {
         const params = JSON.parse(inParams);
         if (!Array.isArray(params))
             throw new Error(`Invalid params`);
-        return { action, params };
+        return { repository, action, params };
     }
     else {
-        return { action, params: [] };
+        return { repository, action, params: [] };
     }
 }
-exports.headerKey = {
-    user: "x-dtt-user",
-    password: "x-dtt-password",
-};
-function validateRequest(req, options) {
+function findRepositoryBackend(req, repository, options) {
     const list = options.allowlist;
-    if (list && (list.enabled ?? true) && list.remoteAddreses) {
+    if (list && (list.enabled ?? true) && list.remoteAddresses) {
         const remoteAddress = getRemoteAddress(req, options);
-        if (!remoteAddress || list.remoteAddreses.includes(remoteAddress))
+        if (!remoteAddress || !list.remoteAddresses.includes(remoteAddress))
             return false;
     }
     const name = req.headers[exports.headerKey.user]?.toString().trim();
     const password = req.headers[exports.headerKey.password]?.toString().trim();
     if (!name?.length || !password?.length)
         return;
-    return (options.users?.some((user) => user.name === name && user.password === password) || false);
+    const backend = options.backends?.find((e) => e.name === repository);
+    if (!backend)
+        return;
+    const user = backend.users?.find((user) => user.name === name && user.password === password);
+    if (!user)
+        return;
+    if (!(user.enabled ?? true))
+        return;
+    return backend;
 }
 const getRemoteAddress = (req, options) => {
     return ((options.trustProxy
@@ -44,24 +55,30 @@ const getRemoteAddress = (req, options) => {
             : req.headers[options.trustProxy.remoteAddressHeader]?.toString()
         : undefined) ?? req.socket.remoteAddress);
 };
-function createDatatruckServer(options) {
-    const log = options.log ?? true;
+function createDatatruckRepositoryServer(options, config = {}) {
     return (0, http_2.createServer)(async (req, res) => {
         try {
-            if (req.url === "/" || req.url === "/favicon.ico") {
+            if (req.url === "/" || req.url === "/favicon.ico")
+                return res.end();
+            const { repository, action, params } = parseUrl(req.url);
+            if (!repository || !action) {
+                res.statusCode = 404;
                 return res.end();
             }
-            else if (!validateRequest(req, options)) {
+            const backend = findRepositoryBackend(req, repository, options);
+            if (!backend) {
                 res.statusCode = 401;
                 return res.end();
             }
-            if (log)
-                console.info(`> ${req.url}`);
+            if (config.log)
+                console.info(`> [repository] ${repository} - ${req.url}`);
             const fs = new virtual_fs_1.LocalFs({
-                backend: options.path ?? ".",
+                backend: backend.path,
             });
-            const { action, params } = parseUrl(req.url);
-            if (action === "upload") {
+            if (action === "comcheck") {
+                res.write(JSON.stringify({ success: true }));
+            }
+            else if (action === "upload") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
                 const file = (0, fs_1.createWriteStream)(path);
@@ -99,17 +116,17 @@ function createDatatruckServer(options) {
                 if (json !== undefined)
                     res.write(JSON.stringify(json));
             }
-            if (log)
-                console.info(`<${action}`);
+            if (config.log)
+                console.info(`< [repository] ${repository} - ${action}`);
             res.end();
         }
         catch (error) {
-            if (log)
-                console.error(`<${req.url}`, error);
+            if (config.log)
+                console.error(`< [repository] ${req.url}`, error);
             res.statusCode = 500;
             res.statusMessage = error.message;
             res.end();
         }
     });
 }
-exports.createDatatruckServer = createDatatruckServer;
+exports.createDatatruckRepositoryServer = createDatatruckRepositoryServer;

package/utils/http.d.ts

@@ -3,10 +3,16 @@ import { BasicProgress } from "./progress";
 import { IncomingMessage, Server } from "http";
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
-export declare function fetchJson<T = any>(url: string, options?: {
+export type FetchOptions = {
     headers?: Record<string, string>;
     query?: Record<string, string>;
-}): Promise<T>;
+    statusError?: boolean;
+};
+export declare function fetch(url: string, options?: FetchOptions): Promise<{
+    data: string | undefined;
+    status: number | undefined;
+}>;
+export declare function fetchJson<T = any>(url: string, options?: FetchOptions): Promise<T | undefined>;
 export declare function post(url: string, data: string, options?: {
     headers?: Record<string, string>;
     query?: Record<string, string>;