@datatruck/cli 0.30.1 → 0.31.0

package/config.schema.json

@@ -1048,67 +1048,217 @@
           "type": "object",
           "additionalProperties": false,
           "properties": {
-            "path": {
-              "type": "string"
-            },
             "log": {
               "type": "boolean"
             },
-            "users": {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "additionalProperties": false,
-                "properties": {
-                  "name": {
-                    "type": "string"
-                  },
-                  "password": {
-                    "type": "string"
-                  }
-                }
-              }
-            },
-            "listen": {
+            "repository": {
               "type": "object",
               "additionalProperties": false,
               "properties": {
-                "port": {
-                  "type": "integer"
-                },
-                "address": {
-                  "type": "string"
-                }
-              }
-            },
-            "trustProxy": {
-              "anyOf": [
-                {
+                "enabled": {
                   "type": "boolean"
                 },
-                {
+                "listen": {
                   "type": "object",
                   "additionalProperties": false,
-                  "required": [
-                    "remoteAddressHeader"
-                  ],
                   "properties": {
-                    "remoteAddressHeader": {
+                    "port": {
+                      "type": "integer"
+                    },
+                    "address": {
                       "type": "string"
                     }
                   }
+                },
+                "trustProxy": {
+                  "anyOf": [
+                    {
+                      "type": "boolean"
+                    },
+                    {
+                      "type": "object",
+                      "additionalProperties": false,
+                      "required": [
+                        "remoteAddressHeader"
+                      ],
+                      "properties": {
+                        "remoteAddressHeader": {
+                          "type": "string"
+                        }
+                      }
+                    }
+                  ]
+                },
+                "allowlist": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "enabled": {
+                      "type": "boolean"
+                    },
+                    "remoteAddresses": {
+                      "$ref": "#/definitions/stringlist-util"
+                    }
+                  }
+                },
+                "backends": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": [
+                      "name",
+                      "path"
+                    ],
+                    "properties": {
+                      "name": {
+                        "type": "string"
+                      },
+                      "path": {
+                        "type": "string"
+                      },
+                      "users": {
+                        "type": "array",
+                        "items": {
+                          "type": "object",
+                          "additionalProperties": false,
+                          "required": [
+                            "name",
+                            "password"
+                          ],
+                          "properties": {
+                            "enabled": {
+                              "type": "boolean"
+                            },
+                            "name": {
+                              "type": "string"
+                            },
+                            "password": {
+                              "type": "string"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
                 }
-              ]
+              }
             },
-            "allowlist": {
+            "cron": {
               "type": "object",
               "additionalProperties": false,
               "properties": {
                 "enabled": {
                   "type": "boolean"
                 },
-                "remoteAddresses": {
-                  "$ref": "#/definitions/stringlist-util"
+                "actions": {
+                  "type": "array",
+                  "items": {
+                    "allOf": [
+                      {
+                        "type": "object",
+                        "required": [
+                          "schedule"
+                        ],
+                        "properties": {
+                          "schedule": {
+                            "type": "string"
+                          }
+                        }
+                      },
+                      {
+                        "anyOf": [
+                          {
+                            "if": {
+                              "type": "object",
+                              "properties": {
+                                "type": {
+                                  "const": "backup"
+                                }
+                              }
+                            },
+                            "then": {
+                              "type": "object",
+                              "properties": {
+                                "options": {
+                                  "type": "object",
+                                  "additionalProperties": false,
+                                  "properties": {
+                                    "package": {
+                                      "type": "string"
+                                    },
+                                    "packageTask": {
+                                      "type": "string"
+                                    },
+                                    "repository": {
+                                      "type": "string"
+                                    },
+                                    "repositoryType": {
+                                      "enum": [
+                                        "restic",
+                                        "datatruck",
+                                        "git"
+                                      ]
+                                    },
+                                    "tag": {
+                                      "type": "string"
+                                    },
+                                    "date": {
+                                      "type": "string"
+                                    },
+                                    "prune": {
+                                      "type": "boolean"
+                                    }
+                                  }
+                                }
+                              }
+                            },
+                            "else": false
+                          },
+                          {
+                            "if": {
+                              "type": "object",
+                              "properties": {
+                                "type": {
+                                  "const": "copy"
+                                }
+                              }
+                            },
+                            "then": {
+                              "type": "object",
+                              "properties": {
+                                "options": {
+                                  "type": "object",
+                                  "additionalProperties": false,
+                                  "properties": {
+                                    "id": {
+                                      "type": "string"
+                                    },
+                                    "last": {
+                                      "type": "integer"
+                                    },
+                                    "package": {
+                                      "type": "string"
+                                    },
+                                    "packageTask": {
+                                      "type": "string"
+                                    },
+                                    "repository": {
+                                      "type": "string"
+                                    },
+                                    "repository2": {
+                                      "type": "string"
+                                    }
+                                  }
+                                }
+                              }
+                            },
+                            "else": false
+                          }
+                        ]
+                      }
+                    ]
+                  }
                 }
               }
             }

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.30.1",
+  "version": "0.31.0",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",
     "async": "^3.2.4",
     "chalk": "^4.1.2",
     "commander": "^11.0.0",
+    "croner": "^7.0.4",
     "dayjs": "^1.11.10",
     "fast-folder-size": "^2.2.0",
     "fast-glob": "^3.3.1",

package/utils/cli.d.ts

@@ -19,4 +19,7 @@ export type OptionsType<T1, T2 extends {
 export declare function parseOptions<T1, T2 extends {
     [K in keyof T1]: unknown;
 }>(object: T1, options: OptionsType<T1, T2>): T2;
+export declare function stringifyOptions<T1, T2 extends {
+    [K in keyof T1]: unknown;
+}>(options: OptionsType<T1, T2>, object: any): string[];
 export declare function confirm(message: string): Promise<unknown>;

package/utils/cli.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.confirm = exports.parseOptions = exports.renderObject = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
+exports.confirm = exports.stringifyOptions = exports.parseOptions = exports.renderObject = exports.renderError = exports.renderResult = exports.logExec = exports.renderProgressBar = exports.showCursorCommand = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const chalk_2 = require("chalk");
 const readline_1 = require("readline");
@@ -96,6 +96,26 @@ function parseOptions(object, options) {
     return result;
 }
 exports.parseOptions = parseOptions;
+function stringifyOptions(options, object) {
+    const result = [];
+    for (const key in options) {
+        const fullOpt = options[key].option;
+        const [opt] = fullOpt.split(",");
+        const isNegative = fullOpt.startsWith("--no");
+        const isBool = !fullOpt.includes("<") && !fullOpt.includes("[");
+        const defaultsValue = isNegative ? true : options[key].defaults;
+        const value = object?.[key] ?? defaultsValue;
+        if (isBool) {
+            if (object[key])
+                result.push(opt);
+        }
+        else if (value !== undefined) {
+            result.push(opt, `${value}`);
+        }
+    }
+    return result;
+}
+exports.stringifyOptions = stringifyOptions;
 function confirm(message) {
     const rl = (0, readline_1.createInterface)({
         input: process.stdin,

package/utils/datatruck/client.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createFs = exports.isRemoteBackend = exports.RemoteFs = void 0;
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
-const server_1 = require("./server");
+const repository_server_1 = require("./repository-server");
 class RemoteFs extends virtual_fs_1.AbstractFs {
     options;
     url;
@@ -13,8 +13,8 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         this.options = options;
         const url = new URL(options.backend);
         this.headers = {
-            [server_1.headerKey.user]: url.username,
-            [server_1.headerKey.password]: url.password,
+            [repository_server_1.headerKey.user]: url.username,
+            [repository_server_1.headerKey.password]: url.password,
         };
         url.username = "";
         url.password = "";

package/utils/datatruck/cron-server.d.ts

@@ -0,0 +1,23 @@
+import { BackupCommandOptions } from "../../Command/BackupCommand";
+import { CopyCommandOptionsType } from "../../Command/CopyCommand";
+export type CronAction = {
+    schedule: string;
+    name: "backup";
+    options: BackupCommandOptions;
+} | {
+    schedule: string;
+    name: "copy";
+    options: CopyCommandOptionsType;
+};
+export type DatatruckCronServerOptions = {
+    enabled?: boolean;
+    actions?: CronAction[];
+};
+export declare function createCronServer(options: DatatruckCronServerOptions, config: {
+    log: boolean;
+    verbose: boolean;
+    configPath: string;
+}): {
+    start: () => void;
+    stop: () => void;
+};

package/utils/datatruck/cron-server.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCronServer = void 0;
+const CommandFactory_1 = require("../../Factory/CommandFactory");
+const cli_1 = require("../cli");
+const process_1 = require("../process");
+const croner_1 = require("croner");
+function createJobs(actions, currentJobs = [], worker) {
+    const jobs = [];
+    for (const action of actions) {
+        const index = actions.indexOf(action);
+        const context = JSON.stringify({
+            index: actions.indexOf(action),
+            data: action,
+        });
+        const job = currentJobs.at(index);
+        if (!job || job.options.context !== context) {
+            job?.stop();
+            jobs.push((0, croner_1.Cron)(action.schedule, {
+                paused: true,
+                context: JSON.stringify(action),
+                catch: true,
+                protect: true,
+            }, () => worker(action, index)));
+        }
+    }
+    return jobs;
+}
+function createCronServer(options, config) {
+    const worker = async (action, index) => {
+        if (config.log)
+            console.info(`> [job] ${index} - ${action.name}`);
+        try {
+            const Command = (0, CommandFactory_1.CommandConstructorFactory)(action.name);
+            const command = new Command({ config: { packages: [], repositories: [] } }, {});
+            const cliOptions = (0, cli_1.stringifyOptions)(command.onOptions(), action.options);
+            const [node, bin] = process.argv;
+            await (0, process_1.exec)(node, [bin, "-c", config.configPath, action.name, ...cliOptions], {}, { log: config.verbose });
+            if (config.log)
+                console.info(`< [job] ${index} - ${action.name}`);
+        }
+        catch (error) {
+            if (config.log)
+                console.error(`< [job] ${index} - ${action.name}`, error);
+        }
+    };
+    const jobs = createJobs(options.actions || [], [], worker);
+    return {
+        start: () => {
+            for (const job of jobs)
+                job.resume();
+        },
+        stop: () => {
+            for (const job of jobs)
+                job.stop();
+        },
+    };
+}
+exports.createCronServer = createCronServer;

package/utils/datatruck/{server.d.ts → repository-server.d.ts}

@@ -1,17 +1,16 @@
 /// <reference types="node" />
 import { IncomingMessage } from "http";
 type User = {
+    enabled?: boolean;
     name: string;
     password: string;
 };
-export type DatatruckServerOptions = {
-    path?: string;
-    log?: boolean;
+export type DatatruckRepositoryServerOptions = {
+    enabled?: boolean;
     listen?: {
         port?: number;
         address?: string;
     };
-    users?: User[];
     trustProxy?: true | {
         remoteAddressHeader: string;
     };
@@ -22,10 +21,17 @@ export type DatatruckServerOptions = {
         enabled?: boolean;
         remoteAddresses?: string[];
     };
+    backends?: {
+        name: string;
+        path: string;
+        users?: User[];
+    }[];
 };
 export declare const headerKey: {
     user: string;
     password: string;
 };
-export declare function createDatatruckServer(options: DatatruckServerOptions): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
+export declare function createDatatruckRepositoryServer(options: Omit<DatatruckRepositoryServerOptions, "listen">, config?: {
+    log?: boolean;
+}): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
 export {};

package/utils/datatruck/{server.js → repository-server.js}

@@ -1,30 +1,33 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createDatatruckServer = exports.headerKey = void 0;
+exports.createDatatruckRepositoryServer = exports.headerKey = void 0;
 const http_1 = require("../http");
 const virtual_fs_1 = require("../virtual-fs");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
 const http_2 = require("http");
-function parseUrl(inUrl) {
+exports.headerKey = {
+    user: "x-dtt-user",
+    password: "x-dtt-password",
+};
+function parseUrl(inUrl, repositoryPrefix = "repo") {
     const url = new URL(`http://127.0.0.1${inUrl}`);
     const inParams = url.searchParams.get("params");
-    const action = url.pathname.slice(1);
-    if (typeof inParams === "string") {
+    const [prefix, repository, action] = url.pathname.slice(1).split("/");
+    if (prefix !== repositoryPrefix) {
+        return { repository: undefined, action: undefined, params: [] };
+    }
+    else if (typeof inParams === "string") {
         const params = JSON.parse(inParams);
         if (!Array.isArray(params))
             throw new Error(`Invalid params`);
-        return { action, params };
+        return { repository, action, params };
     }
     else {
-        return { action, params: [] };
+        return { repository, action, params: [] };
     }
 }
-exports.headerKey = {
-    user: "x-dtt-user",
-    password: "x-dtt-password",
-};
-function validateRequest(req, options) {
+function findRepositoryBackend(req, repository, options) {
     const list = options.allowlist;
     if (list && (list.enabled ?? true) && list.remoteAddresses) {
         const remoteAddress = getRemoteAddress(req, options);
@@ -35,7 +38,15 @@ function validateRequest(req, options) {
     const password = req.headers[exports.headerKey.password]?.toString().trim();
     if (!name?.length || !password?.length)
         return;
-    return (options.users?.some((user) => user.name === name && user.password === password) || false);
+    const backend = options.backends?.find((e) => e.name === repository);
+    if (!backend)
+        return;
+    const user = backend.users?.find((user) => user.name === name && user.password === password);
+    if (!user)
+        return;
+    if (!(user.enabled ?? true))
+        return;
+    return backend;
 }
 const getRemoteAddress = (req, options) => {
     return ((options.trustProxy
@@ -44,24 +55,30 @@ const getRemoteAddress = (req, options) => {
             : req.headers[options.trustProxy.remoteAddressHeader]?.toString()
         : undefined) ?? req.socket.remoteAddress);
 };
-function createDatatruckServer(options) {
-    const log = options.log ?? true;
+function createDatatruckRepositoryServer(options, config = {}) {
     return (0, http_2.createServer)(async (req, res) => {
         try {
-            if (req.url === "/" || req.url === "/favicon.ico") {
+            if (req.url === "/" || req.url === "/favicon.ico")
+                return res.end();
+            const { repository, action, params } = parseUrl(req.url);
+            if (!repository || !action) {
+                res.statusCode = 404;
                 return res.end();
             }
-            else if (!validateRequest(req, options)) {
+            const backend = findRepositoryBackend(req, repository, options);
+            if (!backend) {
                 res.statusCode = 401;
                 return res.end();
             }
-            if (log)
-                console.info(`> ${req.url}`);
+            if (config.log)
+                console.info(`> [repository] ${repository} - ${req.url}`);
             const fs = new virtual_fs_1.LocalFs({
-                backend: options.path ?? ".",
+                backend: backend.path,
             });
-            const { action, params } = parseUrl(req.url);
-            if (action === "upload") {
+            if (action === "comcheck") {
+                res.write(JSON.stringify({ success: true }));
+            }
+            else if (action === "upload") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
                 const file = (0, fs_1.createWriteStream)(path);
@@ -99,17 +116,17 @@ function createDatatruckServer(options) {
                 if (json !== undefined)
                     res.write(JSON.stringify(json));
             }
-            if (log)
-                console.info(`<${action}`);
+            if (config.log)
+                console.info(`< [repository] ${repository} - ${action}`);
             res.end();
         }
         catch (error) {
-            if (log)
-                console.error(`<${req.url}`, error);
+            if (config.log)
+                console.error(`< [repository] ${req.url}`, error);
             res.statusCode = 500;
             res.statusMessage = error.message;
             res.end();
         }
     });
 }
-exports.createDatatruckServer = createDatatruckServer;
+exports.createDatatruckRepositoryServer = createDatatruckRepositoryServer;

package/utils/http.d.ts

@@ -3,10 +3,16 @@ import { BasicProgress } from "./progress";
 import { IncomingMessage, Server } from "http";
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
-export declare function fetchJson<T = any>(url: string, options?: {
+export type FetchOptions = {
     headers?: Record<string, string>;
     query?: Record<string, string>;
-}): Promise<T>;
+    statusError?: boolean;
+};
+export declare function fetch(url: string, options?: FetchOptions): Promise<{
+    data: string | undefined;
+    status: number | undefined;
+}>;
+export declare function fetchJson<T = any>(url: string, options?: FetchOptions): Promise<T | undefined>;
 export declare function post(url: string, data: string, options?: {
     headers?: Record<string, string>;
     query?: Record<string, string>;

package/utils/http.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
+exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.fetch = exports.readRequestData = exports.closeServer = void 0;
 const math_1 = require("./math");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
@@ -37,14 +37,15 @@ function readRequestData(req) {
     });
 }
 exports.readRequestData = readRequestData;
-async function fetchJson(url, options = {}) {
+async function fetch(url, options = {}) {
+    const throwStatusCodeError = options.statusError ?? true;
     return new Promise((resolve, reject) => {
         let data;
         request(href(url, options.query), {
             method: "GET",
             headers: options.headers,
         }, (res) => {
-            if (res.statusCode !== 200)
+            if (throwStatusCodeError && res.statusCode !== 200)
                 return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
             res
                 .on("data", (chunk) => {
@@ -54,23 +55,20 @@ async function fetchJson(url, options = {}) {
             })
                 .on("error", reject)
                 .on("close", () => {
-                if (data === undefined) {
-                    resolve(undefined);
-                }
-                else {
-                    try {
-                        resolve(JSON.parse(data));
-                    }
-                    catch (error) {
-                        reject(error);
-                    }
-                }
+                resolve({ data, status: res.statusCode });
             });
         })
             .on("error", reject)
             .end();
     });
 }
+exports.fetch = fetch;
+async function fetchJson(url, options = {}) {
+    const res = await fetch(url, options);
+    if (res.data === undefined)
+        return;
+    return JSON.parse(res.data);
+}
 exports.fetchJson = fetchJson;
 async function post(url, data, options = {}) {
     await new Promise((resolve, reject) => {

package/utils/object.d.ts

@@ -1,5 +1,5 @@
 export declare function merge<T extends Record<string, unknown>>(target: T, ...sources: Record<string, unknown>[]): T;
-export declare function push<T>(map: Record<string, T[]>, key: string, object: T): void;
+export declare function omitProp<T extends Record<string, any>, N extends keyof T>(object: T, name: N): Omit<T, N>;
 export declare function getErrorProperties(error: Error): Record<string, string>;
 type GroupByKeyParamType<TItem> = ((item: TItem) => string[] | string) | (keyof TItem)[] | keyof TItem;
 export declare function groupBy<TItem>(items: TItem[], keyOrCb: GroupByKeyParamType<TItem>): Record<string, TItem[]>;