@datatruck/restic 0.0.1

package/config.schema.json

@@ -0,0 +1,182 @@
+{
+  "type": "object",
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "hostname": {
+      "type": "string"
+    },
+    "ntfyToken": {
+      "type": "string"
+    },
+    "minFreeSpace": {
+      "type": "string"
+    },
+    "verbose": {
+      "type": "boolean"
+    },
+    "tasks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "type": {
+            "type": "string",
+            "const": "mysql-dump"
+          },
+          "packages": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "name": {
+            "type": "string"
+          },
+          "config": {
+            "type": "object",
+            "properties": {
+              "database": {
+                "type": "string"
+              },
+              "out": {
+                "anyOf": [
+                  {
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "package": {
+                          "type": "string"
+                        },
+                        "tables": {
+                          "type": "array",
+                          "items": {
+                            "type": "string"
+                          }
+                        },
+                        "path": {
+                          "anyOf": [
+                            {
+                              "const": false,
+                              "type": "boolean"
+                            },
+                            {
+                              "type": "string"
+                            }
+                          ]
+                        }
+                      },
+                      "additionalProperties": false,
+                      "required": [
+                        "path",
+                        "tables"
+                      ]
+                    }
+                  },
+                  {
+                    "type": "string"
+                  }
+                ]
+              },
+              "concurrency": {
+                "type": "number"
+              },
+              "connection": {
+                "type": "object",
+                "properties": {
+                  "hostname": {
+                    "type": "string"
+                  },
+                  "username": {
+                    "type": "string"
+                  },
+                  "password": {
+                    "type": "string"
+                  },
+                  "database": {
+                    "type": "string"
+                  }
+                },
+                "additionalProperties": false,
+                "required": [
+                  "hostname",
+                  "password",
+                  "username"
+                ]
+              }
+            },
+            "additionalProperties": false,
+            "required": [
+              "connection",
+              "database",
+              "out"
+            ]
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "config",
+          "name",
+          "packages",
+          "type"
+        ]
+      }
+    },
+    "packages": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "path": {
+            "type": "string"
+          },
+          "exclude": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "name",
+          "path"
+        ]
+      }
+    },
+    "repositories": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "password": {
+            "type": "string"
+          },
+          "uri": {
+            "type": "string"
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "name",
+          "password",
+          "uri"
+        ]
+      }
+    }
+  },
+  "additionalProperties": false,
+  "required": [
+    "packages",
+    "repositories"
+  ],
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/lib/actions/backup.d.ts

@@ -0,0 +1,40 @@
+import { GlobalConfig, type Config } from "../config.js";
+import { MySQLDump } from "../utils/mysql.js";
+import { Ntfy } from "../utils/ntfy.js";
+import { CommonResticBackupTags, ResticBackup } from "../utils/restic-backup.js";
+export type BackupRunOptions = {
+    packages?: string[];
+    repositories?: string[];
+};
+export declare class Backup {
+    readonly config: Config;
+    readonly global?: GlobalConfig | undefined;
+    readonly ntfy: Ntfy;
+    protected verbose: boolean | undefined;
+    readonly tags: CommonResticBackupTags;
+    constructor(config: Config, global?: GlobalConfig | undefined);
+    protected createInstances(packageNames: string[], repositoryNames?: string[]): {
+        repositories: ResticBackup[];
+        sqlDumps: (readonly [MySQLDump, {
+            type: "mysql-dump";
+            packages: string[];
+            name: string;
+            config: {
+                database: string;
+                out: {
+                    package?: string;
+                    tables: string[];
+                    path: string | false;
+                }[] | string;
+                concurrency?: number;
+                connection: {
+                    hostname: string;
+                    username: string;
+                    password: string;
+                    database?: string;
+                };
+            };
+        }])[];
+    };
+    run(options?: BackupRunOptions): Promise<void>;
+}

package/lib/actions/backup.js

@@ -0,0 +1,142 @@
+import { MySQLDump } from "../utils/mysql.js";
+import { Ntfy } from "../utils/ntfy.js";
+import { ResticBackup, } from "../utils/restic-backup.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { duration } from "@datatruck/cli/utils/date.js";
+import { match } from "@datatruck/cli/utils/string.js";
+import { randomUUID } from "crypto";
+import { hostname } from "os";
+export class Backup {
+    config;
+    global;
+    ntfy;
+    verbose;
+    tags;
+    constructor(config, global) {
+        this.config = config;
+        this.global = global;
+        this.tags = {
+            id: randomUUID().replaceAll("-", ""),
+            get shortId() {
+                return this.id.slice(0, 8);
+            },
+            hostname: this.config.hostname ?? hostname(),
+            date: new Date().toISOString(),
+            vendor: "dtt-restic",
+            version: "1",
+        };
+        this.verbose = this.global?.verbose ?? this.config.verbose;
+        this.ntfy = new Ntfy({
+            token: this.config.ntfyToken,
+            titlePrefix: `[${this.tags.hostname}] `,
+        });
+    }
+    createInstances(packageNames, repositoryNames) {
+        const repositories = this.config.repositories
+            .filter((repo) => !repositoryNames || repositoryNames.includes(repo.name))
+            .map((repo) => new ResticBackup({
+            tags: this.tags,
+            minFreeSpace: this.config.minFreeSpace,
+            name: repo.name,
+            connection: {
+                password: repo.password,
+                uri: repo.uri,
+            },
+        }, this.ntfy, this.verbose));
+        const sqlDumps = this.config.tasks
+            ?.filter((task) => task.type === "mysql-dump" &&
+            task.packages.some((name) => match(name, packageNames)))
+            .map((task) => [
+            new MySQLDump({
+                minFreeSpace: this.config.minFreeSpace,
+                verbose: this.verbose,
+                name: task.name,
+                connection: task.config.connection,
+                concurrency: task.config.concurrency,
+            }, this.ntfy),
+            task,
+        ]) ?? [];
+        return { repositories, sqlDumps };
+    }
+    async run(options = {}) {
+        const now = Date.now();
+        const packages = this.config.packages.filter((pkg) => options.packages ? match(pkg.name, options.packages) : true);
+        const packageNames = packages.map((p) => p.name);
+        const { sqlDumps, repositories } = this.createInstances(packageNames, options.repositories);
+        let fatalError;
+        try {
+            await this.ntfy.send(`Backup start`, {
+                "- Packages": packageNames.length,
+            });
+            if (!packages.length)
+                throw new Error("None package found");
+            for (const [sqlDump, task] of sqlDumps) {
+                await sqlDump.run([
+                    {
+                        database: task.config.database,
+                        name: task.name,
+                        out: typeof task.config.out === "string"
+                            ? task.config.out
+                            : task.config.out.map((o) => ({
+                                tables: o.tables,
+                                path: !o.package || match(o.package, packageNames)
+                                    ? o.path
+                                    : false,
+                            })),
+                    },
+                ]);
+            }
+            for (const backup of repositories)
+                await backup.run(packages);
+        }
+        catch (inError) {
+            fatalError = inError;
+        }
+        finally {
+            for (const [sqlDump] of sqlDumps)
+                await sqlDump.cleanup();
+            const backupSummary = {};
+            for (const repo of repositories) {
+                for (const process of repo.processes) {
+                    if (!backupSummary[process.name])
+                        backupSummary[process.name] = {
+                            name: process.name,
+                            total: 0,
+                            success: 0,
+                            errors: 0,
+                            bytes: 0,
+                        };
+                    backupSummary[process.name].total++;
+                    backupSummary[process.name].bytes += process.stats.bytes;
+                    if (process.error) {
+                        backupSummary[process.name].errors++;
+                    }
+                    else {
+                        backupSummary[process.name].success++;
+                    }
+                }
+            }
+            const backups = Object.values(backupSummary);
+            const sqlDumpProccesses = sqlDumps.flatMap(([sql]) => sql.processes);
+            const error = !!fatalError ||
+                sqlDumpProccesses.some((p) => p.error) ||
+                backups.some((p) => p.errors);
+            const size = [
+                ...sqlDumpProccesses.map((p) => p.stats.bytes),
+                ...backups.map((b) => b.bytes),
+            ].reduce((r, b) => r + b, 0);
+            await this.ntfy.send(`Backup end`, [
+                `- Duration: ${duration(Date.now() - now)}`,
+                `- Size: ${formatBytes(size)}`,
+                !!fatalError && `- Fatal error: ${fatalError.message}`,
+                !!sqlDumpProccesses.length && "## SQL Dumps",
+                ...sqlDumpProccesses.map((p) => `- ${p.error ? `❌ ` : ""}${p.name}: ${formatBytes(p.stats.bytes)}`),
+                !!backups.length && "## Backups",
+                ...backups.map((p) => `- ${p.errors ? `❌ ` : ""}${p.name}: ${p.success}/${p.total}`),
+            ], {
+                priority: error ? "high" : "default",
+                tags: [error ? "red_circle" : "green_circle"],
+            });
+        }
+    }
+}

package/lib/actions/copy.d.ts

@@ -0,0 +1,16 @@
+import { Config, GlobalConfig } from "../config.js";
+import { Ntfy } from "../utils/ntfy.js";
+export type CopyRunOptions = {
+    packages?: string[];
+    source: string;
+    targets: string[];
+};
+export declare class Copy {
+    readonly config: Config;
+    readonly global?: GlobalConfig | undefined;
+    readonly ntfy: Ntfy;
+    protected verbose: boolean | undefined;
+    constructor(config: Config, global?: GlobalConfig | undefined);
+    private findRepo;
+    run(options: CopyRunOptions): Promise<void>;
+}

package/lib/actions/copy.js

@@ -0,0 +1,129 @@
+import { checkDiskSpace } from "../utils/fs.js";
+import { Ntfy } from "../utils/ntfy.js";
+import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
+import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { duration } from "@datatruck/cli/utils/date.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
+export class Copy {
+    config;
+    global;
+    ntfy;
+    verbose;
+    constructor(config, global) {
+        this.config = config;
+        this.global = global;
+        this.verbose = this.global?.verbose ?? this.config.verbose;
+        this.ntfy = new Ntfy({
+            token: this.config.ntfyToken,
+            titlePrefix: `[${this.config.hostname}] `,
+        });
+    }
+    findRepo(name) {
+        const repo = this.config.repositories.find((repo) => repo.name === name);
+        if (!repo)
+            throw new Error(`Repository '${name}' not found`);
+        return repo;
+    }
+    async run(options) {
+        const now = Date.now();
+        let globalDiffSize;
+        let error;
+        try {
+            await this.ntfy.send(`Copy start`, {
+                "- Source": options.source,
+                "- Targets": options.targets.join(", "),
+            });
+            const sourceRepo = this.findRepo(options.source);
+            const source = new Restic({
+                log: this.verbose,
+                env: {
+                    RESTIC_REPOSITORY: sourceRepo.uri,
+                    RESTIC_PASSWORD: sourceRepo.password,
+                },
+            });
+            const targetRepos = options.targets.map((target) => this.findRepo(target));
+            if (!targetRepos.length)
+                throw new Error(`No target repositories specified`);
+            const inSnapshots = (await source.snapshots({
+                latest: 1,
+                group: ["path"],
+                tags: options.packages
+                    ? options.packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
+                    : undefined,
+            }));
+            const snapshots = inSnapshots.flatMap((s) => s.snapshots);
+            for (const targetRepo of targetRepos) {
+                const target = new Restic({
+                    log: this.verbose,
+                    env: {
+                        RESTIC_REPOSITORY: targetRepo.uri,
+                        RESTIC_PASSWORD: targetRepo.password,
+                        ["GODEBUG"]: "http2client=0",
+                    },
+                });
+                const exists = await target.checkRepository();
+                if (!exists && isLocalDir(targetRepo.uri))
+                    await target.exec(["init"]);
+                for (const snapshot of snapshots) {
+                    const tags = snapshot.tags
+                        .map((t) => ResticRepository.parseSnapshotTag(t))
+                        .filter((t) => !!t);
+                    const pkgTag = tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
+                    const packageName = pkgTag?.value;
+                    const now = Date.now();
+                    let copyError;
+                    let diffSize;
+                    const targetPath = isLocalDir(targetRepo.uri)
+                        ? targetRepo.uri
+                        : undefined;
+                    try {
+                        diffSize = await checkDiskSpace({
+                            minFreeSpace: this.config.minFreeSpace,
+                            targetPath,
+                            rutine: () => target.copy({
+                                ids: [snapshot.id],
+                                fromRepo: sourceRepo.uri,
+                                fromRepoPassword: sourceRepo.password,
+                            }),
+                        });
+                        if (diffSize !== undefined) {
+                            globalDiffSize = (globalDiffSize ?? 0) + (diffSize ?? 0);
+                        }
+                    }
+                    catch (inError) {
+                        copyError = inError;
+                    }
+                    await this.ntfy.send(`Copy`, {
+                        "- Id": snapshot.short_id,
+                        "- Source": sourceRepo.name,
+                        "- Target": targetRepo.name,
+                        "- Package": packageName,
+                        ...(diffSize !== undefined && {
+                            "- Diff size": (diffSize > 0 ? "+" : "") + formatBytes(diffSize),
+                        }),
+                        "- Duration": duration(Date.now() - now),
+                        "- Error": copyError?.message,
+                    }, {
+                        priority: copyError ? "high" : "default",
+                        tags: [copyError ? "red_circle" : "green_circle"],
+                    });
+                }
+            }
+        }
+        catch (inError) {
+            error = inError;
+        }
+        await this.ntfy.send(`Copy end`, {
+            ...(globalDiffSize !== undefined && {
+                "- Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
+            }),
+            "- Duration": duration(Date.now() - now),
+            "- Error": error?.message,
+        }, {
+            priority: error ? "high" : "default",
+            tags: [error ? "red_circle" : "green_circle"],
+        });
+    }
+}

package/lib/actions/init.d.ts

@@ -0,0 +1,13 @@
+import { Config, GlobalConfig } from "../config.js";
+import { Ntfy } from "../utils/ntfy.js";
+export type InitOptions = {
+    repositories?: string[];
+};
+export declare class Init {
+    readonly config: Config;
+    readonly global?: GlobalConfig | undefined;
+    readonly ntfy: Ntfy;
+    protected verbose: boolean | undefined;
+    constructor(config: Config, global?: GlobalConfig | undefined);
+    run(options: InitOptions): Promise<void>;
+}

package/lib/actions/init.js

@@ -0,0 +1,56 @@
+import { Ntfy } from "../utils/ntfy.js";
+import { duration } from "@datatruck/cli/utils/date.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
+export class Init {
+    config;
+    global;
+    ntfy;
+    verbose;
+    constructor(config, global) {
+        this.config = config;
+        this.global = global;
+        this.verbose = this.global?.verbose ?? this.config.verbose;
+        this.ntfy = new Ntfy({
+            token: this.config.ntfyToken,
+            titlePrefix: `[${this.config.hostname}] `,
+        });
+    }
+    async run(options) {
+        const now = Date.now();
+        await this.ntfy.send(`Init start`, {});
+        const repositories = options.repositories ?? this.config.repositories.map((r) => r.name);
+        for (const name of repositories) {
+            let error;
+            let exists;
+            try {
+                const repo = this.config.repositories.find((repo) => repo.name === name);
+                if (!repo)
+                    throw new Error(`Repository not found`);
+                const source = new Restic({
+                    log: this.verbose,
+                    env: {
+                        RESTIC_REPOSITORY: repo.uri,
+                        RESTIC_PASSWORD: repo.password,
+                    },
+                });
+                exists = await source.checkRepository();
+                if (!exists && isLocalDir(repo.uri))
+                    await source.exec(["init"]);
+            }
+            catch (inError) {
+                error = inError;
+            }
+            await this.ntfy.send(`Init`, {
+                "- Repository": name,
+                "- Exists": exists ? "yes" : "no",
+                "- Duration": duration(Date.now() - now),
+                "- Error": error?.message,
+            }, {
+                priority: error ? "high" : "default",
+                tags: [error ? "red_circle" : "green_circle"],
+            });
+        }
+        await this.ntfy.send(`Init end`, {});
+    }
+}

package/lib/bin.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/bin.js

@@ -0,0 +1,57 @@
+import { Backup } from "./actions/backup.js";
+import { Copy } from "./actions/copy.js";
+import { Init } from "./actions/init.js";
+import { parseConfigFile } from "./config.js";
+import { parseStringList } from "@datatruck/cli/utils/string.js";
+import { program } from "commander";
+import { resolve } from "path";
+function getGlobalOptions() {
+    const options = program.opts();
+    return {
+        ...options,
+        config: resolve(options.config),
+        verbose: process.env.DEBUG ? true : options.verbose,
+    };
+}
+async function load() {
+    const globalOptions = getGlobalOptions();
+    const config = await parseConfigFile(globalOptions.config);
+    return { globalOptions, config };
+}
+program
+    .option("-v, --verbose")
+    .option("-c, --config <path>", "Path to config file", "datatruck.restic.json");
+program
+    .command("init")
+    .alias("i")
+    .description("Run init action")
+    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+    .action(async (options) => {
+    const { config, globalOptions } = await load();
+    const init = new Init(config, globalOptions);
+    await init.run(options);
+});
+program
+    .command("backup")
+    .alias("b")
+    .description("Run backup action")
+    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+    .action(async (options) => {
+    const { config, globalOptions } = await load();
+    const backup = new Backup(config, globalOptions);
+    await backup.run(options);
+});
+program
+    .command("copy")
+    .alias("c")
+    .description("Run copy action")
+    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+    .requiredOption("-s, --source <name>", "Source repository name")
+    .requiredOption("-t, --targets <names>", "Target repository names", (v) => parseStringList(v))
+    .action(async (options) => {
+    const { config, globalOptions } = await load();
+    const copy = new Copy(config, globalOptions);
+    await copy.run(options);
+});
+program.parse();

package/lib/config.d.ts

@@ -0,0 +1,45 @@
+export type GlobalConfig = {
+    config?: string;
+    verbose?: boolean;
+};
+export type Config = {
+    $schema?: string;
+    hostname?: string;
+    ntfyToken?: string;
+    minFreeSpace?: string;
+    verbose?: boolean;
+    tasks?: {
+        type: "mysql-dump";
+        packages: string[];
+        name: string;
+        config: {
+            database: string;
+            out: {
+                package?: string;
+                tables: string[];
+                path: string | false;
+            }[] | string;
+            concurrency?: number;
+            connection: {
+                hostname: string;
+                username: string;
+                password: string;
+                database?: string;
+            };
+        };
+    }[];
+    packages: {
+        name: string;
+        path: string;
+        exclude?: string[];
+    }[];
+    repositories: {
+        name: string;
+        password: string;
+        uri: string;
+    }[];
+};
+export declare function defineConfig(config: Config): Config;
+export declare function validateConfig(config: unknown): Promise<void>;
+export declare function readConfigSchemaFile(): Promise<unknown>;
+export declare function parseConfigFile(path?: string): Promise<Config>;

package/lib/config.js

@@ -0,0 +1,22 @@
+import { parseJSONFile } from "./utils/fs.js";
+import { Ajv } from "ajv";
+export function defineConfig(config) {
+    return config;
+}
+let validate;
+export async function validateConfig(config) {
+    if (!validate) {
+        const schema = await readConfigSchemaFile();
+        validate = new Ajv({ allowUnionTypes: true }).compile(schema);
+    }
+    if (!validate(config))
+        throw new Error("Json schema error: " + JSON.stringify(validate.errors, null, 2));
+}
+export async function readConfigSchemaFile() {
+    return parseJSONFile(`${import.meta.dirname}/../config.schema.json`);
+}
+export async function parseConfigFile(path = "datatruck.restic.json") {
+    const config = await parseJSONFile(path);
+    await validateConfig(config);
+    return config;
+}

package/lib/index.d.ts

@@ -0,0 +1,4 @@
+export { Backup, type BackupRunOptions } from "./actions/backup.js";
+export { Copy, type CopyRunOptions } from "./actions/copy.js";
+export { Init, type InitOptions } from "./actions/init.js";
+export { type Config, type GlobalConfig, parseConfigFile, defineConfig, validateConfig, } from "./config.js";

package/lib/index.js

@@ -0,0 +1,4 @@
+export { Backup } from "./actions/backup.js";
+export { Copy } from "./actions/copy.js";
+export { Init } from "./actions/init.js";
+export { parseConfigFile, defineConfig, validateConfig, } from "./config.js";

package/lib/utils/fs.d.ts

@@ -0,0 +1,7 @@
+export declare function parseJSONFile<T>(path: string): Promise<T>;
+export declare function checkDiskSpace(options: {
+    minFreeSpace?: string | undefined;
+    minFreeSpacePath?: string | undefined;
+    targetPath?: string | undefined;
+    rutine: () => any;
+}): Promise<number | undefined>;

package/lib/utils/fs.js

@@ -0,0 +1,23 @@
+import { ensureFreeDiskSpace, fastFolderSizeAsync, fetchDiskStats, } from "@datatruck/cli/utils/fs.js";
+import { readFile } from "fs/promises";
+export async function parseJSONFile(path) {
+    const buffer = await readFile(path);
+    return JSON.parse(buffer.toString());
+}
+export async function checkDiskSpace(options) {
+    if (options.minFreeSpace) {
+        const minFreeSpacePath = options.minFreeSpacePath ?? options.targetPath;
+        if (minFreeSpacePath) {
+            const diskStats = await fetchDiskStats(minFreeSpacePath);
+            await ensureFreeDiskSpace(diskStats, options.minFreeSpace);
+        }
+    }
+    if (!options.targetPath) {
+        await options.rutine();
+        return;
+    }
+    const prev = await fastFolderSizeAsync(options.targetPath);
+    await options.rutine();
+    const next = await fastFolderSizeAsync(options.targetPath);
+    return next - prev;
+}

package/lib/utils/mysql.d.ts

@@ -0,0 +1,44 @@
+import { Ntfy } from "./ntfy.js";
+import { createMysqlCli } from "@datatruck/cli/utils/mysql.js";
+export type MySQLDumpItem = {
+    name: string;
+    database: string;
+    out: {
+        tables: string[];
+        path: string | false;
+    }[] | string;
+};
+export type MySQLDumpOptions = {
+    name: string;
+    verbose?: boolean;
+    minFreeSpace?: string;
+    concurrency?: number;
+    connection: MySQLDumpConnection;
+};
+export type MySQLDumpConnection = {
+    hostname: string;
+    password: string;
+    username: string;
+    port?: number;
+};
+export type MySQLDumpStats = {
+    bytes: number;
+    files: number;
+};
+export declare class MySQLDump {
+    readonly options: MySQLDumpOptions;
+    protected ntfy: Ntfy;
+    constructor(options: MySQLDumpOptions, ntfy: Ntfy);
+    readonly processes: {
+        name: string;
+        stats: MySQLDumpStats;
+        error?: Error;
+    }[];
+    protected tables: Record<string, string[]>;
+    protected outPaths: Set<string>;
+    protected startTime: number;
+    protected fetchTables(database: string, fetcher: (database: string) => Promise<string[]>): Promise<string[]>;
+    run(input: MySQLDumpItem[] | MySQLDumpItem): Promise<void>;
+    cleanup(init?: boolean): Promise<void>;
+    protected runSingle(sql: Awaited<ReturnType<typeof createMysqlCli>>, item: MySQLDumpItem): Promise<void>;
+}

package/lib/utils/mysql.js

@@ -0,0 +1,172 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import { runParallel } from "@datatruck/cli/utils/async.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { duration } from "@datatruck/cli/utils/date.js";
+import { ensureFreeDiskSpace, fetchDiskStats, } from "@datatruck/cli/utils/fs.js";
+import { createMysqlCli } from "@datatruck/cli/utils/mysql.js";
+import { match } from "@datatruck/cli/utils/string.js";
+import { existsSync } from "fs";
+import { mkdir, rm, stat } from "fs/promises";
+import { basename, dirname } from "path";
+export class MySQLDump {
+    options;
+    ntfy;
+    constructor(options, ntfy) {
+        this.options = options;
+        this.ntfy = ntfy;
+    }
+    processes = [];
+    tables = {};
+    outPaths = new Set();
+    startTime = Date.now();
+    async fetchTables(database, fetcher) {
+        const tables = this.tables[database] ?? (await fetcher(database));
+        this.tables[database] = tables;
+        return tables;
+    }
+    async run(input) {
+        const env_1 = { stack: [], error: void 0, hasError: false };
+        try {
+            const sql = __addDisposableResource(env_1, await createMysqlCli({
+                ...this.options.connection,
+                verbose: this.options.verbose,
+            }), true);
+            const items = Array.isArray(input) ? input : [input];
+            for (const item of items) {
+                await this.runSingle(sql, item);
+            }
+        }
+        catch (e_1) {
+            env_1.error = e_1;
+            env_1.hasError = true;
+        }
+        finally {
+            const result_1 = __disposeResources(env_1);
+            if (result_1)
+                await result_1;
+        }
+    }
+    async cleanup(init = false) {
+        for (const outPath of this.outPaths) {
+            const parentFolder = basename(dirname(outPath));
+            if (!parentFolder.startsWith("sql-dump"))
+                throw new Error(`sql-dump out dir must begins with 'sql-dump': ${outPath}`);
+            if (existsSync(outPath))
+                await rm(outPath, { recursive: true });
+            if (!init)
+                this.outPaths.delete(outPath);
+        }
+    }
+    async runSingle(sql, item) {
+        let error;
+        const tables = await this.fetchTables(item.database, (db) => sql.fetchTableNames(db));
+        const now = Date.now();
+        const stats = { files: 0, bytes: 0 };
+        const outs = typeof item.out === "string"
+            ? [{ path: item.out, tables: ["*"] }]
+            : item.out;
+        const items = tables
+            .map((table) => {
+            const out = outs.find((o) => match(table, o.tables));
+            return (!!out &&
+                !!out.path && {
+                table,
+                out: `${out.path}/${table}.sql`,
+            });
+        })
+            .filter((o) => !!o);
+        for (const item of items)
+            this.outPaths.add(dirname(item.out));
+        await this.cleanup(true);
+        try {
+            await runParallel({
+                items,
+                concurrency: this.options.concurrency ?? 1,
+                onItem: async (data) => {
+                    const output = data.item.out;
+                    const outDir = dirname(output);
+                    await mkdir(outDir, { recursive: true });
+                    if (this.options.minFreeSpace)
+                        await ensureFreeDiskSpace(await fetchDiskStats(outDir), this.options.minFreeSpace);
+                    await sql.dump({
+                        database: item.database,
+                        items: [data.item.table],
+                        output,
+                    });
+                    const infoFile = await stat(output);
+                    stats.files++;
+                    stats.bytes += infoFile.size;
+                },
+            });
+        }
+        catch (inError) {
+            error = inError;
+        }
+        this.processes.push({
+            name: item.name,
+            error,
+            stats,
+        });
+        await this.ntfy.send(`SQL dump`, {
+            "- Package": item.name,
+            "- Size": formatBytes(stats.bytes),
+            "- Files": stats.files,
+            "- Duration": duration(Date.now() - now),
+            "- Error": error?.message,
+        }, {
+            priority: error ? "high" : "default",
+            tags: [error ? "red_circle" : "green_circle"],
+        });
+    }
+}

package/lib/utils/ntfy.d.ts

@@ -0,0 +1,18 @@
+import { Agent } from "undici";
+export declare class Ntfy {
+    readonly options: {
+        token?: string;
+        titlePrefix?: string;
+        delay?: number;
+    };
+    protected agent: Agent | undefined;
+    constructor(options: {
+        token?: string;
+        titlePrefix?: string;
+        delay?: number;
+    });
+    send(inTitle: string, message: any[] | string | Record<string, any>, options?: {
+        priority?: "default" | "high";
+        tags?: string[];
+    }): Promise<void>;
+}

package/lib/utils/ntfy.js

@@ -0,0 +1,52 @@
+import { setTimeout } from "timers/promises";
+import { Agent, fetch } from "undici";
+export class Ntfy {
+    options;
+    agent;
+    constructor(options) {
+        this.options = options;
+        this.agent = new Agent({
+            keepAliveTimeout: 60_000,
+            keepAliveMaxTimeout: 60_000,
+            connections: 1,
+        });
+    }
+    async send(inTitle, message, options = {}) {
+        const title = this.options.titlePrefix
+            ? `${this.options.titlePrefix}${inTitle}`
+            : inTitle;
+        const body = Array.isArray(message)
+            ? message
+                .filter((v) => typeof v === "string" || typeof v === "number")
+                .join("\n")
+            : typeof message === "object" && !!message
+                ? Object.entries(message)
+                    .filter(([, value]) => value !== undefined)
+                    .map(([name, value]) => `${name}: ${value}`)
+                    .join("\n")
+                : message;
+        const lines = [title, body].filter((v) => v.length);
+        if (lines.length)
+            console.info([...lines, ""].join("\n"));
+        try {
+            if (this.options.token)
+                await fetch(`https://ntfy.sh/${this.options.token}`, {
+                    dispatcher: this.agent,
+                    method: "POST",
+                    body,
+                    headers: {
+                        Markdown: "yes",
+                        Title: title,
+                        Priority: options.priority ?? "default",
+                        ...(options.tags && {
+                            Tags: options.tags?.join(","),
+                        }),
+                    },
+                });
+            await setTimeout(this.options.delay ?? 800);
+        }
+        catch (error) {
+            console.error("Ntfy error", error);
+        }
+    }
+}

package/lib/utils/restic-backup.d.ts

@@ -0,0 +1,49 @@
+import { Ntfy } from "./ntfy.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
+export type CommonResticBackupTags = {
+    id: string;
+    shortId: string;
+    hostname: string;
+    date: string;
+    vendor: string;
+    version: string;
+};
+export type ResticBackupTags = CommonResticBackupTags & {
+    package: string;
+    tags?: string[];
+};
+export type ResticBackupPackage = {
+    name: string;
+    tags?: string[];
+    path: string;
+    exclude?: string[];
+};
+export type ResticBackupStats = {
+    files: number;
+    bytes: number;
+    diffBytes: number | undefined;
+};
+export type ResticOptions = {
+    name: string;
+    tags: CommonResticBackupTags;
+    minFreeSpace?: string;
+    connection: {
+        password: string;
+        uri: string;
+    };
+};
+export declare class ResticBackup {
+    readonly options: ResticOptions;
+    protected ntfy: Ntfy;
+    protected log: boolean | undefined;
+    readonly processes: {
+        name: string;
+        error?: Error;
+        stats: ResticBackupStats;
+    }[];
+    protected startTime: number;
+    readonly restic: Restic;
+    constructor(options: ResticOptions, ntfy: Ntfy, log: boolean | undefined);
+    run(input: ResticBackupPackage | ResticBackupPackage[]): Promise<void>;
+    protected runSingle(item: ResticBackupPackage): Promise<void>;
+}

package/lib/utils/restic-backup.js

@@ -0,0 +1,91 @@
+import { checkDiskSpace } from "./fs.js";
+import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { duration } from "@datatruck/cli/utils/date.js";
+import { isLocalDir } from "@datatruck/cli/utils/fs.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
+export class ResticBackup {
+    options;
+    ntfy;
+    log;
+    processes = [];
+    startTime;
+    restic;
+    constructor(options, ntfy, log) {
+        this.options = options;
+        this.ntfy = ntfy;
+        this.log = log;
+        this.startTime = Date.now();
+        this.restic = new Restic({
+            env: {
+                RESTIC_PASSWORD: options.connection.password,
+                RESTIC_REPOSITORY: options.connection.uri,
+            },
+            log,
+        });
+    }
+    async run(input) {
+        const items = Array.isArray(input) ? input : [input];
+        for (const item of items) {
+            await this.runSingle(item);
+        }
+    }
+    async runSingle(item) {
+        const now = Date.now();
+        let error;
+        const stats = {
+            bytes: 0,
+            files: 0,
+            diffBytes: undefined,
+        };
+        try {
+            if (isLocalDir(this.options.connection.uri) &&
+                !(await this.restic.checkRepository()))
+                await this.restic.exec(["init"]);
+            const targetPath = isLocalDir(this.options.connection.uri)
+                ? this.options.connection.uri
+                : undefined;
+            stats.diffBytes = await checkDiskSpace({
+                minFreeSpace: this.options.minFreeSpace,
+                minFreeSpacePath: targetPath ?? process.cwd(),
+                targetPath,
+                rutine: () => {
+                    const tags = {
+                        ...this.options.tags,
+                        package: item.name,
+                        tags: item.tags,
+                    };
+                    return this.restic.backup({
+                        tags: ResticRepository.createSnapshotTags(tags),
+                        paths: [item.path],
+                        exclude: item.exclude,
+                        onStream(data) {
+                            if (data.message_type === "summary") {
+                                stats.files = data.total_files_processed;
+                                stats.bytes = data.total_bytes_processed;
+                            }
+                        },
+                    });
+                },
+            });
+        }
+        catch (inError) {
+            error = inError;
+        }
+        this.processes.push({ name: item.name, error, stats });
+        await this.ntfy.send(`Backup`, {
+            "- Repository": this.options.name,
+            "- Package": item.name,
+            "- Size": formatBytes(stats.bytes),
+            ...(stats.diffBytes !== undefined && {
+                "- Size change": (stats.diffBytes > 0 ? "+" : "") + formatBytes(stats.diffBytes),
+            }),
+            "- Files": stats.files,
+            "- Duration": duration(Date.now() - now),
+            "- Error": error?.message,
+        }, {
+            priority: error ? "high" : "default",
+            tags: [error ? "red_circle" : "green_circle"],
+        });
+    }
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@datatruck/restic",
+  "version": "0.0.1",
+  "description": "Tool for creating and managing backups",
+  "homepage": "https://github.com/swordev/datatruck#readme",
+  "bugs": {
+    "url": "https://github.com/swordev/datatruck/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/swordev/datatruck"
+  },
+  "license": "MIT",
+  "author": {
+    "name": "Juanra GM",
+    "email": "juanrgm724@gmail.com",
+    "url": "https://github.com/juanrgm"
+  },
+  "type": "module",
+  "exports": {
+    ".": "./lib/index.js",
+    "./*": "./lib/*"
+  },
+  "main": "lib/index.js",
+  "bin": {
+    "datatruck-restic": "lib/bin.js",
+    "dttr": "lib/bin.js"
+  },
+  "files": [
+    "lib",
+    "config.schema.json"
+  ],
+  "dependencies": {
+    "ajv": "^8.17.1",
+    "commander": "^14.0.2",
+    "undici": "^7.18.2",
+    "@datatruck/cli": "0.41.5"
+  },
+  "engine": {
+    "node": ">=20.0.0"
+  },
+  "scripts": {
+    "build": "tsc -b && pnpm build:schema",
+    "build:schema": "node ./packages/cli/scripts/gen-schema.mjs",
+    "clean": "tsc -b --clean",
+    "watch": "tsc -b -w"
+  }
+}