@datatruck/restic 0.0.4 → 0.0.6

package/lib/actions/backup.d.ts

@@ -5,16 +5,11 @@ export type BackupOptions = {
     prune?: boolean;
 };
 export type CommonResticBackupTags = {
+    dt: boolean;
     id: string;
-    shortId: string;
-    hostname: string;
-    date: string;
-    vendor: string;
-    version: string;
 };
 export type ResticBackupTags = CommonResticBackupTags & {
-    package: string;
-    tags?: string[];
+    pkg: string;
 };
 export declare class Backup extends Action {
     protected runSingle(repoName: string, pkgName: string, tags: CommonResticBackupTags): Promise<{

package/lib/actions/backup.js

@@ -1,17 +1,15 @@
 import { createRunner, safeRun } from "../utils/async.js";
 import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
 import { MySQLDump } from "../utils/mysql.js";
+import { randomString } from "../utils/string.js";
+import { stringifyTags } from "../utils/tags.js";
 import { Action } from "./base.js";
 import { Prune } from "./prune.js";
-import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
-import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { progressPercent } from "@datatruck/cli/utils/math.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
 import { match } from "@datatruck/cli/utils/string.js";
-import { randomUUID } from "crypto";
-import { hostname } from "os";
 export class Backup extends Action {
     async runSingle(repoName, pkgName, tags) {
         const repo = this.cm.findRepository(repoName);
@@ -23,25 +21,35 @@ export class Backup extends Action {
                 RESTIC_REPOSITORY: repo.uri,
             },
         });
+        let logId;
         let space;
         let snapshotsAmount;
         let bytes = 0;
         let files = 0;
         return await createRunner(async () => {
+            logId = await this.ntfy.send("Backup", {
+                Repository: repo.name,
+                Package: pkg.name,
+            });
             await restic.tryInit();
             const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
             space = await checkDiskSpace({
                 minFreeSpace: this.config.minFreeSpace,
                 minFreeSpacePath: targetPath ?? process.cwd(),
                 targetPath,
-                rutine: () => {
-                    const pkgTags = {
-                        ...tags,
-                        package: pkg.name,
-                        tags: [],
-                    };
-                    return restic.backup({
-                        tags: ResticRepository.createSnapshotTags(pkgTags),
+                rutine: async () => {
+                    const last = await restic.snapshots({
+                        latest: 1,
+                        host: this.config.hostname,
+                        tags: stringifyTags({ pkg: pkg.name }),
+                    });
+                    return await restic.backup({
+                        parent: last.at(0)?.id,
+                        host: this.config.hostname,
+                        tags: stringifyTags({
+                            ...tags,
+                            pkg: pkg.name,
+                        }),
                         paths: [pkg.path],
                         exclude: pkg.exclude,
                         onStream(data) {
@@ -55,9 +63,7 @@ export class Backup extends Action {
             });
             const snapshots = await restic.snapshots({
                 json: true,
-                tags: [
-                    ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkg.name),
-                ],
+                tags: stringifyTags({ pkg: pkg.name }),
             });
             snapshotsAmount = snapshots.length;
         }).start(async (data) => {
@@ -70,7 +76,7 @@ export class Backup extends Action {
                 Snapshots: snapshotsAmount,
                 Duration: data.duration,
                 Error: data.error?.message,
-            }, data.error);
+            }, { error: data.error, logId });
             return {
                 error: data.error,
                 files,
@@ -87,12 +93,17 @@ export class Backup extends Action {
         const sqlDumps = [];
         const backups = [];
         let localRepositoryPaths = [];
+        const tags = {
+            dt: true,
+            id: randomString(8),
+        };
         await createRunner(async () => {
             const repositories = this.cm.filterRepositories(options.repositories);
             const packages = this.cm.filterPackages(options.packages);
             const packageNames = packages.map((p) => p.name);
             const tasks = this.filterTasks(packageNames);
             await this.ntfy.send(`Backup start`, {
+                Id: tags.id,
                 Repositories: repositories.length,
                 Packages: packageNames.length,
                 Tasks: tasks?.length,
@@ -100,16 +111,6 @@ export class Backup extends Action {
             localRepositoryPaths = repositories
                 .filter((repo) => isLocalDir(repo.uri))
                 .map((repo) => repo.uri);
-            const tags = {
-                id: randomUUID().replaceAll("-", ""),
-                get shortId() {
-                    return this.id.slice(0, 8);
-                },
-                hostname: this.config.hostname ?? hostname(),
-                date: new Date().toISOString(),
-                vendor: "dtt-restic",
-                version: "1",
-            };
             for (const task of tasks ?? []) {
                 if (task.type === "mysql-dump") {
                     const mysqlDump = new MySQLDump({
@@ -144,8 +145,13 @@ export class Backup extends Action {
                 }
             }
         }).start(async (data) => {
-            for (const sqlDump of sqlDumps)
-                await sqlDump.cleanup();
+            try {
+                for (const sqlDump of sqlDumps)
+                    await sqlDump.cleanup();
+            }
+            catch (error) {
+                console.error(error);
+            }
             const summary = backups.reduce((acc, p) => {
                 if (!acc[p.pkgName])
                     acc[p.pkgName] = {
@@ -174,6 +180,7 @@ export class Backup extends Action {
             if (diskStats.error)
                 console.error(diskStats.error);
             await this.ntfy.send("Backup end", {
+                Id: tags.id,
                 Duration: data.duration,
                 Size: formatBytes(size),
                 Error: data.error?.message,
@@ -181,7 +188,7 @@ export class Backup extends Action {
                     !!diskStats.result?.length && { key: "Disk stats", value: "" },
                     ...(diskStats.result?.map((p) => ({
                         key: p.name,
-                        value: `${formatBytes(p.free)}/${formatBytes(p.total)} (${progressPercent(p.total, p.free)}%)`,
+                        value: `${formatBytes(p.occupied)}/${formatBytes(p.total)} (${progressPercent(p.total, p.occupied)}%)`,
                         level: 1,
                     })) || []),
                     !!sqlDumpProcesses.length && { key: "SQL Dumps", value: "" },
@@ -197,7 +204,7 @@ export class Backup extends Action {
                         level: 1,
                     })),
                 ],
-            }, error);
+            }, { error });
             if (options.prune) {
                 await new Prune(this.config, this.global).run({
                     packages: options.packages,

package/lib/actions/copy.d.ts

@@ -8,7 +8,6 @@ export type CopyOptions = {
 export declare class Copy extends Action {
     protected initializedRepos: Set<string>;
     private findSnapshots;
-    private findPackageTag;
     runSingle(options: {
         source: string;
         target: string;

package/lib/actions/copy.js

@@ -1,9 +1,8 @@
 import { createRunner } from "../utils/async.js";
 import { checkDiskSpace } from "../utils/fs.js";
+import { parseTags, stringifyTags } from "../utils/tags.js";
 import { Action } from "./base.js";
 import { Prune } from "./prune.js";
-import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
-import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { Restic } from "@datatruck/cli/utils/restic.js";
@@ -22,23 +21,16 @@ export class Copy extends Action {
             latest: 1,
             group: ["path"],
             tags: packages
-                ? packages.map((name) => ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, name))
+                ? packages.flatMap((name) => stringifyTags({ pkg: name }))
                 : undefined,
         }));
         return snapshots.flatMap((s) => s.snapshots);
     }
-    findPackageTag(inTags) {
-        const tags = inTags
-            .map((t) => ResticRepository.parseSnapshotTag(t))
-            .filter((t) => !!t);
-        return tags.find((t) => t.name === SnapshotTagEnum.PACKAGE);
-    }
     async runSingle(options) {
         const { snapshot } = options;
         const targetRepo = this.cm.findRepository(options.target);
         const sourceRepo = this.cm.findRepository(options.source);
-        const pkgTag = this.findPackageTag(snapshot.tags);
-        const packageName = pkgTag?.value;
+        const packageName = parseTags(snapshot.tags).pkg;
         const targetPath = isLocalDir(targetRepo.uri) ? targetRepo.uri : undefined;
         const target = new Restic({
             log: this.verbose,
@@ -48,8 +40,14 @@ export class Copy extends Action {
                 ["GODEBUG"]: "http2client=0",
             },
         });
+        let logId;
         let space;
         await createRunner(async () => {
+            logId = await this.ntfy.send("Copy", {
+                Id: snapshot.short_id,
+                Source: sourceRepo.name,
+                Target: targetRepo.name,
+            });
             if (!this.initializedRepos.has(targetRepo.name)) {
                 await target.tryInit();
                 this.initializedRepos.add(targetRepo.name);
@@ -74,7 +72,7 @@ export class Copy extends Action {
                 }),
                 Duration: data.duration,
                 Error: data.error?.message,
-            }, data.error);
+            }, { error: data.error, logId });
         });
         return space?.diff ?? 0;
     }
@@ -107,7 +105,7 @@ export class Copy extends Action {
                 }),
                 Duration: data.duration,
                 Error: data.error?.message,
-            }, data.error);
+            }, { error: data.error });
             if (options.prune) {
                 await new Prune(this.config, this.global).run({
                     packages: options.packages,

package/lib/actions/init.js

@@ -20,7 +20,7 @@ export class Init extends Action {
                 Exists: exists ? "yes" : "no",
                 Duration: data.duration,
                 Error: data.error?.message,
-            }, data.error);
+            }, { error: data.error });
         });
     }
     async run(options) {

package/lib/actions/prune.js

@@ -1,18 +1,22 @@
 import { createRunner, safeRun } from "../utils/async.js";
 import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
+import { parseTags, stringifyTags } from "../utils/tags.js";
 import { Action } from "./base.js";
-import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
-import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { progressPercent } from "@datatruck/cli/utils/math.js";
 import { match } from "@datatruck/cli/utils/string.js";
 export class Prune extends Action {
     async runSingle(repoName, pkgName, policy) {
+        let logId;
         let stats;
         let space;
         let removed;
         await createRunner(async () => {
+            logId = await this.ntfy.send("Prune", {
+                Repository: repoName,
+                Package: pkgName,
+            });
             const [restic, repo] = this.cm.createRestic(repoName, this.verbose);
             const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
             space = await checkDiskSpace({
@@ -24,9 +28,7 @@ export class Prune extends Action {
                         json: true,
                         prune: true,
                         args: ["--group-by", ""],
-                        tag: [
-                            ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkgName),
-                        ],
+                        tag: stringifyTags({ pkg: pkgName }),
                     });
                     stats = {
                         keep: results.at(0)?.keep?.length ?? 0,
@@ -49,7 +51,7 @@ export class Prune extends Action {
                 }),
                 Duration: data.duration,
                 Error: data.error?.message,
-            }, data.error);
+            }, { error: data.error, logId });
         });
         return {
             diffSize: space?.diff ?? 0,
@@ -61,10 +63,7 @@ export class Prune extends Action {
         if (!(await restic.checkRepository()))
             return [];
         const snapshots = await restic.snapshots({ json: true });
-        const packages = new Set(snapshots.map((s) => {
-            const tags = ResticRepository.parseSnapshotTags(s.tags ?? []);
-            return tags[SnapshotTagEnum.PACKAGE];
-        }));
+        const packages = new Set(snapshots.map((s) => parseTags(s.tags ?? []).pkg));
         return [...packages].filter((v) => typeof v === "string");
     }
     async run(options) {
@@ -113,7 +112,7 @@ export class Prune extends Action {
                     !!diskStats.result?.length && { key: "Disk stats", value: "" },
                     ...(diskStats.result?.map((p) => ({
                         key: p.name,
-                        value: `${formatBytes(p.free)}/${formatBytes(p.total)} (${progressPercent(p.total, p.free)}%)`,
+                        value: `${formatBytes(p.occupied)}/${formatBytes(p.total)} (${progressPercent(p.total, p.occupied)}%)`,
                         level: 1,
                     })) || []),
                 ],

package/lib/actions/run.js

@@ -1,11 +1,16 @@
 import { Action } from "./base.js";
+import { logExec } from "@datatruck/cli/utils/cli.js";
 import { spawnSync } from "child_process";
 export class Run extends Action {
     async run(options) {
         const [restic] = this.cm.createRestic(options.repository, this.verbose);
-        const p = restic["createProcess"](options.args, { $log: true });
-        const exit = spawnSync(p["command"], p["argv"]?.map((v) => v.toString()), { stdio: "inherit", env: p["options"]?.env });
-        if (exit.status)
-            process.exit(exit.status);
+        if (this.verbose)
+            logExec("restic", options.args, restic.options.env);
+        const p = spawnSync("restic", options.args, {
+            stdio: "inherit",
+            env: { ...process.env, ...restic.options.env },
+        });
+        if (p.status)
+            process.exit(p.status);
     }
 }

package/lib/bin.d.ts

@@ -1 +1,2 @@
+#!/usr/bin/env node
 export {};

package/lib/bin.js

@@ -1,2 +1,3 @@
+#!/usr/bin/env node
 import { createBin } from "./create-bin.js";
 createBin().parse();

package/lib/create-bin.js

@@ -34,14 +34,17 @@ export function createBin(inConfig) {
         await create.run(options);
     });
     program
-        .command("run", {})
+        .command("run")
         .description("Run arbitrary restic command")
-        .argument("<repository>", "Repository name")
+        .option("-r, --repository <name>", "Repository name")
         .argument("[args...]", "Restic arguments")
         .allowUnknownOption()
         .allowExcessArguments()
-        .action(async (repository, args) => {
+        .action(async (args, options) => {
         const { config, globalOptions } = await load();
+        if (!options.repository && config.repositories.length !== 1)
+            throw new Error("Repository name is required");
+        const repository = options.repository ?? config.repositories[0].name;
         const run = new Run(config, globalOptions);
         await run.run({ repository, args });
     });
@@ -96,15 +99,17 @@ export function createBin(inConfig) {
     program.parse = function (args, options) {
         if (!args)
             args = process.argv;
-        const [node, script, ...rest] = args;
-        const commandIndex = rest.findIndex((v) => !v.startsWith("-"));
-        const command = rest[commandIndex];
+        const [node, script, command, ...rest] = args;
         if (command === "run") {
-            args = [
-                node,
-                script,
-                ...rest.flatMap((value, index) => commandIndex === index ? [value, "--"] : value),
-            ];
+            const repositoryIndex = rest.findIndex((v) => v === "-r" || v === "--repository");
+            if (repositoryIndex !== -1) {
+                const repositoryName = rest[repositoryIndex + 1];
+                const others = rest.filter((_, i) => i !== repositoryIndex && i !== repositoryIndex + 1);
+                args = [node, script, command, "-r", repositoryName, "--", ...others];
+            }
+            else {
+                args = [node, script, command, "--", ...rest];
+            }
         }
         return parse(args, options);
     };

package/lib/utils/fs.d.ts

@@ -12,5 +12,6 @@ export declare function getDiskName(inPath: string): Promise<string>;
 export declare function fetchMultipleDiskStats(paths: string[]): Promise<{
     name: string;
     free: number;
+    occupied: number;
     total: number;
 }[]>;

package/lib/utils/fs.js

@@ -54,6 +54,7 @@ export async function fetchMultipleDiskStats(paths) {
         result.push({
             name: await getDiskName(path),
             ...stats,
+            occupied: stats.total - stats.free,
         });
     }
     return result;

package/lib/utils/mysql.js

@@ -54,6 +54,7 @@ import { runParallel } from "@datatruck/cli/utils/async.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { duration } from "@datatruck/cli/utils/date.js";
 import { ensureFreeDiskSpace, fetchDiskStats, } from "@datatruck/cli/utils/fs.js";
+import { progressPercent } from "@datatruck/cli/utils/math.js";
 import { createMysqlCli } from "@datatruck/cli/utils/mysql.js";
 import { match } from "@datatruck/cli/utils/string.js";
 import { existsSync } from "fs";
@@ -101,7 +102,7 @@ export class MySQLDump {
         for (const outPath of this.outPaths) {
             const parentFolder = basename(dirname(outPath));
             if (!parentFolder.startsWith("sql-dump"))
-                throw new Error(`sql-dump out dir must begins with 'sql-dump': ${outPath}`);
+                throw new Error(`sql-dump out dir base name must begins with 'sql-dump': ${outPath}`);
             if (existsSync(outPath))
                 await rm(outPath, { recursive: true });
             if (!init)
@@ -129,7 +130,20 @@ export class MySQLDump {
         for (const item of items)
             this.outPaths.add(dirname(item.out));
         await this.cleanup(true);
+        let logId;
+        let logIntervalId;
         try {
+            logId = await this.ntfy.send("SQL dump", {
+                Name: item.name,
+            });
+            logIntervalId = setInterval(async () => {
+                await this.ntfy.send("SQL dump", {
+                    Name: item.name,
+                    Size: formatBytes(stats.bytes),
+                    Files: `${stats.files}/${items.length} (${progressPercent(items.length, stats.files)}%)`,
+                    Duration: duration(Date.now() - now),
+                }, { logId });
+            }, 30_000);
             await runParallel({
                 items,
                 concurrency: this.options.concurrency ?? 1,
@@ -153,17 +167,20 @@ export class MySQLDump {
         catch (inError) {
             error = inError;
         }
+        finally {
+            clearInterval(logIntervalId);
+        }
         this.processes.push({
             name: item.name,
             error,
             stats,
         });
         await this.ntfy.send("SQL dump", {
-            Package: item.name,
+            Name: item.name,
             Size: formatBytes(stats.bytes),
             Files: stats.files,
             Duration: duration(Date.now() - now),
             Error: error?.message,
-        }, error);
+        }, { error, logId });
     }
 }

package/lib/utils/ntfy.d.ts

@@ -21,6 +21,9 @@ export declare class Ntfy {
     private formatTitle;
     private formatMessage;
     private formatMessageObject;
-    send(inTitle: string, message: MessageObject, error?: Error | boolean): Promise<void>;
+    send(inTitle: string, message: MessageObject, options?: {
+        error?: Error | boolean;
+        logId?: string;
+    }): Promise<string | undefined>;
 }
 export {};

package/lib/utils/ntfy.js

@@ -38,31 +38,37 @@ export class Ntfy {
         })
             .join("\n");
     }
-    async send(inTitle, message, error) {
+    async send(inTitle, message, options = {}) {
         const title = this.formatTitle(inTitle);
         const body = this.formatMessageObject(message);
         const lines = [title, body].filter((v) => v.length);
         if (lines.length)
             console.info([...lines, ""].join("\n"));
-        const options = {
-            priority: error ? "high" : "default",
-            tags: [error ? "red_circle" : "green_circle"],
+        const ntfyOptions = {
+            priority: options.error ? "high" : "default",
+            tags: [options.error ? "red_circle" : "green_circle"],
         };
         try {
-            if (this.options.token)
-                await fetch(`https://ntfy.sh/${this.options.token}`, {
+            if (this.options.token) {
+                const token = options.logId
+                    ? `${this.options.token}/${options.logId}`
+                    : this.options.token;
+                const response = await fetch(`https://ntfy.sh/${token}`, {
                     dispatcher: this.agent,
                     method: "POST",
                     body: unstyle(body),
                     headers: {
                         Markdown: "yes",
                         Title: unstyle(title),
-                        Priority: options.priority ?? "default",
-                        ...(options.tags && {
-                            Tags: options.tags?.join(","),
+                        Priority: ntfyOptions.priority ?? "default",
+                        ...(ntfyOptions.tags && {
+                            Tags: ntfyOptions.tags?.join(","),
                         }),
                     },
                 });
+                const json = (await response.json());
+                return json.id;
+            }
             await setTimeout(this.options.delay ?? 800);
         }
         catch (error) {

package/lib/utils/string.d.ts

@@ -1 +1,2 @@
 export declare function unstyle(str: string): string;
+export declare function randomString(length: number): string;

package/lib/utils/string.js

@@ -1,3 +1,13 @@
+import { randomInt } from "crypto";
 export function unstyle(str) {
     return str.replace(/\x1B\[[0-9;]*m/g, "");
 }
+const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+export function randomString(length) {
+    let string = "";
+    for (let i = 0; i < length; i++) {
+        const index = randomInt(0, charset.length);
+        string += charset[index];
+    }
+    return string;
+}

package/lib/utils/tags.d.ts

@@ -0,0 +1,3 @@
+export declare function stringifyTags(inTags: Record<string, any>): string[];
+export declare function parseTagPrefix(tag: string): string | undefined;
+export declare function parseTags(inTags: string[]): Record<string, any>;

package/lib/utils/tags.js

@@ -0,0 +1,37 @@
+const prefix = "dt";
+const prefixSep = "-";
+const valueSep = ":";
+const fullPrefix = `${prefix}${prefixSep}`;
+export function stringifyTags(inTags) {
+    const tags = [];
+    for (const [key, value] of Object.entries(inTags)) {
+        const tagName = key === prefix ? key : `${prefix}${prefixSep}${key}`;
+        tags.push(value === true ? tagName : `${tagName}${valueSep}${value}`);
+    }
+    return tags;
+}
+export function parseTagPrefix(tag) {
+    if (tag === prefix) {
+        return tag;
+    }
+    else if (tag.startsWith(fullPrefix)) {
+        return tag.slice(fullPrefix.length);
+    }
+}
+export function parseTags(inTags) {
+    const tags = {};
+    for (const tag of inTags) {
+        const str = parseTagPrefix(tag);
+        if (!str) {
+            continue;
+        }
+        else if (str.includes(valueSep)) {
+            const [tagName, tagValue] = str.split(valueSep);
+            tags[tagName] = tagValue ?? "";
+        }
+        else {
+            tags[str] = true;
+        }
+    }
+    return tags;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/restic",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {
@@ -34,7 +34,7 @@
     "ajv": "^8.17.1",
     "commander": "^14.0.2",
     "undici": "^7.18.2",
-    "@datatruck/cli": "0.41.6"
+    "@datatruck/cli": "0.41.7"
   },
   "engine": {
     "node": ">=20.0.0"